Here are the examples of the java api com.google.devtools.build.lib.actions.FileArtifactValue taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
60 Examples
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testUptodateCheckDirectory() throws Exception {
// For now, we don't attempt to detect changes to directories.
Path path = scratchDir("/dir", 0L);
FileArtifactValue value = createForTesting(path);
replacedertThat(value.wasModifiedSinceDigest(path)).isFalse();
path.delete();
clock.advanceMillis(1);
replacedertThat(value.wasModifiedSinceDigest(path)).isFalse();
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testUptodateChangeFileToDirectory() throws Exception {
// For now, we don't attempt to detect changes to directories.
Path path = scratchFile("/dir/file", 0L, "");
FileArtifactValue value = createForTesting(path);
replacedertThat(value.wasModifiedSinceDigest(path)).isFalse();
// If we only check ctime, then we need to change the clock here, or we get a ctime match on the
// stat.
path.delete();
path.createDirectoryAndParents();
clock.advanceMillis(1);
replacedertThat(value.wasModifiedSinceDigest(path)).isTrue();
}
19
View Complete Implementation : ArtifactFunction.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private static TreeArtifactValue createTreeArtifactValueFromActionKey(ArtifactDependencies artifactDependencies, Environment env) throws InterruptedException {
// Request the list of expanded actions from the ActionTemplate.
ActionTemplateExpansion actionTemplateExpansion = artifactDependencies.getActionTemplateExpansion(env);
if (actionTemplateExpansion == null) {
// The expanded actions are not yet available.
return null;
}
ActionTemplateExpansionValue expansionValue = actionTemplateExpansion.getValue();
ImmutableList<ActionLookupData> expandedActionExecutionKeys = actionTemplateExpansion.getExpandedActionExecutionKeys();
Map<SkyKey, SkyValue> expandedActionValueMap = env.getValues(expandedActionExecutionKeys);
if (env.valuesMissing()) {
// The execution values of the expanded actions are not yet all available.
return null;
}
// Aggregate the ArtifactValues for individual TreeFileArtifacts into a TreeArtifactValue for
// the parent TreeArtifact.
ImmutableMap.Builder<TreeFileArtifact, FileArtifactValue> map = ImmutableMap.builder();
for (ActionLookupData actionKey : expandedActionExecutionKeys) {
ActionExecutionValue actionExecutionValue = (ActionExecutionValue) Preconditions.checkNotNull(expandedActionValueMap.get(actionKey), "Missing tree value: %s %s %s", artifactDependencies, expansionValue, expandedActionValueMap);
Iterable<TreeFileArtifact> treeFileArtifacts = Iterables.transform(Iterables.filter(actionExecutionValue.getAllFileValues().keySet(), artifact -> {
Preconditions.checkState(artifact.hasParent(), "No parent: %s %s %s", artifact, actionExecutionValue, artifactDependencies);
return artifact.getParent().equals(artifactDependencies.artifact);
}), artifact -> (TreeFileArtifact) artifact);
Preconditions.checkState(!Iterables.isEmpty(treeFileArtifacts), "Action denoted by %s does not output TreeFileArtifact from %s", actionKey, artifactDependencies);
for (TreeFileArtifact treeFileArtifact : treeFileArtifacts) {
FileArtifactValue value = createSimpleFileArtifactValue(treeFileArtifact, actionExecutionValue);
map.put(treeFileArtifact, value);
}
}
// Return the aggregated TreeArtifactValue.
return TreeArtifactValue.create(map.build());
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
// Empty files are the same as normal files -- mtime is not stored.
@Test
public void testEmptyFile() throws Exception {
Path path = scratchFile("/root/empty", 1L, "");
path.setLastModifiedTime(1L);
FileArtifactValue value = createForTesting(path);
replacedertThat(value.getDigest()).isEqualTo(path.getDigest());
replacedertThat(value.getSize()).isEqualTo(0L);
replacedertThrows("mtime for non-empty file should not be stored.", UnsupportedOperationException.clreplaced, () -> value.getModifiedTime());
}
19
View Complete Implementation : ActionMetadataHandlerTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void withFilesetInput() throws Exception {
// This value should be mapped
FileArtifactValue directoryFav = FileArtifactValue.createForDirectoryWithMtime(10L);
// This value should not be mapped
FileArtifactValue regularFav = FileArtifactValue.createForVirtualActionInput(new byte[] { 1, 2, 3, 4 }, 10L);
// This value should not be mapped
HasDigest.ByteStringDigest byteStringDigest = new ByteStringDigest(new byte[] { 2, 3, 4 });
ImmutableMap<Artifact, ImmutableList<FilesetOutputSymlink>> filesetMap = createFilesetOutputSymlinkMap(directoryFav, regularFav, byteStringDigest);
ActionMetadataHandler handler = new ActionMetadataHandler(new ActionInputMap(0), filesetMap, /* missingArtifactsAllowed= */
false, /* outputs= */
ImmutableList.of(), /* tsgm= */
null, ArtifactPathResolver.forExecRoot(outputRoot.getRoot().asPath()), new MinimalOutputStore(), outputRoot.getRoot().asPath());
ImmutableMap<PathFragment, FileArtifactValue> filesetMapping = handler.getFilesetMapping();
replacedertThat(filesetMapping).hreplacedize(1);
PathFragment filesetPathFragment = filesetMapping.keySet().iterator().next();
replacedertThat(filesetPathFragment.getPathString()).isEqualTo("target/bytestring2");
replacedertThat(filesetMapping.get(filesetPathFragment)).isEqualTo(regularFav);
}
19
View Complete Implementation : ActionMetadataHandler.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private TreeArtifactValue constructTreeArtifactValue(Collection<TreeFileArtifact> contents) throws IOException {
Map<TreeFileArtifact, FileArtifactValue> values = Maps.newHashMapWithExpectedSize(contents.size());
for (TreeFileArtifact treeFileArtifact : contents) {
FileArtifactValue fileMetadata = store.getArtifactData(treeFileArtifact);
// This is similar to what's present in getRealMetadataForArtifact, except
// we get back the ArtifactFileMetadata, not the metadata.
// We do not cache exceptions besides nonexistence here, because it is unlikely that the
// file will be requested from this cache too many times.
if (fileMetadata == null) {
try {
fileMetadata = constructFileArtifactValue(treeFileArtifact, /*statNoFollow=*/
null);
} catch (FileNotFoundException e) {
String errorMessage = String.format("Failed to resolve relative path %s inside TreeArtifact %s. " + "The replacedociated file is either missing or is an invalid symlink.", treeFileArtifact.getParentRelativePath(), treeFileArtifact.getParent().getExecPathString());
throw new IOException(errorMessage, e);
}
// A minor hack: maybeStoreAdditionalData will force the data to be stored via
// store.putAdditionalOutputData, if the underlying OutputStore supports it.
fileMetadata = maybeStoreAdditionalData(treeFileArtifact, fileMetadata, null);
}
values.put(treeFileArtifact, fileMetadata);
}
return TreeArtifactValue.create(values);
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testIsMarkerValue_notMarker() throws Exception {
FileArtifactValue value = createForTesting(scratchFile("/dir/artifact1", 0L, "content"));
replacedertThat(value.isMarkerValue()).isFalse();
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testDirectory() throws Exception {
Path path = scratchDir("/dir", /*mtime=*/
1L);
FileArtifactValue value = createForTesting(path);
replacedertThat(value.getDigest()).isNull();
replacedertThat(value.getModifiedTime()).isEqualTo(1L);
}
19
View Complete Implementation : FilesystemValueCheckerTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private ActionExecutionValue actionValueWithTreeArtifacts(List<TreeFileArtifact> contents) {
Map<Artifact, FileArtifactValue> fileData = new HashMap<>();
Map<Artifact, Map<TreeFileArtifact, FileArtifactValue>> directoryData = new HashMap<>();
for (TreeFileArtifact output : contents) {
try {
Map<TreeFileArtifact, FileArtifactValue> dirDatum = directoryData.get(output.getParent());
if (dirDatum == null) {
dirDatum = new HashMap<>();
directoryData.put(output.getParent(), dirDatum);
}
Path path = output.getPath();
FileArtifactValue noDigest = ActionMetadataHandler.fileArtifactValueFromArtifact(output, FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)), null);
FileArtifactValue withDigest = FileArtifactValue.createFromInjectedDigest(noDigest, path.getDigest(), !output.isConstantMetadata());
dirDatum.put(output, withDigest);
fileData.put(output, withDigest);
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
for (Map.Entry<Artifact, Map<TreeFileArtifact, FileArtifactValue>> dirDatum : directoryData.entrySet()) {
treeArtifactData.put(dirDatum.getKey(), TreeArtifactValue.create(dirDatum.getValue()));
}
return ActionExecutionValue.create(fileData, treeArtifactData, /*outputSymlinks=*/
null, /*discoveredModules=*/
null, /*actionDependsOnBuildId=*/
false);
}
19
View Complete Implementation : RemoteActionFileSystem.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
// -------------------- File Permissions --------------------
@Override
protected boolean isReadable(Path path) throws IOException {
FileArtifactValue m = getRemoteInputMetadata(path);
return m != null || super.isReadable(path);
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testUptodateCheck() throws Exception {
Path path = scratchFile("/dir/artifact1", 0L, "content");
FileArtifactValue value = createForTesting(path);
clock.advanceMillis(1);
replacedertThat(value.wasModifiedSinceDigest(path)).isFalse();
clock.advanceMillis(1);
replacedertThat(value.wasModifiedSinceDigest(path)).isFalse();
clock.advanceMillis(1);
// Changing mtime implicitly updates ctime.
path.setLastModifiedTime(123);
replacedertThat(value.wasModifiedSinceDigest(path)).isTrue();
clock.advanceMillis(1);
replacedertThat(value.wasModifiedSinceDigest(path)).isTrue();
}
19
View Complete Implementation : ActionMetadataHandlerTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void withNonArtifactInput() throws Exception {
ActionInput input = ActionInputHelper.fromPath("foo/bar");
FileArtifactValue metadata = FileArtifactValue.createForNormalFile(new byte[] { 1, 2, 3 }, /*proxy=*/
null, 10L, /*isShareable=*/
true);
ActionInputMap map = new ActionInputMap(1);
map.putWithNoDepOwner(input, metadata);
replacedertThat(map.getMetadata(input)).isEqualTo(metadata);
ActionMetadataHandler handler = new ActionMetadataHandler(map, ImmutableMap.of(), /* missingArtifactsAllowed= */
false, /* outputs= */
ImmutableList.of(), /* tsgm= */
null, ArtifactPathResolver.IDENreplacedY, new MinimalOutputStore(), outputRoot.getRoot().asPath());
replacedertThat(handler.getMetadata(input)).isNull();
}
19
View Complete Implementation : ActionMetadataHandler.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private static FileArtifactValue metadataFromValue(FileArtifactValue value) throws FileNotFoundException {
if (value == FileArtifactValue.MISSING_FILE_MARKER || value == FileArtifactValue.OMITTED_FILE_MARKER) {
throw new FileNotFoundException();
}
return value;
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testUptodateCheckDeleteFile() throws Exception {
Path path = scratchFile("/dir/artifact1", 0L, "content");
FileArtifactValue value = createForTesting(path);
replacedertThat(value.wasModifiedSinceDigest(path)).isFalse();
path.delete();
replacedertThat(value.wasModifiedSinceDigest(path)).isTrue();
}
19
View Complete Implementation : RecursiveFilesystemTraversalFunction.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Transform the HasDigest to the appropriate type based on the current state of the digest. If
* fsVal is type RegularFileStateValue or FileArtifactValue and has a valid digest value, then we
* want to convert it to a new FileArtifactValue type. Otherwise if they are of the two
* forementioned types but do not have a digest, then we will create a FileArtifactValue using its
* {@link Path}. Otherwise we will fingerprint the digest and return it as a new {@link
* HasDigest.ByteStringDigest} object.
*
* @param fsVal - the HasDigest value that was in the graph.
* @param path - the Path of the digest.
* @return transformed HasDigest value based on the digest field and object type.
*/
@VisibleForTesting
static HasDigest withDigest(HasDigest fsVal, Path path) throws IOException {
if (fsVal instanceof FileStateValue) {
FileStateValue fsv = (FileStateValue) fsVal;
if (fsv instanceof RegularFileStateValue) {
RegularFileStateValue rfsv = (RegularFileStateValue) fsv;
return rfsv.getDigest() != null ? // If we have the digest, then simply convert it with the digest value.
FileArtifactValue.createForVirtualActionInput(rfsv.getDigest(), rfsv.getSize()) : // Otherwise, create a file FileArtifactValue (RegularFileArtifactValue) based on the
// path and size.
FileArtifactValue.createForNormalFileUsingPath(path, rfsv.getSize());
}
return new HasDigest.ByteStringDigest(fsv.getValueFingerprint().toByteArray());
} else if (fsVal instanceof FileArtifactValue) {
FileArtifactValue fav = ((FileArtifactValue) fsVal);
if (fav.getDigest() != null) {
return fav;
}
// In the case there is a directory, the HasDigest value should not be converted. Otherwise,
// if the HasDigest value is a file, convert it using the Path and size values.
return fav.getType().isFile() ? FileArtifactValue.createForNormalFileUsingPath(path, fav.getSize()) : new HasDigest.ByteStringDigest(fav.getValueFingerprint().toByteArray());
}
return fsVal;
}
19
View Complete Implementation : RemoteActionFileSystem.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
// -------------------- Symlinks --------------------
@Override
protected PathFragment readSymbolicLink(Path path) throws IOException {
FileArtifactValue m = getRemoteInputMetadata(path);
if (m != null) {
// We don't support symlinks as remote action outputs.
throw new IOException(path + " is not a symbolic link");
}
return super.readSymbolicLink(path);
}
19
View Complete Implementation : ArtifactFunction.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Create {@link FileArtifactValue} for artifact that must be non-middleman non-tree derived
* artifact.
*/
static FileArtifactValue createSimpleFileArtifactValue(Artifact.DerivedArtifact artifact, ActionExecutionValue actionValue) {
Preconditions.checkState(!artifact.isMiddlemanArtifact(), "%s %s", artifact, actionValue);
Preconditions.checkState(!artifact.isTreeArtifact(), "%s %s", artifact, actionValue);
FileArtifactValue value = actionValue.getArtifactValue(artifact);
if (value != null) {
return value;
}
FileArtifactValue data = Preconditions.checkNotNull(actionValue.getArtifactValue(artifact), "%s %s", artifact, actionValue);
Preconditions.checkNotNull(data.getDigest(), "Digest should already have been calculated for %s (%s)", artifact, data);
// Directories are special-cased because their mtimes are used, so should have been constructed
// during execution of the action (in ActionMetadataHandler#maybeStoreAdditionalData).
Preconditions.checkState(data.getType() == FileStateType.REGULAR_FILE || data.getType() == FileStateType.SYMLINK, "Should be file or symlink %s (%s)", artifact, data);
return data;
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testNoMtimeIfNonemptyFile() throws Exception {
Path path = scratchFile("/root/non-empty", 1L, "abc");
FileArtifactValue value = createForTesting(path);
replacedertThat(value.getDigest()).isEqualTo(path.getDigest());
replacedertThat(value.getSize()).isEqualTo(3L);
replacedertThrows("mtime for non-empty file should not be stored.", UnsupportedOperationException.clreplaced, () -> value.getModifiedTime());
}
19
View Complete Implementation : RemoteActionFileSystem.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Override
protected boolean isWritable(Path path) throws IOException {
FileArtifactValue m = getRemoteInputMetadata(path);
return m != null || super.isWritable(path);
}
19
View Complete Implementation : FileArtifactValueTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testCtimeInEquality() throws Exception {
Path path = scratchFile("/dir/artifact1", 0L, "content");
FileArtifactValue before = createForTesting(path);
clock.advanceMillis(1);
path.chmod(0777);
FileArtifactValue after = createForTesting(path);
replacedertThat(before).isNotEqualTo(after);
}
19
View Complete Implementation : ActionMetadataHandlerTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void getMetadataFromFilesetMapping() throws Exception {
FileArtifactValue directoryFav = FileArtifactValue.createForDirectoryWithMtime(10L);
FileArtifactValue regularFav = FileArtifactValue.createForVirtualActionInput(new byte[] { 1, 2, 3, 4 }, 10L);
HasDigest.ByteStringDigest byteStringDigest = new ByteStringDigest(new byte[] { 2, 3, 4 });
ImmutableMap<Artifact, ImmutableList<FilesetOutputSymlink>> filesetMap = createFilesetOutputSymlinkMap(directoryFav, regularFav, byteStringDigest);
ActionMetadataHandler handler = new ActionMetadataHandler(new ActionInputMap(0), filesetMap, /* missingArtifactsAllowed= */
false, /* outputs= */
ImmutableList.of(), /* tsgm= */
null, ArtifactPathResolver.forExecRoot(outputRoot.getRoot().asPath()), new MinimalOutputStore(), outputRoot.getRoot().asPath());
replacedertThat(handler.getMetadata(ActionInputHelper.fromPath("/output/bin/target/bytestring1"))).isNull();
replacedertThat(handler.getMetadata(ActionInputHelper.fromPath("/output/bin/target/bytestring2"))).isEqualTo(regularFav);
replacedertThat(handler.getMetadata(ActionInputHelper.fromPath("/output/bin/target/bytestring3"))).isNull();
replacedertThat(handler.getMetadata(ActionInputHelper.fromPath("/does/not/exist"))).isNull();
}
19
View Complete Implementation : FilesystemValueCheckerTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
// TODO(bazel-team): Add some tests for FileSystemValueChecker#changedKeys*() methods.
// Presently these appear to be untested.
private ActionExecutionValue actionValue(Action action) {
Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
for (Artifact output : action.getOutputs()) {
try {
Path path = output.getPath();
FileArtifactValue noDigest = ActionMetadataHandler.fileArtifactValueFromArtifact(output, FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)), null);
FileArtifactValue withDigest = FileArtifactValue.createFromInjectedDigest(noDigest, path.getDigest(), !output.isConstantMetadata());
artifactData.put(output, withDigest);
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
return ActionExecutionValue.create(artifactData, ImmutableMap.<Artifact, TreeArtifactValue>of(), /*outputSymlinks=*/
null, /*discoveredModules=*/
null, /*actionDependsOnBuildId=*/
false);
}
19
View Complete Implementation : RemoteActionFileSystem.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private void downloadFileIfRemote(Path path) throws IOException {
FileArtifactValue m = getRemoteInputMetadata(path);
if (m != null) {
try {
inputFetcher.downloadFile(toDelegatePath(path), m);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException(String.format("Received interrupt while fetching file '%s'", path), e);
}
}
}
19
View Complete Implementation : RemoteActionInputFetcher.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
void downloadFile(Path path, FileArtifactValue metadata) throws IOException, InterruptedException {
try {
downloadFileAsync(path, metadata).get();
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException) {
throw (IOException) e.getCause();
}
throw new IOException(e.getCause());
}
}
19
View Complete Implementation : RemoteActionFileSystem.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Override
protected boolean isExecutable(Path path) throws IOException {
FileArtifactValue m = getRemoteInputMetadata(path);
return m != null || super.isExecutable(path);
}
19
View Complete Implementation : ArtifactFunctionTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private void replacedertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value) throws IOException {
replacedertThat(value.getSize()).isEqualTo(file.getSize());
if (digest == null) {
replacedertThat(value.getDigest()).isNull();
replacedertThat(value.getModifiedTime()).isEqualTo(file.getLastModifiedTime());
} else {
replacedertThat(value.getDigest()).isEqualTo(digest);
}
}
18
View Complete Implementation : ActionMetadataHandler.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Override
public FileArtifactValue getMetadata(ActionInput actionInput) throws IOException {
if (!(actionInput instanceof Artifact)) {
PathFragment inputPath = actionInput.getExecPath();
PathFragment filesetKeyPath = inputPath.startsWith(execRoot.asFragment()) ? inputPath.relativeTo(execRoot.asFragment()) : inputPath;
return filesetMapping.get(filesetKeyPath);
}
Artifact artifact = (Artifact) actionInput;
FileArtifactValue value = getInputFileArtifactValue(artifact);
if (value != null) {
return metadataFromValue(value);
}
if (artifact.isSourceArtifact()) {
// A discovered input we didn't have data for.
// TODO(bazel-team): Change this to an replacedertion once Skyframe has native input discovery, so
// all inputs will already have metadata known.
if (!missingArtifactsAllowed) {
throw new IllegalStateException(String.format("null for %s", artifact));
}
return null;
} else if (artifact.isMiddlemanArtifact()) {
// A middleman artifact's data was either already injected from the action cache checker using
// #setDigestForVirtualArtifact, or it has the default middleman value.
value = store.getArtifactData(artifact);
if (value != null) {
return metadataFromValue(value);
}
value = FileArtifactValue.DEFAULT_MIDDLEMAN;
store.putArtifactData(artifact, value);
return metadataFromValue(value);
} else if (artifact.isTreeArtifact()) {
TreeArtifactValue setValue = getTreeArtifactValue((SpecialArtifact) artifact);
if (setValue != null && !setValue.equals(TreeArtifactValue.MISSING_TREE_ARTIFACT)) {
return setValue.getMetadata();
}
// We use FileNotFoundExceptions to determine if an Artifact was or wasn't found.
// Calling code depends on this particular exception.
throw new FileNotFoundException(artifact + " not found");
}
// Fallthrough: the artifact must be a non-tree, non-middleman output artifact.
// Don't store metadata for output artifacts that are not declared outputs of the action.
if (!isKnownOutput(artifact)) {
// Throw in strict mode.
if (!missingArtifactsAllowed) {
throw new IllegalStateException(String.format("null for %s", artifact));
}
return null;
}
// Check for existing metadata. It may have been injected. In either case, this method is called
// from SkyframeActionExecutor to make sure that we have metadata for all action outputs, as the
// results are then stored in Skyframe (and the action cache).
FileArtifactValue fileMetadata = store.getArtifactData(artifact);
if (fileMetadata != null) {
return metadataFromValue(fileMetadata);
}
// No existing metadata; this can happen if the output metadata is not injected after a spawn
// is executed. SkyframeActionExecutor.checkOutputs calls this method for every output file of
// the action, which hits this code path. Another possibility is that an action runs multiple
// spawns, and a subsequent spawn requests the metadata of an output of a previous spawn.
//
// Stat the file. All output artifacts of an action are deleted before execution, so if a file
// exists, it was most likely created by the current action. There is a race condition here if
// an external process creates (or modifies) the file between the deletion and this stat, which
// we cannot solve.
//
// We only cache nonexistence here, not file system errors. It is unlikely that the file will be
// requested from this cache too many times.
fileMetadata = constructFileArtifactValue(artifact, /*statNoFollow=*/
null);
return maybeStoreAdditionalData(artifact, fileMetadata, null);
}
18
View Complete Implementation : ActionMetadataHandler.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Override
public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
Preconditions.checkState(executionMode.get());
Preconditions.checkState(!output.isSymlink());
// replacedumption: any non-Artifact output is 'virtual' and should be ignored here.
if (output instanceof Artifact) {
final Artifact artifact = (Artifact) output;
// We have to add the artifact to injectedFiles before calling constructFileArtifactValue
// to avoid duplicate chmod calls.
store.injectedFiles().add(artifact);
FileArtifactValue fileMetadata;
try {
// This call may do an unnecessary call to Path#getFastDigest to see if the digest is
// readily available. We cannot preplaced the digest in, though, because if it is not available
// from the filesystem, this ArtifactFileMetadata will not compare equal to another one
// created for the
// same file, because the other one will be missing its digest.
fileMetadata = constructFileArtifactValue(artifact, FileStatusWithDigestAdapter.adapt(statNoFollow));
// Ensure the digest supplied matches the actual digest if it exists.
byte[] fileDigest = fileMetadata.getDigest();
if (fileDigest != null && !Arrays.equals(digest, fileDigest)) {
BaseEncoding base16 = BaseEncoding.base16();
String digestString = (digest != null) ? base16.encode(digest) : "null";
String fileDigestString = base16.encode(fileDigest);
throw new IllegalStateException("Expected digest " + digestString + " for artifact " + artifact + ", but got " + fileDigestString + " (" + fileMetadata + ")");
}
} catch (IOException e) {
// Do nothing - we just failed to inject metadata. Real error handling will be done later,
// when somebody will try to access that file.
return;
}
// If needed, insert additional data. Note that this can only be true if the file is empty or
// the filesystem does not support fast digests. Since we usually only inject digests when
// running with a filesystem that supports fast digests, this is fairly unlikely.
try {
maybeStoreAdditionalData(artifact, fileMetadata, digest);
} catch (IOException e) {
throw new IllegalStateException("Filesystem should not have been accessed while injecting data for " + artifact.prettyPrint(), e);
}
}
}
18
View Complete Implementation : AggregatingArtifactValue.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Value for aggregating artifacts, which must be expanded to a set of other artifacts.
*/
clreplaced AggregatingArtifactValue implements SkyValue {
private final FileArtifactValue selfData;
private final ImmutableList<Pair<Artifact, FileArtifactValue>> fileInputs;
private final ImmutableList<Pair<Artifact, TreeArtifactValue>> directoryInputs;
AggregatingArtifactValue(ImmutableList<Pair<Artifact, FileArtifactValue>> fileInputs, ImmutableList<Pair<Artifact, TreeArtifactValue>> directoryInputs, FileArtifactValue selfData) {
this.fileInputs = Preconditions.checkNotNull(fileInputs);
this.directoryInputs = Preconditions.checkNotNull(directoryInputs);
this.selfData = Preconditions.checkNotNull(selfData);
}
/**
* Returns the none tree artifacts that this artifact expands to, together with their data.
*/
Collection<Pair<Artifact, FileArtifactValue>> getFileArtifacts() {
return fileInputs;
}
/**
* Returns the tree artifacts that this artifact expands to, together with the information
* to which artifacts the tree artifacts expand to.
*/
Collection<Pair<Artifact, TreeArtifactValue>> getTreeArtifacts() {
return directoryInputs;
}
/**
* Returns the data of the artifact for this value, as computed by the action cache checker.
*/
FileArtifactValue getSelfData() {
return selfData;
}
// RunfilesArtifactValue not equal to Aggregating.
@SuppressWarnings("EqualsGetClreplaced")
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClreplaced() != o.getClreplaced()) {
return false;
}
AggregatingArtifactValue that = (AggregatingArtifactValue) o;
return selfData.equals(that.selfData) && fileInputs.equals(that.fileInputs) && directoryInputs.equals(that.directoryInputs);
}
@Override
public int hashCode() {
return 31 * 31 * directoryInputs.hashCode() + 31 * fileInputs.hashCode() + selfData.hashCode();
}
}
18
View Complete Implementation : ArtifactFunction.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws ArtifactFunctionException, InterruptedException {
Artifact artifact = (Artifact) skyKey;
if (artifact.isSourceArtifact()) {
try {
return createSourceValue(artifact, env);
} catch (IOException e) {
throw new ArtifactFunctionException(e, Transience.TRANSIENT);
}
}
Artifact.DerivedArtifact derivedArtifact = (DerivedArtifact) artifact;
ArtifactDependencies artifactDependencies = ArtifactDependencies.discoverDependencies(derivedArtifact, env);
if (artifactDependencies == null) {
return null;
}
// If the action is an ActionTemplate, we need to expand the ActionTemplate into concrete
// actions, execute those actions in parallel and then aggregate the action execution results.
if (artifactDependencies.isTemplateActionForTreeArtifact()) {
if (mkdirForTreeArtifacts.get()) {
mkdirForTreeArtifact(artifact, env);
}
return createTreeArtifactValueFromActionKey(artifactDependencies, env);
}
ActionLookupData generatingActionKey = derivedArtifact.getGeneratingActionKey();
ActionExecutionValue actionValue = (ActionExecutionValue) env.getValue(generatingActionKey);
if (actionValue == null) {
return null;
}
if (artifact.isTreeArtifact()) {
// We got a request for the whole tree artifact. We can just return the replacedociated
// TreeArtifactValue.
return Preconditions.checkNotNull(actionValue.getTreeArtifactValue(artifact), artifact);
}
Preconditions.checkState(artifact.isMiddlemanArtifact(), artifact);
Action action = Preconditions.checkNotNull(artifactDependencies.actionLookupValue.getAction(generatingActionKey.getActionIndex()), "Null middleman action? %s", artifactDependencies);
FileArtifactValue individualMetadata = Preconditions.checkNotNull(actionValue.getArtifactValue(artifact), "%s %s", artifact, actionValue);
if (isAggregatingValue(action)) {
return createAggregatingValue(artifact, action, individualMetadata, env);
}
return individualMetadata;
}
18
View Complete Implementation : FilesystemValueChecker.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue, ImmutableSet<PathFragment> knownModifiedOutputFiles, Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles) {
boolean isDirty = false;
for (Map.Entry<Artifact, FileArtifactValue> entry : actionValue.getAllFileValues().entrySet()) {
Artifact file = entry.getKey();
FileArtifactValue lastKnownData = entry.getValue();
if (!file.isMiddlemanArtifact() && shouldCheckFile(knownModifiedOutputFiles, file)) {
try {
FileArtifactValue fileMetadata = ActionMetadataHandler.fileArtifactValueFromArtifact(file, null, tsgm);
FileArtifactValue fileValue = actionValue.getArtifactValue(file);
boolean lastSeenRemotely = (fileValue != null) && fileValue.isRemote();
boolean trustRemoteValue = fileMetadata.getType() == FileStateType.NONEXISTENT && lastSeenRemotely;
if (!trustRemoteValue && fileMetadata.couldBeModifiedSince(lastKnownData)) {
updateIntraBuildModifiedCounter(fileMetadata.getType() != FileStateType.NONEXISTENT ? file.getPath().getLastModifiedTime(Symlinks.FOLLOW) : -1);
modifiedOutputFilesCounter.getAndIncrement();
isDirty = true;
}
} catch (IOException e) {
// This is an unexpected failure getting a digest or symlink target.
modifiedOutputFilesCounter.getAndIncrement();
isDirty = true;
}
}
}
for (Map.Entry<Artifact, TreeArtifactValue> entry : actionValue.getAllTreeArtifactValues().entrySet()) {
Artifact artifact = entry.getKey();
if (shouldCheckTreeArtifact(sortedKnownModifiedOutputFiles.get(), artifact) && treeArtifactIsDirty(artifact, entry.getValue())) {
Path path = artifact.getPath();
// Count the changed directory as one "file".
try {
updateIntraBuildModifiedCounter(path.exists() ? path.getLastModifiedTime() : -1);
} catch (IOException e) {
// Do nothing here.
}
modifiedOutputFilesCounter.getAndIncrement();
isDirty = true;
}
}
return isDirty;
}
18
View Complete Implementation : TreeArtifactValue.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Returns a TreeArtifactValue out of the given Artifact-relative path fragments and their
* corresponding FileArtifactValues.
*/
static TreeArtifactValue create(Map<TreeFileArtifact, FileArtifactValue> childFileValues) {
if (childFileValues.isEmpty()) {
return EMPTY;
}
OrderIndependentHasher hasher = new OrderIndependentHasher();
boolean remote = true;
for (Map.Entry<TreeFileArtifact, FileArtifactValue> e : childFileValues.entrySet()) {
FileArtifactValue value = e.getValue();
// TODO(buchgr): Enforce that all children in a tree artifact are either remote or local
// once b/70354083 is fixed.
remote = remote && value.isRemote();
hasher.addArtifact(e.getKey().getParentRelativePath().getPathString(), value);
}
return new TreeArtifactValue(hasher.finish(), ImmutableSortedMap.copyOf(childFileValues), remote);
}
18
View Complete Implementation : FakeActionInputFileCache.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
public void put(ActionInput artifact, FileArtifactValue metadata) {
inputs.put(artifact, metadata);
}
18
View Complete Implementation : ByteStreamBuildEventArtifactUploaderTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void remoteFileShouldNotBeUploaded_actionFs() throws Exception {
// Test that we don't attempt to upload remotely stored file but convert the remote path
// to a bytestream:// URI.
// arrange
ByteStreamUploader uploader = Mockito.mock(ByteStreamUploader.clreplaced);
RemoteActionInputFetcher actionInputFetcher = Mockito.mock(RemoteActionInputFetcher.clreplaced);
ByteStreamBuildEventArtifactUploader artifactUploader = newArtifactUploader(uploader);
ActionInputMap outputs = new ActionInputMap(2);
Artifact artifact = createRemoteArtifact("file1.txt", "foo", outputs);
RemoteActionFileSystem remoteFs = new RemoteActionFileSystem(fs, execRoot.asFragment(), outputRoot.getRoot().asPath().relativeTo(execRoot).getPathString(), outputs, actionInputFetcher);
Path remotePath = remoteFs.getPath(artifact.getPath().getPathString());
replacedertThat(remotePath.getFileSystem()).isEqualTo(remoteFs);
LocalFile file = new LocalFile(remotePath, LocalFileType.OUTPUT);
// act
PathConverter pathConverter = artifactUploader.upload(ImmutableMap.of(remotePath, file)).get();
FileArtifactValue metadata = outputs.getMetadata(artifact);
Digest digest = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
// replacedert
String conversion = pathConverter.apply(remotePath);
replacedertThat(conversion).isEqualTo("bytestream://localhost/instance/blobs/" + digest.getHash() + "/" + digest.getSizeBytes());
verifyNoMoreInteractions(uploader);
}
18
View Complete Implementation : ByteStreamBuildEventArtifactUploaderTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Returns a remote artifact and puts its metadata into the action input map.
*/
private Artifact createRemoteArtifact(String pathFragment, String contents, ActionInputMap inputs) {
Path p = outputRoot.getRoot().asPath().getRelative(pathFragment);
Artifact a = ActionsTestUtil.createArtifact(outputRoot, p);
byte[] b = contents.getBytes(StandardCharsets.UTF_8);
HashCode h = HashCode.fromString(DIGEST_UTIL.compute(b).getHash());
FileArtifactValue f = new RemoteFileArtifactValue(h.asBytes(), b.length, /* locationIndex= */
1);
inputs.putWithNoDepOwner(a, f);
return a;
}
18
View Complete Implementation : RemoteActionFileSystemTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Returns a remote artifact and puts its metadata into the action input map.
*/
private Artifact createRemoteArtifact(String pathFragment, String contents, ActionInputMap inputs) {
Path p = outputRoot.getRoot().asPath().getRelative(pathFragment);
Artifact a = ActionsTestUtil.createArtifact(outputRoot, p);
byte[] b = contents.getBytes(StandardCharsets.UTF_8);
HashCode h = HASH_FUNCTION.getHashFunction().hashBytes(b);
FileArtifactValue f = new RemoteFileArtifactValue(h.asBytes(), b.length, /* locationIndex= */
1);
inputs.putWithNoDepOwner(a, f);
return a;
}
18
View Complete Implementation : RemoteActionInputFetcherTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private Artifact createRemoteArtifact(String pathFragment, String contents, Map<ActionInput, FileArtifactValue> metadata, Map<Digest, ByteString> cacheEntries) {
Path p = artifactRoot.getRoot().getRelative(pathFragment);
Artifact a = ActionsTestUtil.createArtifact(artifactRoot, p);
byte[] b = contents.getBytes(StandardCharsets.UTF_8);
HashCode h = HASH_FUNCTION.getHashFunction().hashBytes(b);
FileArtifactValue f = new RemoteFileArtifactValue(h.asBytes(), b.length, /* locationIndex= */
1);
metadata.put(a, f);
cacheEntries.put(DigestUtil.buildDigest(h.asBytes(), b.length), ByteString.copyFrom(b));
return a;
}
18
View Complete Implementation : ActionMetadataHandlerTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void injectRemoteArtifactMetadata() throws Exception {
PathFragment path = PathFragment.create("foo/bar");
Artifact artifact = ActionsTestUtil.createArtifactWithRootRelativePath(outputRoot, path);
ActionMetadataHandler handler = new ActionMetadataHandler(/* inputArtifactData= */
new ActionInputMap(0), ImmutableMap.of(), /* missingArtifactsAllowed= */
false, /* outputs= */
ImmutableList.of(artifact), /* tsgm= */
null, ArtifactPathResolver.IDENreplacedY, new OutputStore(), outputRoot.getRoot().asPath());
handler.discardOutputMetadata();
byte[] digest = new byte[] { 1, 2, 3 };
int size = 10;
handler.injectRemoteFile(artifact, digest, size, /* locationIndex= */
1);
FileArtifactValue v = handler.getMetadata(artifact);
replacedertThat(v).isNotNull();
replacedertThat(v.getDigest()).isEqualTo(digest);
replacedertThat(v.getSize()).isEqualTo(size);
}
18
View Complete Implementation : ActionMetadataHandlerTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void withArtifactInput() throws Exception {
PathFragment path = PathFragment.create("src/a");
Artifact artifact = ActionsTestUtil.createArtifactWithRootRelativePath(sourceRoot, path);
FileArtifactValue metadata = FileArtifactValue.createForNormalFile(new byte[] { 1, 2, 3 }, /*proxy=*/
null, 10L, /*isShareable=*/
true);
ActionInputMap map = new ActionInputMap(1);
map.putWithNoDepOwner(artifact, metadata);
ActionMetadataHandler handler = new ActionMetadataHandler(map, ImmutableMap.of(), /* missingArtifactsAllowed= */
false, /* outputs= */
ImmutableList.of(), /* tsgm= */
null, ArtifactPathResolver.IDENreplacedY, new MinimalOutputStore(), outputRoot.getRoot().asPath());
replacedertThat(handler.getMetadata(artifact)).isEqualTo(metadata);
}
18
View Complete Implementation : ArtifactFunctionTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void testUnreadableInputWithFsWithAvailableDigest() throws Throwable {
final byte[] expectedDigest = { 1, 2, 3, 4 };
setupRoot(new CustomInMemoryFs() {
@Override
public byte[] getDigest(Path path) throws IOException {
return path.getBaseName().equals("unreadable") ? expectedDigest : super.getDigest(path);
}
});
Artifact input = createSourceArtifact("unreadable");
Path inputPath = input.getPath();
file(inputPath, "dummynotused");
inputPath.chmod(0);
FileArtifactValue value = (FileArtifactValue) evaluateArtifactValue(input);
FileStatus stat = inputPath.stat();
replacedertThat(value.getSize()).isEqualTo(stat.getSize());
replacedertThat(value.getDigest()).isEqualTo(expectedDigest);
}
18
View Complete Implementation : ArtifactFunctionTest.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Test
public void actionExecutionValueSerialization() throws Exception {
ActionLookupData dummyData = ActionLookupData.create(ALL_OWNER, 0);
Artifact.DerivedArtifact artifact1 = createDerivedArtifact("one");
FileArtifactValue metadata1 = ActionMetadataHandler.fileArtifactValueFromArtifact(artifact1, null, null);
SpecialArtifact treeArtifact = createDerivedTreeArtifactOnly("tree");
treeArtifact.setGeneratingActionKey(dummyData);
TreeFileArtifact treeFileArtifact = ActionInputHelper.treeFileArtifact(treeArtifact, "subpath");
Path path = treeFileArtifact.getPath();
FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
writeFile(path, "contents");
TreeArtifactValue treeArtifactValue = TreeArtifactValue.create(ImmutableMap.of(treeFileArtifact, FileArtifactValue.createForTesting(treeFileArtifact)));
Artifact.DerivedArtifact artifact3 = createDerivedArtifact("three");
FilesetOutputSymlink filesetOutputSymlink = FilesetOutputSymlink.createForTesting(PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT, PathFragment.EMPTY_FRAGMENT);
ActionExecutionValue actionExecutionValue = ActionExecutionValue.create(ImmutableMap.of(artifact1, metadata1, artifact3, FileArtifactValue.DEFAULT_MIDDLEMAN), ImmutableMap.of(treeArtifact, treeArtifactValue), ImmutableList.of(filesetOutputSymlink), null, true);
new SerializationTester(actionExecutionValue).addDependency(FileSystem.clreplaced, root.getFileSystem()).runTests();
}
18
View Complete Implementation : SingleBuildFileCache.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Override
public FileArtifactValue getMetadata(ActionInput input) throws IOException {
try {
return pathToMetadata.get(input.getExecPathString(), () -> {
Path path = ActionInputHelper.toInputPath(input, execRoot);
try {
FileArtifactValue metadata = FileArtifactValue.createFromStat(path, path.stat(Symlinks.FOLLOW), true);
if (metadata.getType().isDirectory()) {
throw new DigestOfDirectoryException("Input is a directory: " + input.getExecPathString());
}
return new ActionInputMetadata(input, metadata);
} catch (IOException e) {
return new ActionInputMetadata(input, e);
}
}).getMetadata();
} catch (ExecutionException e) {
// Should never happen.
throw new IllegalStateException("Unexpected cache loading error", e);
}
}
18
View Complete Implementation : SpawnInputExpander.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private static void failIfDirectory(MetadataProvider actionFileCache, ActionInput input) throws IOException {
FileArtifactValue metadata = actionFileCache.getMetadata(input);
if (metadata != null && !metadata.getType().isFile()) {
throw new IOException("Not a file: " + input.getExecPathString());
}
}
18
View Complete Implementation : RemoteActionFileSystem.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Nullable
private RemoteFileArtifactValue getRemoteInputMetadata(String execPathString) {
FileArtifactValue m = inputArtifactData.getMetadata(execPathString);
if (m != null && m.isRemote()) {
return (RemoteFileArtifactValue) m;
}
return null;
}
18
View Complete Implementation : ActionMetadataHandler.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
private FileArtifactValue maybeStoreAdditionalData(Artifact artifact, FileArtifactValue data, @Nullable byte[] injectedDigest) throws IOException {
if (data.getType() == FileStateType.NONEXISTENT) {
// Nonexistent files should only occur before executing an action.
throw new FileNotFoundException(artifact.prettyPrint() + " does not exist");
}
if (data.getType() == FileStateType.SYMLINK) {
// We never create a FileArtifactValue for an unresolved symlink without a digest (calling
// readlink() is easy, unlike checksumming a potentially huge file)
Preconditions.checkState(data.getDigest() != null);
return data;
}
boolean isFile = data.getType() == FileStateType.REGULAR_FILE;
if (isFile && !artifact.hasParent() && data.getDigest() != null) {
// We do not need to store the FileArtifactValue separately -- the digest is in the file value
// and that is all that is needed for this file's metadata.
return data;
}
final FileArtifactValue value;
if (data.getType() == FileStateType.DIRECTORY) {
// This branch is taken when the output of an action is a directory:
// - A Fileset (in this case, Blaze is correct)
// - A directory someone created in a local action (in this case, changes under the
// directory may not be detected since we use the mtime of the directory for
// up-to-dateness checks)
// - A symlink to a source directory due to Filesets
value = FileArtifactValue.createForDirectoryWithMtime(artifactPathResolver.toPath(artifact).getLastModifiedTime());
} else {
// Unfortunately, the ArtifactFileMetadata does not contain enough information for us to
// calculate the corresponding FileArtifactValue -- either the metadata must use the modified
// time, which we do not expose in the ArtifactFileMetadata, or the ArtifactFileMetadata
// didn't store the digest So we store the metadata separately.
// Use the ArtifactFileMetadata's digest if no digest was injected, or if the file can't be
// digested.
if (injectedDigest == null && isFile) {
injectedDigest = DigestUtils.getDigestOrFail(artifactPathResolver.toPath(artifact), data.getSize());
}
value = FileArtifactValue.createFromInjectedDigest(data, injectedDigest, !artifact.isConstantMetadata());
}
store.putArtifactData(artifact, value);
return metadataFromValue(value);
}
18
View Complete Implementation : ActionMetadataHandler.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Constructs a new {@link FileArtifactValue}, saves it, and checks inconsistent data. This calls
* chmod on the file if we're in executionMode.
*/
private FileArtifactValue constructFileArtifactValue(Artifact artifact, @Nullable FileStatusWithDigest statNoFollow) throws IOException {
// We first chmod the output files before we construct the FileContentsProxy. The proxy may use
// ctime, which is affected by chmod.
if (executionMode.get()) {
Preconditions.checkState(!artifact.isTreeArtifact());
setPathReadOnlyAndExecutable(artifact);
}
FileArtifactValue value = fileArtifactValueFromArtifact(artifact, artifactPathResolver, statNoFollow, getTimestampGranularityMonitor(artifact));
store.putArtifactData(artifact, value);
return value;
}
17
View Complete Implementation : SpawnLogContext.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Computes the digest of the given ActionInput or corresponding path. Will try to access the
* Metadata cache first, if it is available, and fall back to digesting the contents manually.
*/
private Digest computeDigest(@Nullable ActionInput input, @Nullable Path path, MetadataProvider metadataProvider) throws IOException {
Preconditions.checkArgument(input != null || path != null);
DigestHashFunction hashFunction = execRoot.getFileSystem().getDigestFunction();
Digest.Builder digest = Digest.newBuilder().setHashFunctionName(hashFunction.toString());
if (input != null) {
if (input instanceof VirtualActionInput) {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
((VirtualActionInput) input).writeTo(buffer);
byte[] blob = buffer.toByteArray();
return digest.setHash(hashFunction.getHashFunction().hashBytes(blob).toString()).setSizeBytes(blob.length).build();
}
// Try to access the cached metadata, otherwise fall back to local computation.
try {
FileArtifactValue metadata = metadataProvider.getMetadata(input);
if (metadata != null) {
byte[] hash = metadata.getDigest();
if (hash != null) {
return digest.setHash(HashCode.fromBytes(hash).toString()).setSizeBytes(metadata.getSize()).build();
}
}
} catch (IOException | IllegalStateException e) {
// Preplaced through to local computation.
}
}
if (path == null) {
path = execRoot.getRelative(input.getExecPath());
}
// Compute digest manually.
return digest.setHash(HashCode.fromBytes(path.getDigest()).toString()).setSizeBytes(path.getFileSize()).build();
}
17
View Complete Implementation : RemoteActionInputFetcher.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
/**
* Fetches remotely stored action outputs, that are inputs to this spawn, and stores them under
* their path in the output base.
*
* <p>This method blocks until all downloads have finished.
*
* <p>This method is safe to be called concurrently from spawn runners before running any local
* spawn.
*/
@Override
public void prefetchFiles(Iterable<? extends ActionInput> inputs, MetadataProvider metadataProvider) throws IOException, InterruptedException {
try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.REMOTE_DOWNLOAD, "stage remote inputs")) {
Map<Path, ListenableFuture<Void>> downloadsToWaitFor = new HashMap<>();
for (ActionInput input : inputs) {
if (input instanceof VirtualActionInput) {
VirtualActionInput paramFileActionInput = (VirtualActionInput) input;
Path outputPath = execRoot.getRelative(paramFileActionInput.getExecPath());
outputPath.getParentDirectory().createDirectoryAndParents();
try (OutputStream out = outputPath.getOutputStream()) {
paramFileActionInput.writeTo(out);
}
} else {
FileArtifactValue metadata = metadataProvider.getMetadata(input);
if (metadata == null || !metadata.isRemote()) {
continue;
}
Path path = execRoot.getRelative(input.getExecPath());
synchronized (lock) {
if (downloadedPaths.contains(path)) {
continue;
}
ListenableFuture<Void> download = downloadFileAsync(path, metadata);
downloadsToWaitFor.putIfAbsent(path, download);
}
}
}
IOException ioException = null;
InterruptedException interruptedException = null;
for (Map.Entry<Path, ListenableFuture<Void>> entry : downloadsToWaitFor.entrySet()) {
try {
Utils.getFromFuture(entry.getValue());
} catch (IOException e) {
if (e instanceof CacheNotFoundException) {
e = new IOException(String.format("Failed to fetch file with hash '%s' because it does not exist remotely." + " --experimental_remote_outputs=minimal does not work if" + " your remote cache evicts files during builds.", ((CacheNotFoundException) e).getMissingDigest().getHash()));
}
ioException = ioException == null ? e : ioException;
} catch (InterruptedException e) {
interruptedException = interruptedException == null ? e : interruptedException;
}
}
if (interruptedException != null) {
throw interruptedException;
}
if (ioException != null) {
throw ioException;
}
}
}
17
View Complete Implementation : ArtifactFunction.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
@Nullable
private static AggregatingArtifactValue createAggregatingValue(Artifact artifact, ActionreplacedysisMetadata action, FileArtifactValue value, SkyFunction.Environment env) throws InterruptedException {
ImmutableList.Builder<Pair<Artifact, FileArtifactValue>> fileInputsBuilder = ImmutableList.builder();
ImmutableList.Builder<Pair<Artifact, TreeArtifactValue>> directoryInputsBuilder = ImmutableList.builder();
// Avoid iterating over nested set twice.
Iterable<Artifact> inputs = action.getInputs().toList();
Map<SkyKey, SkyValue> values = env.getValues(Artifact.keys(inputs));
if (env.valuesMissing()) {
return null;
}
for (Artifact input : inputs) {
SkyValue inputValue = Preconditions.checkNotNull(values.get(Artifact.key(input)), input);
if (inputValue instanceof FileArtifactValue) {
fileInputsBuilder.add(Pair.of(input, (FileArtifactValue) inputValue));
} else if (inputValue instanceof ActionExecutionValue) {
fileInputsBuilder.add(Pair.of(input, createSimpleFileArtifactValue((DerivedArtifact) input, (ActionExecutionValue) inputValue)));
} else if (inputValue instanceof TreeArtifactValue) {
directoryInputsBuilder.add(Pair.of(input, (TreeArtifactValue) inputValue));
} else {
// We do not recurse in aggregating middleman artifacts.
Preconditions.checkState(!(inputValue instanceof AggregatingArtifactValue), "%s %s %s", artifact, action, inputValue);
}
}
ImmutableList<Pair<Artifact, FileArtifactValue>> fileInputs = ImmutableList.sortedCopyOf(Comparator.comparing(pair -> pair.getFirst().getExecPathString()), fileInputsBuilder.build());
ImmutableList<Pair<Artifact, TreeArtifactValue>> directoryInputs = ImmutableList.sortedCopyOf(Comparator.comparing(pair -> pair.getFirst().getExecPathString()), directoryInputsBuilder.build());
return (action.getActionType() == MiddlemanType.AGGREGATING_MIDDLEMAN) ? new AggregatingArtifactValue(fileInputs, directoryInputs, value) : new RunfilesArtifactValue(fileInputs, directoryInputs, value);
}
17
View Complete Implementation : OutputStore.java
Copyright Apache License 2.0
Author : bazelbuild
Copyright Apache License 2.0
Author : bazelbuild
void putArtifactData(Artifact artifact, FileArtifactValue value) {
artifactData.put(artifact, value);
}