diff --git a/src/main/java/com/google/devtools/build/lib/exec/BUILD b/src/main/java/com/google/devtools/build/lib/exec/BUILD index 9d31c1d26e8577..2644880f1569d1 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/BUILD +++ b/src/main/java/com/google/devtools/build/lib/exec/BUILD @@ -275,6 +275,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", + "//src/main/java/com/google/devtools/build/lib/actions:runfiles_metadata", "//src/main/java/com/google/devtools/build/lib/actions:runfiles_tree", "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry", "//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils", diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java index 0648e21857a2a1..6babbab6721823 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java +++ b/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java @@ -13,14 +13,20 @@ // limitations under the License. package com.google.devtools.build.lib.exec; +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.flogger.GoogleLogger; import com.google.devtools.build.lib.actions.AbstractAction; import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.FileArtifactValue.UnresolvedSymlinkArtifactValue; import com.google.devtools.build.lib.actions.InputMetadataProvider; +import com.google.devtools.build.lib.actions.RunfilesArtifactValue; +import com.google.devtools.build.lib.actions.RunfilesTree; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.actions.Spawns; @@ -152,6 +158,15 @@ public void logSpawn( builder.addAllEnvironmentVariables(getEnvironmentVariables(spawn)); ImmutableSet toolFiles = spawn.getToolFiles().toSet(); + ImmutableList toolRunfilesDirectories = + toolFiles.stream() + .filter( + actionInput -> + actionInput instanceof Artifact artifact && artifact.isMiddlemanArtifact()) + .map(inputMetadataProvider::getRunfilesMetadata) + .map(RunfilesArtifactValue::getRunfilesTree) + .map(RunfilesTree::getExecPath) + .collect(toImmutableList()); try (SilentCloseable c1 = Profiler.instance().profile("logSpawn/inputs")) { for (Map.Entry e : inputMap.entrySet()) { @@ -160,14 +175,18 @@ public void logSpawn( if (input instanceof VirtualActionInput.EmptyActionInput) { // Do not include a digest, as it's a waste of space. - builder.addInputsBuilder().setPath(displayPath.getPathString()); + builder + .addInputsBuilder() + .setPath(displayPath.getPathString()) + .setIsTool(toolRunfilesDirectories.stream().anyMatch(displayPath::startsWith)); continue; } boolean isTool = toolFiles.contains(input) || input instanceof TreeFileArtifact treeFileArtifact - && toolFiles.contains(treeFileArtifact.getParent()); + && toolFiles.contains(treeFileArtifact.getParent()) + || toolRunfilesDirectories.stream().anyMatch(displayPath::startsWith); Path contentPath = fileSystem.getPath(execRoot.getRelative(input.getExecPathString())); diff --git a/src/test/java/com/google/devtools/build/lib/exec/SpawnLogContextTestBase.java b/src/test/java/com/google/devtools/build/lib/exec/SpawnLogContextTestBase.java index 8e2fa663407d94..0cc53f1d432b2d 100644 --- a/src/test/java/com/google/devtools/build/lib/exec/SpawnLogContextTestBase.java +++ b/src/test/java/com/google/devtools/build/lib/exec/SpawnLogContextTestBase.java @@ -411,7 +411,7 @@ public void testUnresolvedSymlinkInput(@TestParameter InputsMode inputsMode) thr } @Test - public void testRunfilesFileInput() throws Exception { + public void testRunfilesFileInput(@TestParameter InputsMode inputsMode) throws Exception { Artifact runfilesInput = ActionsTestUtil.createArtifact(rootDir, "data.txt"); Artifact runfilesMiddleman = ActionsTestUtil.createArtifact(middlemanDir, "runfiles"); @@ -420,12 +420,15 @@ public void testRunfilesFileInput() throws Exception { PathFragment runfilesRoot = outputDir.getExecPath().getRelative("foo.runfiles"); RunfilesTree runfilesTree = createRunfilesTree(runfilesRoot, runfilesInput); - Spawn spawn = defaultSpawnBuilder().withInput(runfilesMiddleman).build(); + SpawnBuilder spawnBuilder = defaultSpawnBuilder().withInput(runfilesMiddleman); + if (inputsMode.isTool()) { + spawnBuilder.withTool(runfilesMiddleman); + } SpawnLogContext context = createSpawnLogContext(); context.logSpawn( - spawn, + spawnBuilder.build(), createInputMetadataProvider(runfilesMiddleman, runfilesTree, runfilesInput), createInputMap(runfilesTree), fs, @@ -442,12 +445,15 @@ public void testRunfilesFileInput() throws Exception { + "-out/k8-fastbuild/bin/foo.runfiles/" + WORKSPACE_NAME + "/data.txt") - .setDigest(getDigest("abc"))) + .setDigest(getDigest("abc")) + .setIsTool(inputsMode.isTool())) .build()); } @Test - public void testRunfilesDirectoryInput(@TestParameter DirContents dirContents) throws Exception { + public void testRunfilesDirectoryInput( + @TestParameter DirContents dirContents, @TestParameter InputsMode inputsMode) + throws Exception { Artifact runfilesMiddleman = ActionsTestUtil.createArtifact(middlemanDir, "runfiles"); Artifact runfilesInput = ActionsTestUtil.createArtifact(rootDir, "dir"); @@ -459,12 +465,15 @@ public void testRunfilesDirectoryInput(@TestParameter DirContents dirContents) t PathFragment runfilesRoot = outputDir.getExecPath().getRelative("foo.runfiles"); RunfilesTree runfilesTree = createRunfilesTree(runfilesRoot, runfilesInput); - Spawn spawn = defaultSpawnBuilder().withInput(runfilesMiddleman).build(); + SpawnBuilder spawnBuilder = defaultSpawnBuilder().withInput(runfilesMiddleman); + if (inputsMode.isTool()) { + spawnBuilder.withTool(runfilesMiddleman); + } SpawnLogContext context = createSpawnLogContext(); context.logSpawn( - spawn, + spawnBuilder.build(), createInputMetadataProvider(runfilesMiddleman, runfilesTree, runfilesInput), createInputMap(runfilesTree), fs, @@ -485,12 +494,13 @@ public void testRunfilesDirectoryInput(@TestParameter DirContents dirContents) t + WORKSPACE_NAME + "/dir/data.txt") .setDigest(getDigest("abc")) + .setIsTool(inputsMode.isTool()) .build())) .build()); } @Test - public void testRunfilesEmptyInput() throws Exception { + public void testRunfilesEmptyInput(@TestParameter InputsMode inputsMode) throws Exception { Artifact runfilesMiddleman = ActionsTestUtil.createArtifact(middlemanDir, "runfiles"); Artifact runfilesInput = ActionsTestUtil.createArtifact(rootDir, "sub/dir/script.py"); @@ -518,12 +528,15 @@ public void testRunfilesEmptyInput() throws Exception { createRunfilesTree( runfilesRoot, runfilesInput, externalGenArtifact, externalSourceArtifact); - Spawn spawn = defaultSpawnBuilder().withInput(runfilesMiddleman).build(); + SpawnBuilder spawnBuilder = defaultSpawnBuilder().withInput(runfilesMiddleman); + if (inputsMode.isTool()) { + spawnBuilder.withTool(runfilesMiddleman); + } SpawnLogContext context = createSpawnLogContext(); context.logSpawn( - spawn, + spawnBuilder.build(), createInputMetadataProvider( runfilesMiddleman, runfilesTree, @@ -540,21 +553,24 @@ public void testRunfilesEmptyInput() throws Exception { defaultSpawnExecBuilder() .addInputs( File.newBuilder() - .setPath(PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/__init__.py")) + .setPath(PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/__init__.py") + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/" + WORKSPACE_NAME - + "/sub/__init__.py")) + + "/sub/__init__.py") + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/" + WORKSPACE_NAME - + "/sub/dir/__init__.py")) + + "/sub/dir/__init__.py") + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( @@ -562,37 +578,44 @@ public void testRunfilesEmptyInput() throws Exception { + "-out/k8-fastbuild/bin/foo.runfiles/" + WORKSPACE_NAME + "/sub/dir/script.py") - .setDigest(getDigest("abc"))) + .setDigest(getDigest("abc")) + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( - PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/__init__.py")) + PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/__init__.py") + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( PRODUCT_NAME - + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/other/__init__.py")) + + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/other/__init__.py") + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( PRODUCT_NAME - + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/other/pkg/__init__.py")) + + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/other/pkg/__init__.py") + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/other/pkg/gen.py") - .setDigest(getDigest("external_gen"))) + .setDigest(getDigest("external_gen")) + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( PRODUCT_NAME - + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/pkg/__init__.py")) + + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/pkg/__init__.py") + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( PRODUCT_NAME + "-out/k8-fastbuild/bin/foo.runfiles/some_repo/pkg/lib.py") - .setDigest(getDigest("external_source"))) + .setDigest(getDigest("external_source")) + .setIsTool(inputsMode.isTool())) .build()); } @@ -1291,7 +1314,8 @@ public void testRunfilesPostOrderCollision(@TestParameter boolean nestBoth) thro } @Test - public void testRunfilesSymlinkTargets(@TestParameter boolean rootSymlinks) throws Exception { + public void testRunfilesSymlinkTargets( + @TestParameter boolean rootSymlinks, @TestParameter InputsMode inputsMode) throws Exception { Artifact sourceFile = ActionsTestUtil.createArtifact(rootDir, "pkg/file.txt"); writeFile(sourceFile, "source"); Artifact sourceDir = ActionsTestUtil.createArtifact(rootDir, "pkg/source_dir"); @@ -1329,12 +1353,15 @@ public void testRunfilesSymlinkTargets(@TestParameter boolean rootSymlinks) thro : ImmutableMap.of(), /* legacyExternalRunfiles= */ false); - Spawn spawn = defaultSpawnBuilder().withInput(runfilesMiddleman).build(); + var spawnBuilder = defaultSpawnBuilder().withInput(runfilesMiddleman); + if (inputsMode.isTool()) { + spawnBuilder.withTool(runfilesMiddleman); + } SpawnLogContext context = createSpawnLogContext(); context.logSpawn( - spawn, + spawnBuilder.build(), createInputMetadataProvider( runfilesMiddleman, runfilesTree, sourceFile, sourceDir, genDir, symlink), createInputMap(runfilesTree), @@ -1352,7 +1379,8 @@ public void testRunfilesSymlinkTargets(@TestParameter boolean rootSymlinks) thro + "-out/k8-fastbuild/bin/tools/foo.runfiles/" + WORKSPACE_NAME + "/file") - .setDigest(getDigest("source"))) + .setDigest(getDigest("source")) + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( @@ -1360,7 +1388,8 @@ public void testRunfilesSymlinkTargets(@TestParameter boolean rootSymlinks) thro + "-out/k8-fastbuild/bin/tools/foo.runfiles/" + WORKSPACE_NAME + "/gen_dir/other_file") - .setDigest(getDigest("gen_dir_file"))) + .setDigest(getDigest("gen_dir_file")) + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( @@ -1368,7 +1397,8 @@ public void testRunfilesSymlinkTargets(@TestParameter boolean rootSymlinks) thro + "-out/k8-fastbuild/bin/tools/foo.runfiles/" + WORKSPACE_NAME + "/source_dir/some_file") - .setDigest(getDigest("source_dir_file"))) + .setDigest(getDigest("source_dir_file")) + .setIsTool(inputsMode.isTool())) .addInputs( File.newBuilder() .setPath( @@ -1376,7 +1406,8 @@ public void testRunfilesSymlinkTargets(@TestParameter boolean rootSymlinks) thro + "-out/k8-fastbuild/bin/tools/foo.runfiles/" + WORKSPACE_NAME + "/symlink") - .setSymlinkTargetPath("/some/path")) + .setSymlinkTargetPath("/some/path") + .setIsTool(inputsMode.isTool())) .build()); } @@ -1407,7 +1438,7 @@ public void testRunfileSymlinkFileWithDirectoryContents( spawn, createInputMetadataProvider(runfilesMiddleman, runfilesTree, genFile), createInputMap(runfilesTree), - fs, + outputsMode.getActionFileSystem(fs), defaultTimeout(), defaultSpawnResult());