skyfocus: refactor focus command into BlazeModule (SkyfocusModule) an…

…d implement `build --experimental_working_set=<files>`.

The new `build` flags are:

* `--experimental_enable_skyfocus`: the main feature flag.
* `--experimental_working_set`: replacement of `focus --experimental_working_set`
* `--experimental_skyfocus_dump_keys`: replacement of `focus --dump_keys`
* `--experimental_skyfocus_dump_post_gc_stats`: replacement of `focus --dump_used_heap_size_after_gc`

In addition, with the move to the `BlazeModule`, I also replaced the trigger point for Skyfocus with an EventBus `@Subscribe`r. The nice thing about that is we can easily hook Skyfocus into various points of the Blaze lifecycle, such as post-analysis for reducing peak heap in the future, instead of just post-execution for reducing retained heap.

I do acknowledge that EventBus makes debugging and tracing the control flow a bit more difficult, but that's a tradeoff I believe it's worth making for the added lifecycle control.

There is no change to the `focus` algorithm in this CL.

[1] https://github.com/google/guava/wiki/EventBusExplained#avoid-eventbus

PiperOrigin-RevId: 604620930
Change-Id: Ie0713f7ae0dc02babfcfdf7a89df5d85a1c35d89

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -196,6 +196,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/remote",
         "//src/main/java/com/google/devtools/build/lib/runtime/mobileinstall",
         "//src/main/java/com/google/devtools/build/lib/sandbox:sandbox_module",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:skyfocus_module",
         "//src/main/java/com/google/devtools/build/lib/skyframe:skymeld_module",
         "//src/main/java/com/google/devtools/build/lib/standalone",
         "//src/main/java/com/google/devtools/build/lib/starlarkdebug/module",

src/main/java/com/google/devtools/build/lib/bazel/Bazel.java

@@ -86,6 +86,7 @@ public final class Bazel {
           BazelBuiltinCommandModule.class,
           com.google.devtools.build.lib.includescanning.IncludeScanningModule.class,
           com.google.devtools.build.lib.skyframe.SkymeldModule.class,
+          com.google.devtools.build.lib.skyframe.SkyfocusModule.class,
           // This module needs to be registered after any module submitting tasks with its {@code
           // submit} method.
           com.google.devtools.build.lib.runtime.BlockWaitingModule.class);

src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java

@@ -655,7 +655,6 @@ public void stopRequest(
     InterruptedException ie = null;
     try {
       env.getSkyframeExecutor().notifyCommandComplete(env.getReporter());
-      env.getSkyframeExecutor().getEvaluator().updateTopLevelEvaluations();
     } catch (InterruptedException e) {
       env.getReporter().handle(Event.error("Build interrupted during command completion"));
       ie = e;

src/main/java/com/google/devtools/build/lib/runtime/SkyfocusOptions.java

@@ -0,0 +1,67 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionsBase;
+import java.util.List;
+
+/** The set of options for the Skyfocus feature. */
+public final class SkyfocusOptions extends OptionsBase {
+
+  @Option(
+      name = "experimental_enable_skyfocus",
+      defaultValue = "false",
+      effectTags = OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS,
+      documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
+      help =
+          "If true, enable the use of --experimental_working_set to reduce Bazel's memory footprint"
+              + " for incremental builds. This feature is known as Skyfocus.")
+  public boolean skyfocusEnabled;
+
+  @Option(
+      name = "experimental_working_set",
+      defaultValue = "",
+      effectTags = OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS,
+      documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
+      converter = Converters.CommaSeparatedOptionListConverter.class,
+      help =
+          "The working set for Skyfocus. Specify as comma-separated workspace root-relative paths."
+              + " This is a stateful flag. Defining a working set persists it for subsequent"
+              + " invocations, until it is redefined with a new set.")
+  public List<String> workingSet;
+
+  @Option(
+      name = "experimental_skyfocus_dump_keys",
+      defaultValue = "false",
+      effectTags = OptionEffectTag.TERMINAL_OUTPUT,
+      documentationCategory = OptionDocumentationCategory.LOGGING,
+      help =
+          "For debugging Skyfocus. Dump the focused SkyKeys (roots, leafs, focused deps, focused"
+              + " rdeps).")
+  public boolean dumpKeys;
+
+  @Option(
+      name = "experimental_skyfocus_dump_post_gc_stats",
+      defaultValue = "false",
+      effectTags = OptionEffectTag.TERMINAL_OUTPUT,
+      documentationCategory = OptionDocumentationCategory.LOGGING,
+      help =
+          "For debugging Skyfocus. If enabled, trigger manual GC before/after focusing to report"
+              + " heap sizes reductions. This will increase the Skyfocus latency.")
+  public boolean dumpPostGcStats;
+}

src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java

@@ -32,6 +32,7 @@
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
 import com.google.devtools.build.lib.runtime.KeepGoingOption;
 import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption;
+import com.google.devtools.build.lib.runtime.SkyfocusOptions;
 import com.google.devtools.build.lib.util.DetailedExitCode;
 import com.google.devtools.common.options.OptionsParsingResult;
 import java.util.List;
@@ -52,7 +53,8 @@
       LoadingOptions.class,
       KeepGoingOption.class,
       LoadingPhaseThreadsOption.class,
-      BuildEventProtocolOptions.class
+      BuildEventProtocolOptions.class,
+      SkyfocusOptions.class,
     },
     usesConfigurationOptions = true,
     shortDescription = "Builds the specified targets.",

src/main/java/com/google/devtools/build/lib/runtime/commands/BuiltinCommandModule.java

@@ -34,7 +34,6 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde
         new CleanCommand(),
         new CoverageCommand(),
         new DumpCommand(),
-        new FocusCommand(),
         new HelpCommand(),
         new InfoCommand(),
         new PrintActionCommand(),