Cp18269

.bazelrc

@@ -25,7 +25,7 @@ build:ubuntu1804_java11 --config=remote_shared
 # Alias
 build:remote --config=ubuntu1804_java11
 
-build:macos --macos_minimum_os=10.10
+build:macos --macos_minimum_os=10.11
 
 # Enable Bzlmod
 build:bzlmod --experimental_enable_bzlmod

scripts/release/relnotes.py

@@ -14,7 +14,6 @@
 
 """Script to generate release notes."""
 
-import os
 import re
 import subprocess
 import sys
@@ -87,13 +86,13 @@ def get_relnotes_between(base, head, is_major_release):
 
 def get_label(issue_id):
   """Get team-X label added to issue."""
-  auth = os.system(
+  auth = subprocess.check_output(
       "gsutil cat"
       " gs://bazel-trusted-encrypted-secrets/github-trusted-token.enc |"
       " gcloud kms decrypt --project bazel-public --location global"
       " --keyring buildkite --key github-trusted-token --ciphertext-file"
-      " - --plaintext-file -"
-  )
+      " - --plaintext-file -", shell=True
+  ).decode("utf-8").strip().split("\n")[0]
   headers = {
       "Authorization": "Bearer " + auth,
       "Accept": "application/vnd.github+json",
@@ -159,11 +158,16 @@ def get_external_authors_between(base, head):
   # e.g. if current_release is 5.3.3, last_release should be 5.3.2 even if
   # latest release is 6.1.1
   current_release = git("rev-parse", "--abbrev-ref", "HEAD")[0]
-  if not current_release.startswith("release-"):
-    print("Error: Not a release- branch")
-    sys.exit(1)
 
-  current_release = re.sub(r"rc\d", "", current_release[len("release-"):])
+  if current_release.startswith("release-"):
+    current_release = re.sub(r"rc\d", "", current_release[len("release-"):])
+  else:
+    try:
+      current_release = git("describe", "--tags")[0]
+    except Exception:  # pylint: disable=broad-exception-caught
+      print("Error: Not a release branch.")
+      sys.exit(1)
+
   is_major = bool(re.fullmatch(r"\d+.0.0", current_release))
 
   tags = [tag for tag in git("tag", "--sort=refname") if "pre" not in tag]
@@ -205,11 +209,13 @@ def get_external_authors_between(base, head):
         print("+", note)
       print()
   else:
+    print()
     for note in filtered_relnotes:
       print("+", note)
 
   print()
   print("Acknowledgements:")
   external_authors = get_external_authors_between(merge_base, "HEAD")
+  print()
   print("This release contains contributions from many people at Google" +
         ("." if not external_authors else f", as well as {external_authors}."))

site/en/reference/test-encyclopedia.md

@@ -223,8 +223,9 @@ index, beginning at 0. Runners use this information to select which tests to
 run - for example, using a round-robin strategy. Not all test runners support
 sharding. If a runner supports sharding, it must create or update the last
 modified date of the file specified by
-[`TEST_SHARD_STATUS_FILE`](#initial-conditions). Otherwise, Bazel assumes it
-does not support sharding and will not launch additional runners.
+[`TEST_SHARD_STATUS_FILE`](#initial-conditions). Otherwise, if
+[`--incompatible_check_sharding_support`](/reference/command-line-reference#flag--incompatible_check_sharding_support)
+is enabled, Bazel will fail the test if it is sharded.
 
 ## Initial conditions {:#initial-conditions}
 

src/main/java/com/google/devtools/build/lib/analysis/test/TestConfiguration.java

@@ -280,6 +280,19 @@ public static class TestOptions extends FragmentOptions {
         help = "If true, undeclared test outputs will be archived in a zip file.")
     public boolean zipUndeclaredTestOutputs;
 
+    @Option(
+        name = "incompatible_check_sharding_support",
+        defaultValue = "false",
+        documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+        metadataTags = {OptionMetadataTag.INCOMPATIBLE_CHANGE},
+        effectTags = {OptionEffectTag.UNKNOWN},
+        help =
+            "If true, Bazel will fail a sharded test if the test runner does not indicate that it "
+                + "supports sharding by touching the file at the path in TEST_SHARD_STATUS_FILE. "
+                + "If false, a test runner that does not support sharding will lead to all tests "
+                + "running in each shard.")
+    public boolean checkShardingSupport;
+
     @Override
     public FragmentOptions getHost() {
       // Options here are either:
@@ -380,6 +393,10 @@ public boolean getZipUndeclaredTestOutputs() {
     return options.zipUndeclaredTestOutputs;
   }
 
+  public boolean checkShardingSupport() {
+    return options.checkShardingSupport;
+  }
+
   /**
    * Option converter that han handle two styles of value for "--runs_per_test":
    *

src/main/java/com/google/devtools/build/lib/analysis/test/TestRunnerAction.java

@@ -336,6 +336,10 @@ public boolean showsOutputUnconditionally() {
     return true;
   }
 
+  public boolean checkShardingSupport() {
+    return testConfiguration.checkShardingSupport();
+  }
+
   public List<ActionInput> getSpawnOutputs() {
     final List<ActionInput> outputs = new ArrayList<>();
     outputs.add(ActionInputHelper.fromPath(getXmlOutputPath()));

src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java

@@ -161,7 +161,7 @@ public class AuthAndTLSOptions extends OptionsBase {
 
   @Option(
       name = "experimental_credential_helper_timeout",
-      defaultValue = "5s",
+      defaultValue = "10s",
       converter = DurationConverter.class,
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},

src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java

@@ -82,12 +82,8 @@ public GetCredentialsResponse getCredentials(CredentialHelperEnvironment environ
           GSON.toJson(GetCredentialsRequest.newBuilder().setUri(uri).build(), stdin);
         } catch (IOException e) {
           // This can happen if the helper prints a static set of credentials without reading from
-          // stdin (e.g., with a simple shell script running `echo "{...}"`). If the process is
-          // already finished even though we failed to write to its stdin, ignore the error and
-          // assume the process did not need the request payload.
-          if (!process.finished()) {
-            throw e;
-          }
+          // stdin (e.g., with a simple shell script running `echo "{...}"`). This is fine to
+          // ignore.
         }
 
         try {

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java

@@ -188,7 +188,7 @@ private static Optional<Event> generateFixupMessage(
             .collect(toImmutableSet());
     var actualImports =
         rootUsages.stream()
-            .flatMap(usage -> usage.getImports().keySet().stream())
+            .flatMap(usage -> usage.getImports().values().stream())
             .filter(repo -> !actualDevImports.contains(repo))
             .collect(toImmutableSet());
 

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java

@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.bazel.repository.downloader;
 
+import com.google.common.base.Ascii;
 import com.google.common.hash.HashCode;
 import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType;
 import java.util.Base64;
@@ -44,7 +45,7 @@ public static Checksum fromString(KeyType keyType, String hash) throws InvalidCh
     if (!keyType.isValid(hash)) {
       throw new InvalidChecksumException(keyType, hash);
     }
-    return new Checksum(keyType, HashCode.fromString(hash));
+    return new Checksum(keyType, HashCode.fromString(Ascii.toLowerCase(hash)));
   }
 
   /** Constructs a new Checksum from a hash in Subresource Integrity format. */

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadProgressEvent.java

@@ -17,6 +17,10 @@
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
 import com.google.devtools.build.lib.remote.util.Utils;
 import java.net.URL;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.Locale;
+import java.util.OptionalLong;
 
 /**
  * Postable event reporting on progress made downloading an URL. It can be used to report the URL
@@ -26,17 +30,20 @@ public class DownloadProgressEvent implements ExtendedEventHandler.FetchProgress
   private final URL originalUrl;
   private final URL actualUrl;
   private final long bytesRead;
+  private final OptionalLong totalBytes;
   private final boolean downloadFinished;
 
-  public DownloadProgressEvent(URL originalUrl, URL actualUrl, long bytesRead, boolean finished) {
+  public DownloadProgressEvent(
+      URL originalUrl, URL actualUrl, long bytesRead, OptionalLong totalBytes, boolean finished) {
     this.originalUrl = originalUrl;
     this.actualUrl = actualUrl;
     this.bytesRead = bytesRead;
+    this.totalBytes = totalBytes;
     this.downloadFinished = finished;
   }
 
   public DownloadProgressEvent(URL originalUrl, long bytesRead, boolean finished) {
-    this(originalUrl, null, bytesRead, finished);
+    this(originalUrl, null, bytesRead, OptionalLong.empty(), finished);
   }
 
   public DownloadProgressEvent(URL url, long bytesRead) {
@@ -69,10 +76,22 @@ public long getBytesRead() {
     return bytesRead;
   }
 
+  private static final DecimalFormat PERCENTAGE_FORMAT =
+      new DecimalFormat("0.0%", new DecimalFormatSymbols(Locale.US));
+
   @Override
   public String getProgress() {
     if (bytesRead > 0) {
-      return String.format("%s (%,dB)", Utils.bytesCountToDisplayString(bytesRead), bytesRead);
+      if (totalBytes.isPresent()) {
+        double totalBytesDouble = this.totalBytes.getAsLong();
+        double ratio = totalBytesDouble != 0 ? bytesRead / totalBytesDouble : 1;
+        // 10.1 MiB (20.2%)
+        return String.format(
+            "%s (%s)", Utils.bytesCountToDisplayString(bytesRead), PERCENTAGE_FORMAT.format(ratio));
+      } else {
+        // 10.1 MiB (10,590,000B)
+        return String.format("%s (%,dB)", Utils.bytesCountToDisplayString(bytesRead), bytesRead);
+      }
     } else {
       return "";
     }

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java

@@ -30,6 +30,7 @@
 import java.io.SequenceInputStream;
 import java.net.URL;
 import java.net.URLConnection;
+import java.util.OptionalLong;
 import java.util.zip.GZIPInputStream;
 import javax.annotation.WillCloseWhenClosed;
 
@@ -87,17 +88,19 @@ HttpStream create(
           stream = retrier;
         }
 
+        OptionalLong totalBytes = OptionalLong.empty();
         try {
           String contentLength = connection.getHeaderField("Content-Length");
           if (contentLength != null) {
-            long expectedSize = Long.parseLong(contentLength);
-            stream = new CheckContentLengthInputStream(stream, expectedSize);
+            totalBytes = OptionalLong.of(Long.parseUnsignedLong(contentLength));
+            stream = new CheckContentLengthInputStream(stream, totalBytes.getAsLong());
           }
         } catch (NumberFormatException ignored) {
           // ignored
         }
 
-        stream = progressInputStreamFactory.create(stream, connection.getURL(), originalUrl);
+        stream =
+            progressInputStreamFactory.create(stream, connection.getURL(), originalUrl, totalBytes);
 
         // Determine if we need to transparently gunzip. See RFC2616 § 3.5 and § 14.11. Please note
         // that some web servers will send Content-Encoding: gzip even when we didn't request it if

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java

@@ -24,6 +24,7 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.util.Locale;
+import java.util.OptionalLong;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.WillCloseWhenClosed;
 
@@ -50,9 +51,20 @@ static class Factory {
       this.eventHandler = eventHandler;
     }
 
-    InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL originalUrl) {
+    InputStream create(
+        @WillCloseWhenClosed InputStream delegate,
+        URL url,
+        URL originalUrl,
+        OptionalLong totalBytes) {
       return new ProgressInputStream(
-          locale, clock, eventHandler, PROGRESS_INTERVAL_MS, delegate, url, originalUrl);
+          locale,
+          clock,
+          eventHandler,
+          PROGRESS_INTERVAL_MS,
+          delegate,
+          url,
+          originalUrl,
+          totalBytes);
     }
   }
 
@@ -63,6 +75,7 @@ InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL origi
   private final long intervalMs;
   private final URL url;
   private final URL originalUrl;
+  private final OptionalLong totalBytes;
   private final AtomicLong toto = new AtomicLong();
   private final AtomicLong nextEvent;
 
@@ -73,7 +86,8 @@ InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL origi
       long intervalMs,
       InputStream delegate,
       URL url,
-      URL originalUrl) {
+      URL originalUrl,
+      OptionalLong totalBytes) {
     Preconditions.checkArgument(intervalMs >= 0);
     this.locale = locale;
     this.clock = clock;
@@ -82,8 +96,9 @@ InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL origi
     this.delegate = delegate;
     this.url = url;
     this.originalUrl = originalUrl;
+    this.totalBytes = totalBytes;
     this.nextEvent = new AtomicLong(clock.currentTimeMillis() + intervalMs);
-    eventHandler.post(new DownloadProgressEvent(originalUrl, url, 0, false));
+    eventHandler.post(new DownloadProgressEvent(originalUrl, url, 0, totalBytes, false));
   }
 
   @Override
@@ -112,7 +127,7 @@ public int available() throws IOException {
   @Override
   public void close() throws IOException {
     delegate.close();
-    eventHandler.post(new DownloadProgressEvent(originalUrl, url, toto.get(), true));
+    eventHandler.post(new DownloadProgressEvent(originalUrl, url, toto.get(), totalBytes, true));
   }
 
   private void reportProgress(long bytesRead) {
@@ -124,7 +139,7 @@ private void reportProgress(long bytesRead) {
     if (!url.getHost().equals(originalUrl.getHost())) {
       via = " via " + url.getHost();
     }
-    eventHandler.post(new DownloadProgressEvent(originalUrl, url, bytesRead, false));
+    eventHandler.post(new DownloadProgressEvent(originalUrl, url, bytesRead, totalBytes, false));
     eventHandler.handle(
         Event.progress(
             String.format(locale, "Downloading %s%s: %,d bytes", originalUrl, via, bytesRead)));

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java

@@ -31,6 +31,7 @@
 import com.google.devtools.build.lib.packages.StructProvider;
 import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
 import com.google.devtools.build.lib.repository.RepositoryFetchProgress;
+import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
 import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper;
 import com.google.devtools.build.lib.runtime.ProcessWrapper;
@@ -516,26 +517,42 @@ public String toString() {
    */
   // TODO(wyv): somehow migrate this to the base context too.
   public void enforceLabelAttributes() throws EvalException, InterruptedException {
+    // TODO: If a labels fails to resolve to an existing regular file, we do not add a dependency on
+    //  that fact - if the file is created later or changes its type, it will not trigger a rerun of
+    //  the repository function.
     StructImpl attr = getAttr();
     for (String name : attr.getFieldNames()) {
       Object value = attr.getValue(name);
       if (value instanceof Label) {
-        getPathFromLabel((Label) value);
+        dependOnLabelIgnoringErrors((Label) value);
       }
       if (value instanceof Sequence) {
         for (Object entry : (Sequence) value) {
           if (entry instanceof Label) {
-            getPathFromLabel((Label) entry);
+            dependOnLabelIgnoringErrors((Label) entry);
           }
         }
       }
       if (value instanceof Dict) {
         for (Object entry : ((Dict) value).keySet()) {
           if (entry instanceof Label) {
-            getPathFromLabel((Label) entry);
+            dependOnLabelIgnoringErrors((Label) entry);
           }
         }
       }
     }
   }
+
+  private void dependOnLabelIgnoringErrors(Label label)
+      throws InterruptedException, NeedsSkyframeRestartException {
+    try {
+      getPathFromLabel(label);
+    } catch (NeedsSkyframeRestartException e) {
+      throw e;
+    } catch (EvalException e) {
+      // EvalExceptions indicate labels not referring to existing files. This is fine,
+      // as long as they are never resolved to files in the execution of the rule; we allow
+      // non-strict rules.
+    }
+  }
 }