Skip to content

Commit

Permalink
Make it possible to avoid an extra stat() when obtaining a digest fro…
Browse files Browse the repository at this point in the history
…m the cache.

For now, it's only used to optimize out a stat() required by the execution log, which adds up for very large tree artifacts. However, there are other callsites that could be refactored to benefit from it.

PiperOrigin-RevId: 606891771
Change-Id: Ib2f6428009b86e609f4e46c4ac0ac4b2714414ff
  • Loading branch information
tjgq authored and copybara-github committed Feb 14, 2024
1 parent d51f948 commit bb80319
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.DigestUtils;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
Expand Down Expand Up @@ -172,14 +173,13 @@ protected Digest computeDigest(
}
}

long fileSize = path.getFileSize();

// Try to obtain a digest from the filesystem.
// Obtain a digest from the filesystem.
FileStatus status = path.stat();
return builder
.setHash(
HashCode.fromBytes(DigestUtils.getDigestWithManualFallback(path, xattrProvider))
HashCode.fromBytes(DigestUtils.getDigestWithManualFallback(path, xattrProvider, status))
.toString())
.setSizeBytes(fileSize)
.setSizeBytes(status.getSize())
.build();
}

Expand Down
41 changes: 36 additions & 5 deletions src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
import java.io.IOException;
import javax.annotation.Nullable;

/**
* Utility class for getting digests of files.
Expand Down Expand Up @@ -159,27 +160,57 @@ public static CacheStats getCacheStats() {
* <p>If {@link Path#getFastDigest} has already been attempted and was not available, call {@link
* #manuallyComputeDigest} to skip an additional attempt to obtain the fast digest.
*
* @param path Path of the file.
* <p>Prefer calling {@link #manuallyComputeDigest(Path, FileStatus)} when a recently obtained
* {@link FileStatus} is available.
*
* @param path the file path
*/
public static byte[] getDigestWithManualFallback(Path path, XattrProvider xattrProvider)
throws IOException {
return getDigestWithManualFallback(path, xattrProvider, null);
}

/**
* Same as {@link #getDigestWithManualFallback(Path, XattrProvider)}, but providing the ability to
* reuse a recently obtained {@link FileStatus}.
*
* @param path the file path
* @param status a recently obtained file status, if available
*/
public static byte[] getDigestWithManualFallback(
Path path, XattrProvider xattrProvider, @Nullable FileStatus status) throws IOException {
byte[] digest = xattrProvider.getFastDigest(path);
return digest != null ? digest : manuallyComputeDigest(path);
return digest != null ? digest : manuallyComputeDigest(path, status);
}

/**
* Calculates the digest manually.
* Calculates a digest manually (i.e., assuming that a fast digest can't obtained).
*
* <p>Prefer calling {@link #manuallyComputeDigest(Path, FileStatus)} when a recently obtained
* {@link FileStatus} is available.
*
* @param path Path of the file.
* @param path the file path
*/
public static byte[] manuallyComputeDigest(Path path) throws IOException {
return manuallyComputeDigest(path, null);
}

/**
* Same as {@link #manuallyComputeDigest(Path)}, but providing the ability to reuse a recently
* obtained {@link FileStatus}.
*
* @param path the file path
* @param status a recently obtained file status, if available
*/
public static byte[] manuallyComputeDigest(Path path, @Nullable FileStatus status)
throws IOException {
byte[] digest;

// Attempt a cache lookup if the cache is enabled.
Cache<CacheKey, byte[]> cache = globalCache;
CacheKey key = null;
if (cache != null) {
key = new CacheKey(path, path.stat());
key = new CacheKey(path, status != null ? status : path.stat());
digest = cache.getIfPresent(key);
if (digest != null) {
return digest;
Expand Down

0 comments on commit bb80319

Please sign in to comment.