Merge branch 'master' into feature/support-ldap

README.md

@@ -665,6 +665,10 @@ If you want the docker container to run in the background pass the `-d` flag rig
 You can adjust the maximum cache size by appending `--max_size N`, where N is
 the maximum size in Gibibytes.
 
+### Docker Compose notes
+
+See [examples/docker-compose.yml](examples/docker-compose.yml) for an example configuration (modify the `--max_size` flag in there to suit your needs).
+
 ### Kubernetes notes
 
 * See [examples/kubernetes.yml](examples/kubernetes.yml) for an example
@@ -688,7 +692,6 @@ the maximum size in Gibibytes.
   alb.ingress.kubernetes.io/target-type: ip
   ```
 
-
 ### Build your own
 
 The command below will build a docker image from source and install it into your local docker registry.

cache/disk/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//cache/disk/casblob:go_default_library",
         "//cache/disk/zstdimpl:go_default_library",
         "//genproto/build/bazel/remote/execution/v2:go_default_library",
+        "//utils/annotate:go_default_library",
         "//utils/tempfile:go_default_library",
         "//utils/validate:go_default_library",
         "@com_github_djherbis_atime//:go_default_library",

cache/disk/casblob/casblob.go

@@ -561,6 +561,7 @@ func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t Compressio
 
 	nextChunk := 0 // Index in h.chunkOffsets.
 	remainingRawData := size
+	var numRead int
 
 	uncompressedChunk := make([]byte, chunkSize)
 
@@ -576,9 +577,9 @@ func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t Compressio
 		}
 		remainingRawData -= chunkEnd
 
-		_, err = io.ReadFull(r, uncompressedChunk[0:chunkEnd])
+		numRead, err = io.ReadFull(r, uncompressedChunk[0:chunkEnd])
 		if err != nil {
-			return -1, err
+			return -1, fmt.Errorf("Only managed to read %d of %d bytes: %w", numRead, chunkEnd, err)
 		}
 
 		compressedChunk := zstd.EncodeAll(uncompressedChunk[0:chunkEnd])
@@ -587,7 +588,7 @@ func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t Compressio
 
 		written, err := f.Write(compressedChunk)
 		if err != nil {
-			return -1, err
+			return -1, fmt.Errorf("Failed to write compressed chunk to disk: %w", err)
 		}
 
 		fileOffset += int64(written)
@@ -599,7 +600,7 @@ func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t Compressio
 	if err == nil {
 		return -1, fmt.Errorf("expected %d bytes but got at least %d more", size, bytesAfter)
 	} else if err != io.EOF {
-		return -1, err
+		return -1, fmt.Errorf("Failed to read chunk of size %d: %w", len(uncompressedChunk), err)
 	}
 
 	actualHash := hex.EncodeToString(hasher.Sum(nil))
@@ -611,18 +612,23 @@ func WriteAndClose(zstd zstdimpl.ZstdImpl, r io.Reader, f *os.File, t Compressio
 	// We know all the chunk offsets now, go back and fill those in.
 	_, err = f.Seek(chunkTableOffset, io.SeekStart)
 	if err != nil {
-		return -1, err
+		return -1, fmt.Errorf("Failed to seek to offset %d: %w", chunkTableOffset, err)
 	}
 
 	err = binary.Write(f, binary.LittleEndian, h.chunkOffsets)
 	if err != nil {
-		return -1, err
+		return -1, fmt.Errorf("Failed to write chunk offsets: %w", err)
 	}
 
 	err = f.Sync()
 	if err != nil {
-		return -1, err
+		return -1, fmt.Errorf("Failed to sync file: %w", err)
+	}
+
+	err = f.Close()
+	if err != nil {
+		return -1, fmt.Errorf("Failed to close file: %w", err)
 	}
 
-	return fileOffset, f.Close()
+	return fileOffset, nil
 }

cache/disk/disk.go

@@ -19,6 +19,7 @@ import (
 	"github.com/buchgr/bazel-remote/v2/cache"
 	"github.com/buchgr/bazel-remote/v2/cache/disk/casblob"
 	"github.com/buchgr/bazel-remote/v2/cache/disk/zstdimpl"
+	"github.com/buchgr/bazel-remote/v2/utils/annotate"
 	"github.com/buchgr/bazel-remote/v2/utils/tempfile"
 	"github.com/buchgr/bazel-remote/v2/utils/validate"
 
@@ -315,7 +316,7 @@ func (c *diskCache) Put(ctx context.Context, kind cache.EntryKind, hash string,
 	removeTempfile = true
 
 	var sizeOnDisk int64
-	sizeOnDisk, err = c.writeAndCloseFile(r, kind, hash, size, tf)
+	sizeOnDisk, err = c.writeAndCloseFile(ctx, r, kind, hash, size, tf)
 	if err != nil {
 		return internalErr(err)
 	}
@@ -340,7 +341,7 @@ func (c *diskCache) Put(ctx context.Context, kind cache.EntryKind, hash string,
 	return nil
 }
 
-func (c *diskCache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) (int64, error) {
+func (c *diskCache) writeAndCloseFile(ctx context.Context, r io.Reader, kind cache.EntryKind, hash string, size int64, f *os.File) (int64, error) {
 	closeFile := true
 	defer func() {
 		if closeFile {
@@ -354,27 +355,27 @@ func (c *diskCache) writeAndCloseFile(r io.Reader, kind cache.EntryKind, hash st
 	if kind == cache.CAS && c.storageMode != casblob.Identity {
 		sizeOnDisk, err = casblob.WriteAndClose(c.zstd, r, f, c.storageMode, hash, size)
 		if err != nil {
-			return -1, err
+			return -1, annotate.Err(ctx, "Failed to write compressed CAS blob to disk", err)
 		}
 		closeFile = false
 		return sizeOnDisk, nil
 	}
 
 	if sizeOnDisk, err = io.Copy(f, r); err != nil {
-		return -1, err
+		return -1, annotate.Err(ctx, "Failed to copy data to disk", err)
 	}
 
 	if isSizeMismatch(sizeOnDisk, size) {
 		return -1, fmt.Errorf(
-			"sizes don't match. Expected %d, found %d", size, sizeOnDisk)
+			"Sizes don't match. Expected %d, found %d", size, sizeOnDisk)
 	}
 
 	if err = f.Sync(); err != nil {
-		return -1, err
+		return -1, fmt.Errorf("Failed to sync file to disk: %w", err)
 	}
 
 	if err = f.Close(); err != nil {
-		return -1, err
+		return -1, fmt.Errorf("Failed to close file: %w", err)
 	}
 	closeFile = false