Add branch delta cache support (restore-delta, save-delta)

Introduces a delta workflow for PR builds that layers a branch-specific
bundle on top of the main-branch base cache. After a base restore, save-delta
scans for files with mtime newer than a restore marker and uploads a cumulative
delta keyed by branch name (survives rebases/force-pushes). On the next build,
restore --branch downloads both bundles concurrently so the delta's network
latency is hidden behind the base extraction.

Also adds a bundleStore abstraction over S3 and cachew, consolidates
extract_linux.go into extract_default.go, and adds tests and benchmarks for
the parallel mtime scanner (4× faster than filepath.Walk on cold APFS for
212k files: 3.4s vs 14s).

Author: joshfriend

README.md

@@ -2,10 +2,10 @@
 
 A CLI tool for saving and restoring Gradle build cache bundles from S3.
 
-Bundles are stored in S3 keyed by commit SHA, so `restore` doesn't need to know
+Bundles are stored keyed by commit SHA, so `restore` doesn't need to know
 exactly which commit produced a given bundle. Instead, it walks the local git
 history from a given ref (default: `HEAD`) and tries each commit SHA in order,
-newest first, until it finds a bundle that exists in S3. This means a developer
+newest first, until it finds a bundle that exists. This means a developer
 on a feature branch will automatically get the bundle from the most recent
 main-branch commit that has one, without needing to know its SHA in advance.
 
@@ -23,6 +23,8 @@ This installs the latest release to `~/.local/bin`. Set `INSTALL_DIR` to overrid
 
 ## Usage
 
+### Base cache (main branch)
+
 ```
 gradle-cache restore --bucket <bucket> --cache-key <key> [--ref main]
 gradle-cache save    --bucket <bucket> --cache-key <key>
@@ -37,8 +39,27 @@ are archived alongside `$GRADLE_USER_HOME/caches`. Accepts a direct path
 (`buildSrc`, `build-logic`) or a glob (`plugins/*`) to include all
 subdirectories. Defaults to `buildSrc`.
 
-Credentials are resolved via the standard AWS credential chain (environment
-variables, IRSA, instance profiles, etc.).
+### Branch delta cache (PR branches)
+
+For PR builds, pass `--branch` to `restore` to apply a branch delta in the same invocation. The delta bundle is downloaded concurrently with the base extraction so it adds no extra latency:
+
+```sh
+# Restore phase (single invocation)
+gradle-cache restore     --bucket <bucket> --cache-key <key> --ref main --branch $BRANCH_NAME
+
+# ... run the Gradle build ...
+
+# Save phase
+gradle-cache save-delta  --bucket <bucket> --cache-key <key> --branch $BRANCH_NAME
+```
+
+After the build, `save-delta` scans for files created since the restore marker and uploads a cumulative delta bundle keyed by branch name — so it survives rebases and force-pushes without any extra bookkeeping.
+
+If you need to apply a delta separately (e.g. the base was already restored by another step), `restore-delta` is still available as a standalone subcommand.
+
+### Credentials
+
+S3 credentials are resolved via the standard AWS credential chain (environment variables, IRSA, instance profiles, etc.).
 
 ## License
 

cmd/gradle-cache/cachew.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/alecthomas/errors"
+)
+
+// cachewClient stores and retrieves Gradle cache bundles via cachew's generic
+// object API, removing the need for AWS credentials on the client.
+type cachewClient struct {
+	baseURL string
+	http    *http.Client
+}
+
+func newCachewClient(baseURL string) *cachewClient {
+	return &cachewClient{
+		baseURL: strings.TrimRight(baseURL, "/"),
+		http:    &http.Client{},
+	}
+}
+
+func (c *cachewClient) objectURL(commit, cacheKey string) string {
+	return fmt.Sprintf("%s/api/v1/object/%s/%s",
+		c.baseURL,
+		url.PathEscape(cacheKey),
+		url.PathEscape(commit),
+	)
+}
+
+// stat returns (0, nil) when the bundle exists. Size is not used by this
+// backend since cachew does not expose Content-Length on HEAD and parallel
+// range downloads are handled server-side.
+func (c *cachewClient) stat(ctx context.Context, commit, cacheKey string) (int64, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, c.objectURL(commit, cacheKey), nil)
+	if err != nil {
+		return 0, err
+	}
+	resp, err := c.http.Do(req) //nolint:gosec
+	if err != nil {
+		return 0, err
+	}
+	io.Copy(io.Discard, resp.Body) //nolint:errcheck,gosec
+	resp.Body.Close()              //nolint:errcheck,gosec
+	if resp.StatusCode == http.StatusNotFound {
+		return 0, errors.Errorf("cachew: not found for %.8s", commit)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return 0, errors.Errorf("cachew HEAD %.8s: status %d", commit, resp.StatusCode)
+	}
+	return 0, nil
+}
+
+func (c *cachewClient) get(ctx context.Context, commit, cacheKey string, _ int64) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.objectURL(commit, cacheKey), nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.http.Do(req) //nolint:gosec
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		resp.Body.Close() //nolint:errcheck,gosec
+		return nil, errors.Errorf("cachew GET %.8s: status %d: %s", commit, resp.StatusCode, msg)
+	}
+	return resp.Body, nil
+}
+
+func (c *cachewClient) put(ctx context.Context, commit, cacheKey string, r io.ReadSeeker, size int64) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.objectURL(commit, cacheKey), r)
+	if err != nil {
+		return err
+	}
+	req.ContentLength = size
+	req.Header.Set("Content-Type", "application/zstd")
+	req.Header.Set("Time-To-Live", "168h") // 7 days
+	resp, err := c.http.Do(req)            //nolint:gosec
+	if err != nil {
+		return err
+	}
+	msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+	resp.Body.Close() //nolint:errcheck,gosec
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("cachew POST %.8s: status %d: %s", commit, resp.StatusCode, msg)
+	}
+	return nil
+}

cmd/gradle-cache/extract_default.go

@@ -1,4 +1,4 @@
-//go:build !darwin && !linux
+//go:build !darwin
 
 package main
 
@@ -12,16 +12,30 @@ import (
 	"github.com/alecthomas/errors"
 )
 
-// extractTarPlatform falls back to sequential streaming extraction on unknown platforms.
+// extractTarPlatform uses sequential extraction on non-darwin platforms. The
+// Linux VFS writeback path coalesces dirty pages most efficiently when a single
+// writer produces sequential writes, matching the behaviour of GNU tar. Parallel
+// writes from multiple goroutines fragment the writeback queue and are
+// consistently slower on Linux ext4/xfs despite identical throughput on APFS.
 func extractTarPlatform(r io.Reader, dir string) error {
 	return extractTarSeq(r, dir)
 }
 
+// extractTarSeq extracts a tar stream sequentially using a fixed-size copy
+// buffer. Files are streamed directly from the tar reader to disk one 1 MiB
+// block at a time — the same block-streaming pattern GNU tar uses — so the
+// decompressor pipe keeps flowing without large per-file allocations.
 func extractTarSeq(r io.Reader, dir string) error {
+	// Single fixed-size copy buffer for all file writes in this call.
+	// 1 MiB is large enough to amortise write syscall overhead without
+	// creating memory pressure for many-file archives.
 	copyBuf := make([]byte, 1<<20)
+
 	tr := tar.NewReader(r)
 	cleanDir := filepath.Clean(dir) + string(os.PathSeparator)
 
+	// createdDirs tracks parent directories we have already MkdirAll'd so
+	// each unique path is only created once (same optimisation as darwin).
 	createdDirs := make(map[string]struct{})
 	ensureDir := func(d string) error {
 		if _, ok := createdDirs[d]; ok {
@@ -53,6 +67,7 @@ func extractTarSeq(r io.Reader, dir string) error {
 			if err := ensureDir(target); err != nil {
 				return errors.Errorf("mkdir %s: %w", hdr.Name, err)
 			}
+
 		case tar.TypeReg:
 			if err := ensureDir(filepath.Dir(target)); err != nil {
 				return errors.Errorf("mkdir %s: %w", hdr.Name, err)
@@ -68,6 +83,7 @@ func extractTarSeq(r io.Reader, dir string) error {
 			if err := f.Close(); err != nil {
 				return errors.Errorf("close %s: %w", hdr.Name, err)
 			}
+
 		case tar.TypeSymlink:
 			if err := ensureDir(filepath.Dir(target)); err != nil {
 				return errors.Errorf("mkdir for symlink %s: %w", hdr.Name, err)