Extract bundle directly to final destinations, no staging dir or symlinks

Replace the tmpDir → rename/symlink approach with prefix-aware routing that
extracts tar entries directly to their final locations. Bundle entries are
routed by their first path component:
  ./caches/...              → GRADLE_USER_HOME/caches/...
  ./configuration-cache/... → <project>/.gradle/configuration-cache/...
  everything else           → <project>/...  (buildSrc/build, plugins/*/build, …)

Existing files are skipped (skipExisting=true) so a partial pre-existing cache
is merged rather than overwritten. This eliminates the cc-keystore lock
contention issue that arose when /tmp used a different filesystem than
~/.gradle, and removes all the tmpDir/rename/symlink/isExdev machinery.

🤖 Generated with [Firebender](https://firebender.com)

Co-Authored-By: Firebender <help@firebender.com>

Author: joshfriend

cmd/gradle-cache/extract_darwin.go

@@ -33,7 +33,16 @@ const mmapThreshold = 64 * 1024
 // handles concurrent writes to independent files efficiently, so parallel
 // workers saturate IOPS better than a single sequential writer would.
 func extractTarPlatform(r io.Reader, dir string) error {
-	return extractTarGo(r, dir)
+	return extractTarGoRouted(r, func(name string) string {
+		return filepath.Join(dir, name)
+	}, false)
+}
+
+// extractTarPlatformRouted is the routing-aware variant of extractTarPlatform.
+// targetFn maps a cleaned tar entry name to its absolute destination path.
+// If skipExisting is true, files that already exist on disk are left untouched.
+func extractTarPlatformRouted(r io.Reader, targetFn func(string) string, skipExisting bool) error {
+	return extractTarGoRouted(r, targetFn, skipExisting)
 }
 
 // extractTarGo reads a tar stream and extracts it into dir using a Go worker
@@ -49,6 +58,15 @@ func extractTarPlatform(r io.Reader, dir string) error {
 //     allocates one contiguous extent; the Mach VM subsystem then writes pages
 //     lazily via fault-in rather than blocking on each write() call.
 func extractTarGo(r io.Reader, dir string) error {
+	return extractTarGoRouted(r, func(name string) string {
+		return filepath.Join(dir, name)
+	}, false)
+}
+
+// extractTarGoRouted is the core parallel extractor. targetFn maps a cleaned
+// tar entry name (e.g. "caches/8.14.3/foo") to its absolute destination path.
+// skipExisting skips writing files that already exist on disk.
+func extractTarGoRouted(r io.Reader, targetFn func(string) string, skipExisting bool) error {
 	type fileJob struct {
 		path string
 		mode os.FileMode
@@ -93,7 +111,6 @@ func extractTarGo(r io.Reader, dir string) error {
 	}
 
 	tr := tar.NewReader(r)
-	cleanDir := filepath.Clean(dir) + string(os.PathSeparator)
 	var readErr error
 readLoop:
 	for {
@@ -106,12 +123,15 @@ readLoop:
 			break
 		}
 
-		target := filepath.Join(dir, hdr.Name)
-		if !strings.HasPrefix(filepath.Clean(target)+string(os.PathSeparator), cleanDir) {
+		// Reject path traversal before passing to targetFn.
+		name := filepath.Clean(hdr.Name)
+		if strings.HasPrefix(name, "..") {
 			readErr = errors.Errorf("tar entry %q escapes destination directory", hdr.Name)
 			break
 		}
 
+		target := targetFn(name)
+
 		switch hdr.Typeflag {
 		case tar.TypeDir:
 			if err := ensureDir(target); err != nil {
@@ -120,6 +140,11 @@ readLoop:
 			}
 
 		case tar.TypeReg:
+			if skipExisting {
+				if _, err := os.Lstat(target); err == nil {
+					continue
+				}
+			}
 			if err := ensureDir(filepath.Dir(target)); err != nil {
 				readErr = errors.Errorf("mkdir %s: %w", hdr.Name, err)
 				break readLoop
@@ -138,6 +163,11 @@ readLoop:
 			jobs <- fileJob{path: target, mode: hdr.FileInfo().Mode(), buf: bufPtr}
 
 		case tar.TypeSymlink:
+			if skipExisting {
+				if _, err := os.Lstat(target); err == nil {
+					continue
+				}
+			}
 			if err := ensureDir(filepath.Dir(target)); err != nil {
 				readErr = errors.Errorf("mkdir for symlink %s: %w", hdr.Name, err)
 				break readLoop

cmd/gradle-cache/extract_default.go

@@ -18,21 +18,38 @@ import (
 // writes from multiple goroutines fragment the writeback queue and are
 // consistently slower on Linux ext4/xfs despite identical throughput on APFS.
 func extractTarPlatform(r io.Reader, dir string) error {
-	return extractTarSeq(r, dir)
+	return extractTarSeqRouted(r, func(name string) string {
+		return filepath.Join(dir, name)
+	}, false)
+}
+
+// extractTarPlatformRouted is the routing-aware variant of extractTarPlatform.
+// targetFn maps a cleaned tar entry name to its absolute destination path.
+// If skipExisting is true, files that already exist on disk are left untouched.
+func extractTarPlatformRouted(r io.Reader, targetFn func(string) string, skipExisting bool) error {
+	return extractTarSeqRouted(r, targetFn, skipExisting)
 }
 
 // extractTarSeq extracts a tar stream sequentially using a fixed-size copy
 // buffer. Files are streamed directly from the tar reader to disk one 1 MiB
 // block at a time — the same block-streaming pattern GNU tar uses — so the
 // decompressor pipe keeps flowing without large per-file allocations.
 func extractTarSeq(r io.Reader, dir string) error {
+	return extractTarSeqRouted(r, func(name string) string {
+		return filepath.Join(dir, name)
+	}, false)
+}
+
+// extractTarSeqRouted is the core sequential extractor. targetFn maps a
+// cleaned tar entry name (e.g. "caches/8.14.3/foo") to its absolute
+// destination path. skipExisting skips writing files that already exist.
+func extractTarSeqRouted(r io.Reader, targetFn func(string) string, skipExisting bool) error {
 	// Single fixed-size copy buffer for all file writes in this call.
 	// 1 MiB is large enough to amortise write syscall overhead without
 	// creating memory pressure for many-file archives.
 	copyBuf := make([]byte, 1<<20)
 
 	tr := tar.NewReader(r)
-	cleanDir := filepath.Clean(dir) + string(os.PathSeparator)
 
 	// createdDirs tracks parent directories we have already MkdirAll'd so
 	// each unique path is only created once (same optimisation as darwin).
@@ -57,18 +74,26 @@ func extractTarSeq(r io.Reader, dir string) error {
 			return errors.Wrap(err, "read tar entry")
 		}
 
-		target := filepath.Join(dir, hdr.Name)
-		if !strings.HasPrefix(filepath.Clean(target)+string(os.PathSeparator), cleanDir) {
+		// Reject path traversal before passing to targetFn.
+		name := filepath.Clean(hdr.Name)
+		if strings.HasPrefix(name, "..") {
 			return errors.Errorf("tar entry %q escapes destination directory", hdr.Name)
 		}
 
+		target := targetFn(name)
+
 		switch hdr.Typeflag {
 		case tar.TypeDir:
 			if err := ensureDir(target); err != nil {
 				return errors.Errorf("mkdir %s: %w", hdr.Name, err)
 			}
 
 		case tar.TypeReg:
+			if skipExisting {
+				if _, err := os.Lstat(target); err == nil {
+					continue
+				}
+			}
 			if err := ensureDir(filepath.Dir(target)); err != nil {
 				return errors.Errorf("mkdir %s: %w", hdr.Name, err)
 			}
@@ -85,6 +110,11 @@ func extractTarSeq(r io.Reader, dir string) error {
 			}
 
 		case tar.TypeSymlink:
+			if skipExisting {
+				if _, err := os.Lstat(target); err == nil {
+					continue
+				}
+			}
 			if err := ensureDir(filepath.Dir(target)); err != nil {
 				return errors.Errorf("mkdir for symlink %s: %w", hdr.Name, err)
 			}

cmd/gradle-cache/main.go

@@ -5,8 +5,8 @@
 // This format is compatible with the bundled-cache-manager Ruby script.
 //
 // On restore, the tool walks the local git history (counting distinct-author
-// "blocks") to find the most recent S3 hit, downloads it, extracts it to a
-// temporary directory, and symlinks $GRADLE_USER_HOME/caches into place.
+// "blocks") to find the most recent S3 hit, downloads it, and extracts it
+// directly into the final destination directories (no staging dir, no symlinks).
 // A restore marker file is written immediately after extraction; save-delta uses
 // its mtime as the baseline to identify files created during the build.
 //
@@ -233,9 +233,27 @@ func (c *RestoreCmd) Run(ctx context.Context) error {
 	dlStart := time.Now()
 	slog.Info("downloading bundle", "commit", hitCommit[:min(8, len(hitCommit))])
 
-	tmpDir, err := os.MkdirTemp("", "gradle-cache-*")
+	// Ensure GRADLE_USER_HOME exists before extracting into it.
+	if err := os.MkdirAll(c.GradleUserHome, 0o750); err != nil {
+		return errors.Wrap(err, "create gradle user home dir")
+	}
+
+	// Resolve the project directory upfront; bundle entries are routed here for
+	// configuration-cache and convention build dirs.
+	projectDir, err := os.Getwd()
 	if err != nil {
-		return errors.Wrap(err, "create temp dir")
+		return errors.Wrap(err, "get working directory")
+	}
+
+	// Route tar entries to their final destinations directly:
+	//   ./caches/...               → GRADLE_USER_HOME/caches/...
+	//   ./configuration-cache/...  → <project>/.gradle/configuration-cache/...
+	//   everything else            → <project>/...  (buildSrc/build, plugins/*/build, …)
+	// Existing files are left untouched (skipExisting=true) so a partial
+	// pre-existing cache is merged rather than overwritten.
+	rules := []extractRule{
+		{prefix: "caches/", baseDir: c.GradleUserHome},
+		{prefix: "configuration-cache/", baseDir: filepath.Join(projectDir, ".gradle")},
 	}
 
 	body, err := store.get(ctx, hitCommit, c.CacheKey, hitSize)
@@ -247,7 +265,7 @@ func (c *RestoreCmd) Run(ctx context.Context) error {
 	// countingBody records bytes consumed and timestamps when the S3 body is
 	// exhausted so we can log download speed independently of extraction.
 	cb := &countingBody{r: body, dlStart: dlStart}
-	if err := extractTarZstd(ctx, cb, tmpDir); err != nil {
+	if err := extractBundleZstd(ctx, cb, rules, projectDir); err != nil {
 		return errors.Wrap(err, "extract bundle")
 	}
 
@@ -268,23 +286,6 @@ func (c *RestoreCmd) Run(ctx context.Context) error {
 	slog.Info("restore pipeline complete",
 		"total_duration", totalElapsed.Round(time.Millisecond))
 
-	// Symlink $GRADLE_USER_HOME/caches → tmpDir/caches.
-	cachesTarget := filepath.Join(tmpDir, "caches")
-	if _, err := os.Stat(cachesTarget); err != nil {
-		return errors.Errorf("extracted bundle does not contain a caches/ directory: %w", err)
-	}
-	localCaches := filepath.Join(c.GradleUserHome, "caches")
-	if err := os.MkdirAll(c.GradleUserHome, 0o750); err != nil {
-		return errors.Wrap(err, "create gradle user home dir")
-	}
-	if err := os.RemoveAll(localCaches); err != nil {
-		return errors.Wrap(err, "remove existing caches dir")
-	}
-	if err := os.Symlink(cachesTarget, localCaches); err != nil {
-		return errors.Wrap(err, "symlink caches dir")
-	}
-	slog.Info("restored", "link", localCaches, "target", cachesTarget)
-
 	// Write a marker recording when the base restore finished.
 	// save-delta compares file mtimes against this to identify files created
 	// during the build. Our Go extractor never calls chtimes, so all restored
@@ -321,57 +322,59 @@ func (c *RestoreCmd) Run(ctx context.Context) error {
 		}
 	}
 
-	// Restore configuration-cache and convention build dirs from the current directory.
-	projectDir, err := os.Getwd()
-	if err != nil {
-		return errors.Wrap(err, "get working directory")
-	}
-	if err := restoreProjectDirs(tmpDir, projectDir, c.IncludedBuilds); err != nil {
-		return err
-	}
-
 	slog.Debug("restore complete", "total_duration", time.Since(totalStart))
 	return nil
 }
 
-// restoreProjectDirs symlinks configuration-cache and included build output dirs
-// from tmpDir into projectDir, if present in the extracted bundle.
-// includedBuilds specifies which directories to check (see conventionBuildDirs).
-func restoreProjectDirs(tmpDir, projectDir string, includedBuilds []string) error {
-	// configuration-cache: archived at ./configuration-cache/ relative to the bundle root
-	// (not under .gradle/), matching the bundled-cache-manager.rb archive format.
-	srcCC := filepath.Join(tmpDir, "configuration-cache")
-	if _, err := os.Stat(srcCC); err == nil {
-		dstCC := filepath.Join(projectDir, ".gradle", "configuration-cache")
-		if err := os.MkdirAll(filepath.Dir(dstCC), 0o750); err != nil {
-			return errors.Wrap(err, "create .gradle dir")
-		}
-		if err := os.RemoveAll(dstCC); err != nil {
-			return errors.Wrap(err, "remove existing configuration-cache")
-		}
-		if err := os.Symlink(srcCC, dstCC); err != nil {
-			return errors.Wrap(err, "symlink configuration-cache")
-		}
-		slog.Info("restored", "link", dstCC, "target", srcCC)
+// extractRule maps a tar entry path prefix to a destination base directory.
+// For an entry "prefix/rest/of/path", the file is placed at
+// filepath.Join(baseDir, "prefix/rest/of/path").
+type extractRule struct {
+	prefix  string // without leading "./"
+	baseDir string
+}
+
+// extractBundleZstd decompresses and extracts a base bundle, routing tar
+// entries to their final destinations based on rules. Any entry whose path does
+// not match a rule is placed under defaultDir. Existing files are not
+// overwritten (skipExisting semantics), so a partial pre-existing cache is
+// merged rather than replaced.
+func extractBundleZstd(ctx context.Context, r io.Reader, rules []extractRule, defaultDir string) error {
+	zstdCmd := zstdDecompressCmd(ctx)
+	zstdCmd.Stdin = r
+
+	var zstdStderr bytes.Buffer
+	zstdCmd.Stderr = &zstdStderr
+
+	zstdOut, err := zstdCmd.StdoutPipe()
+	if err != nil {
+		return errors.Wrap(err, "zstd stdout pipe")
 	}
 
-	// Included build output dirs present in the extracted bundle.
-	for _, rel := range conventionBuildDirs(tmpDir, includedBuilds) {
-		src := filepath.Join(tmpDir, rel)
-		dst := filepath.Join(projectDir, rel)
-		if err := os.MkdirAll(filepath.Dir(dst), 0o750); err != nil {
-			return errors.Errorf("create parent of %s: %w", dst, err)
-		}
-		if err := os.RemoveAll(dst); err != nil {
-			return errors.Errorf("remove existing %s: %w", dst, err)
-		}
-		if err := os.Symlink(src, dst); err != nil {
-			return errors.Errorf("symlink %s: %w", rel, err)
+	if err := zstdCmd.Start(); err != nil {
+		return errors.Wrap(err, "start zstd")
+	}
+
+	targetFn := func(name string) string {
+		for _, rule := range rules {
+			if strings.HasPrefix(name, rule.prefix) {
+				return filepath.Join(rule.baseDir, name)
+			}
 		}
-		slog.Info("restored", "link", dst, "target", src)
+		return filepath.Join(defaultDir, name)
 	}
 
-	return nil
+	extractErr := extractTarPlatformRouted(zstdOut, targetFn, true)
+	zstdErr := zstdCmd.Wait()
+
+	var errs []error
+	if extractErr != nil {
+		errs = append(errs, extractErr)
+	}
+	if zstdErr != nil {
+		errs = append(errs, errors.Errorf("zstd: %w: %s", zstdErr, zstdStderr.String()))
+	}
+	return errors.Join(errs...)
 }
 
 // RestoreDeltaCmd downloads and applies a branch-specific delta bundle on top of