feat(cache): cache v2 in the agent

.buildkite/cache.yml

@@ -0,0 +1,31 @@
+caches:
+  - name: gomodcache
+    # Module source (downloaded .go files, zips, go.mod) is
+    # platform-independent, so one key is shared across every os/arch.
+    cache_key:
+      - v2
+      - { checksum: go.sum }
+    # ~-anchored: resolved against $HOME, independent of cwd
+    target_paths:
+      - ~/.gomodcache
+  - name: gocache
+    # Build cache holds compiled objects, which are platform-specific, so
+    # it stays keyed by os+arch.
+    cache_key:
+      - v2
+      - { agent: os }
+      - { agent: arch }
+    # ~-anchored: resolved against $HOME, independent of cwd
+    target_paths:
+      - ~/.gocache
+  - name: gocache_race
+    # -race compiles to a different build hash than normal objects, so race
+    # runs get their own build cache (separate key + dir) rather than colliding
+    # with the plain gocache.
+    cache_key:
+      - v2
+      - { agent: os }
+      - { agent: arch }
+    # ~-anchored: resolved against $HOME, independent of cwd
+    target_paths:
+      - ~/.gocache_race

.buildkite/docker-compose.yml

@@ -5,29 +5,25 @@ services:
       dockerfile: Dockerfile-lint
     volumes:
       - ../:/work:cached
-      - ~/gocache:/gocache
-      - ~/gomodcache:/gomodcache
     working_dir: /work
-    environment:
-      - GOCACHE=/gocache
-      - GOMODCACHE=/gomodcache
+    # GOCACHE/GOMODCACHE are set to $HOME/.gocache and $HOME/.gomodcache by
+    # the scripts themselves (see check-code-committed.sh /
+    # check-protobuf-generation.sh), matching cache.yml's ~-anchored
+    # target_paths -- not set here, so there's exactly one place that can
+    # get the two out of sync.
 
   agent:
     build:
       context: .
       dockerfile: Dockerfile-compile
     volumes:
       - ../:/work:cached
-      - ~/gocache:/gocache
-      - ~/gomodcache:/gomodcache
     working_dir: /work
     environment:
       - BUILDKITE_BUILD_NUMBER
       - BUILDKITE_JOB_ID
       - "BUILDKITE_AGENT_TAGS=queue=default"
       - "BUILDKITE_BUILD_PATH=/buildkite"
-      - GOCACHE=/gocache
-      - GOMODCACHE=/gomodcache
       - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN
 
   e2e:

.buildkite/pipeline.yml

@@ -17,25 +17,49 @@ steps:
       - go.{mod,sum}
       - "**.go"
       - .buildkite/steps/check-code-committed.sh
+    env:
+      BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
     plugins:
+      - &cache_v2_role
+        aws-assume-role-with-web-identity#v1.4.0:
+          role_arn: "arn:aws:iam::445615400570:role/pipeline-buildkite-agent"
+          session-tags:
+            - organization_slug
+            - organization_id
+            - pipeline_slug
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
           cli-version: 2
           mount-buildkite-agent: true
           run: lint
+          env:
+            - BUILDKITE_AGENT_ACCESS_TOKEN
+            - BUILDKITE_AGENT_CACHE_STORE_URL
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - AWS_SESSION_TOKEN
 
   - name: ":go::robot_face: Check protobuf generation"
     key: check-protobuf-genreation
     command: .buildkite/steps/check-protobuf-generation.sh
     if_changed:
       - api/proto/**
       - .buildkite/steps/check-protobuf-generation.sh
+    env:
+      BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
     plugins:
+      - *cache_v2_role
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
           cli-version: 2
           mount-buildkite-agent: true
           run: lint
+          env:
+            - BUILDKITE_AGENT_ACCESS_TOKEN
+            - BUILDKITE_AGENT_CACHE_STORE_URL
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - AWS_SESSION_TOKEN
 
   # Send git commit metadata to the Test Engine plan-metadata pipeline on
   # every build. Plan output is discarded -- the Tests and Coverage group
@@ -82,7 +106,10 @@ steps:
         artifact_paths:
           - junit-*.xml
           - "coverage-*/**"
+        env:
+          BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
         plugins:
+          - *cache_v2_role
           - tests#v1.0.0:
               suite-slug: buildkite-agent
               test-runner: gotest
@@ -98,7 +125,13 @@ steps:
               config: .buildkite/docker-compose.yml
               cli-version: 2
               propagate-environment: true
+              mount-buildkite-agent: true
               run: agent
+              env:
+                - BUILDKITE_AGENT_CACHE_STORE_URL
+                - AWS_ACCESS_KEY_ID
+                - AWS_SECRET_ACCESS_KEY
+                - AWS_SESSION_TOKEN
 
       - name: ":linux: Linux ARM64 Tests"
         key: test-linux-arm64
@@ -109,7 +142,10 @@ steps:
           - "coverage-*/**"
         agents:
           queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
+        env:
+          BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
         plugins:
+          - *cache_v2_role
           - tests#v1.0.0:
               suite-slug: buildkite-agent
               test-runner: gotest
@@ -125,8 +161,19 @@ steps:
               config: .buildkite/docker-compose.yml
               cli-version: 2
               propagate-environment: true
+              mount-buildkite-agent: true
               run: agent
-
+              env:
+                - BUILDKITE_AGENT_CACHE_STORE_URL
+                - AWS_ACCESS_KEY_ID
+                - AWS_SECRET_ACCESS_KEY
+                - AWS_SESSION_TOKEN
+
+      # No docker-compose here (bare Windows host) -- buildkite-agent is
+      # already on PATH natively, so no mount-buildkite-agent equivalent is
+      # needed. aws-assume-role-with-web-identity's hooks are bash scripts,
+      # same as the command itself; not yet verified end-to-end on this
+      # queue specifically.
       - name: ":windows: Windows AMD64 Tests"
         key: test-windows
         command: "bash .buildkite\\steps\\tests.sh"
@@ -136,7 +183,10 @@ steps:
           - "coverage-*/**"
         agents:
           queue: $AGENT_RUNNERS_WINDOWS_QUEUE
+        env:
+          BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
         plugins:
+          - *cache_v2_role
           - tests#v1.0.0:
               suite-slug: buildkite-agent
               test-runner: gotest
@@ -159,7 +209,10 @@ steps:
           - "coverage-*/**"
         agents:
           queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
+        env:
+          BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
         plugins:
+          - *cache_v2_role
           - tests#v1.0.0:
               suite-slug: buildkite-agent
               test-runner: gotest
@@ -175,7 +228,13 @@ steps:
               config: .buildkite/docker-compose.yml
               cli-version: 2
               propagate-environment: true
+              mount-buildkite-agent: true
               run: agent
+              env:
+                - BUILDKITE_AGENT_CACHE_STORE_URL
+                - AWS_ACCESS_KEY_ID
+                - AWS_SECRET_ACCESS_KEY
+                - AWS_SESSION_TOKEN
 
       - name: ":coverage: Test coverage report Linux ARM64"
         key: test-coverage-linux-arm64
@@ -248,11 +307,20 @@ steps:
           - test-linux-amd64
           - test-linux-arm64
         artifact_paths: "pkg/*"
+        env:
+          BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
         plugins:
+          - *cache_v2_role
           - docker-compose#v4.14.0:
               config: .buildkite/docker-compose.yml
               cli-version: 2
+              mount-buildkite-agent: true
               run: agent
+              env:
+                - BUILDKITE_AGENT_CACHE_STORE_URL
+                - AWS_ACCESS_KEY_ID
+                - AWS_SECRET_ACCESS_KEY
+                - AWS_SESSION_TOKEN
           - generate-provenance-attestation#v1.1.0:
               artifacts: "pkg/*"
               attestation_name: "buildkite-agent-{{matrix.os}}-{{matrix.arch}}.build-attestation.json"
@@ -312,7 +380,10 @@ steps:
     key: test-bk-cli
     depends_on: build-binary
     command: ".buildkite/steps/test-bk.sh"
+    env:
+      BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1"
     plugins:
+      - *cache_v2_role
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
           cli-version: 2
@@ -321,7 +392,13 @@ steps:
             - BUILDKITE_AGENT_ACCESS_TOKEN
             - BUILDKITE_BUILD_ID
             - BUILDKITE_JOB_ID
+            - BUILDKITE_AGENT_CACHE_STORE_URL
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - AWS_SESSION_TOKEN
           volumes:
+            # buildkite-agent comes in via this volume mount, so no
+            # mount-buildkite-agent is needed for the cache CLI calls.
             - "/usr/bin/buildkite-agent:/usr/bin/buildkite-agent"
 
   - name: ":mag: Extract Agent Version Metadata"

.buildkite/steps/build-binary.sh

@@ -2,6 +2,15 @@
 
 set -euo pipefail
 
+# $HOME-anchored, matching cache.yml's ~-anchored target_paths.
+export GOMODCACHE="$HOME/.gomodcache"
+
+# gomodcache ONLY. We deliberately do NOT restore gocache: Go keys compiled objects by
+# the *target* GOOS/GOARCH, so the linux/amd64 gocache holds nothing a
+# cross-build can reuse.
+echo --- :inbox_tray: cache restore
+buildkite-agent cache restore --name gomodcache
+
 echo "--- :${1}: Building ${1}/${2}"
 
 rm -rf pkg

.buildkite/steps/check-code-committed.sh

@@ -2,6 +2,12 @@
 
 set -euf
 
+export GOCACHE="$HOME/.gocache"
+export GOMODCACHE="$HOME/.gomodcache"
+
+echo --- :inbox_tray: cache restore
+buildkite-agent cache restore --name gomodcache --name gocache
+
 echo --- :go: Checking go mod tidyness
 go mod tidy
 if ! git diff --no-ext-diff --exit-code; then
@@ -65,3 +71,16 @@ EOF
 fi
 
 echo +++ Everything is clean and tidy! 🎉
+
+# Ensure the module cache holds the full dependency graph before saving.
+
+# `go mod download` fetches every module's source (including deps imported
+# only under other-platform build tags) so the single, platform-independent
+# gomodcache key we save serves arm64 and windows restores too.
+echo --- :arrow_down: go mod download
+go mod download
+
+# Writer for two keys: the single platform-independent gomodcache, and the
+# linux/amd64 gocache. This step is unparallelized, so it never races itself.
+echo --- :outbox_tray: cache save
+buildkite-agent cache save --name gomodcache --name gocache

.buildkite/steps/check-protobuf-generation.sh

@@ -2,6 +2,12 @@
 
 set -euf
 
+export GOCACHE="$HOME/.gocache"
+export GOMODCACHE="$HOME/.gomodcache"
+
+echo --- :inbox_tray: cache restore
+buildkite-agent cache restore --name gomodcache --name gocache
+
 cd api/proto
 
 echo --- :buf: Installing buf...

.buildkite/steps/test-bk.sh

@@ -1,6 +1,13 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# $HOME-anchored, matching cache.yml's ~-anchored target_paths.
+export GOCACHE="$HOME/.gocache"
+export GOMODCACHE="$HOME/.gomodcache"
+
+# Restore only (Lint owns the linux/amd64 keys).
+echo --- :inbox_tray: cache restore
+buildkite-agent cache restore --name gomodcache --name gocache
 
 echo "--- :package: Downloading bk binary"
 go install github.com/buildkite/cli/v2/cmd/bk@cdcc5fa4b6e209f5ffa79469dad04938d6eed0cd

.buildkite/steps/tests.sh

@@ -4,11 +4,38 @@ set -euo pipefail
 go version
 echo arch is "$(uname -m)"
 
+# Detect race mode first -- it selects a SEPARATE gocache key + dir, because
+# -race compiles to a different build hash than normal objects, so the two
+# can't share one cache.
 RACE=''
 if [[ $* == *-race* ]] ; then
   RACE='-race'
 fi
 
+goos="$(go env GOOS)"
+goarch="$(go env GOARCH)"
+
+# The cache dir must resolve to the SAME location the agent expands.
+if [[ "${goos}" == "windows" ]]; then
+  cache_home="${USERPROFILE}"
+else
+  cache_home="${HOME}"
+fi
+
+# Non-race builds use gocache (~/.gocache). Race builds use a dedicated
+# gocache_race (~/.gocache_race).
+if [[ -n "${RACE}" ]]; then
+  gocache_name="gocache_race"
+  export GOCACHE="${cache_home}/.gocache_race"
+else
+  gocache_name="gocache"
+  export GOCACHE="${cache_home}/.gocache"
+fi
+export GOMODCACHE="${cache_home}/.gomodcache"
+
+echo --- :inbox_tray: cache restore
+buildkite-agent cache restore --name gomodcache --name "${gocache_name}"
+
 export BUILDKITE_TEST_ENGINE_SUITE_SLUG=buildkite-agent
 export BUILDKITE_TEST_ENGINE_TEST_RUNNER=gotest
 export BUILDKITE_TEST_ENGINE_RESULT_PATH="junit-${BUILDKITE_JOB_ID}.xml"
@@ -24,3 +51,9 @@ else
 fi
 
 go tool test-engine-client run
+
+# One writer per key, gated to shard 0 so parallel shards don't race it.
+if [[ "${BUILDKITE_PARALLEL_JOB:-0}" == "0" ]] && ! [[ "${goos}/${goarch}" == "linux/amd64" && -z "${RACE}" ]]; then
+  echo --- :outbox_tray: cache save
+  buildkite-agent cache save --name "${gocache_name}"
+fi