diff --git a/.buildkite/cache.yml b/.buildkite/cache.yml new file mode 100644 index 0000000000..2bf260bde7 --- /dev/null +++ b/.buildkite/cache.yml @@ -0,0 +1,31 @@ +caches: + - name: gomodcache + # Module source (downloaded .go files, zips, go.mod) is + # platform-independent, so one key is shared across every os/arch. + cache_key: + - v2 + - { checksum: go.sum } + # ~-anchored: resolved against $HOME, independent of cwd + target_paths: + - ~/.gomodcache + - name: gocache + # Build cache holds compiled objects, which are platform-specific, so + # it stays keyed by os+arch. + cache_key: + - v2 + - { agent: os } + - { agent: arch } + # ~-anchored: resolved against $HOME, independent of cwd + target_paths: + - ~/.gocache + - name: gocache_race + # -race compiles to a different build hash than normal objects, so race + # runs get their own build cache (separate key + dir) rather than colliding + # with the plain gocache. + cache_key: + - v2 + - { agent: os } + - { agent: arch } + # ~-anchored: resolved against $HOME, independent of cwd + target_paths: + - ~/.gocache_race diff --git a/.buildkite/docker-compose.yml b/.buildkite/docker-compose.yml index af3061cb71..d6162a3845 100644 --- a/.buildkite/docker-compose.yml +++ b/.buildkite/docker-compose.yml @@ -5,12 +5,12 @@ services: dockerfile: Dockerfile-lint volumes: - ../:/work:cached - - ~/gocache:/gocache - - ~/gomodcache:/gomodcache working_dir: /work - environment: - - GOCACHE=/gocache - - GOMODCACHE=/gomodcache + # GOCACHE/GOMODCACHE are set to $HOME/.gocache and $HOME/.gomodcache by + # the scripts themselves (see check-code-committed.sh / + # check-protobuf-generation.sh), matching cache.yml's ~-anchored + # target_paths -- not set here, so there's exactly one place that can + # get the two out of sync. agent: build: @@ -18,16 +18,12 @@ services: dockerfile: Dockerfile-compile volumes: - ../:/work:cached - - ~/gocache:/gocache - - ~/gomodcache:/gomodcache working_dir: /work environment: - BUILDKITE_BUILD_NUMBER - BUILDKITE_JOB_ID - "BUILDKITE_AGENT_TAGS=queue=default" - "BUILDKITE_BUILD_PATH=/buildkite" - - GOCACHE=/gocache - - GOMODCACHE=/gomodcache - BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN e2e: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7839d92b5a..3696a7a0ca 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -17,12 +17,27 @@ steps: - go.{mod,sum} - "**.go" - .buildkite/steps/check-code-committed.sh + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - &cache_v2_role + aws-assume-role-with-web-identity#v1.4.0: + role_arn: "arn:aws:iam::445615400570:role/pipeline-buildkite-agent" + session-tags: + - organization_slug + - organization_id + - pipeline_slug - docker-compose#v4.14.0: config: .buildkite/docker-compose.yml cli-version: 2 mount-buildkite-agent: true run: lint + env: + - BUILDKITE_AGENT_ACCESS_TOKEN + - BUILDKITE_AGENT_CACHE_STORE_URL + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - name: ":go::robot_face: Check protobuf generation" key: check-protobuf-genreation @@ -30,12 +45,21 @@ steps: if_changed: - api/proto/** - .buildkite/steps/check-protobuf-generation.sh + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - *cache_v2_role - docker-compose#v4.14.0: config: .buildkite/docker-compose.yml cli-version: 2 mount-buildkite-agent: true run: lint + env: + - BUILDKITE_AGENT_ACCESS_TOKEN + - BUILDKITE_AGENT_CACHE_STORE_URL + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN # Send git commit metadata to the Test Engine plan-metadata pipeline on # every build. Plan output is discarded -- the Tests and Coverage group @@ -82,7 +106,10 @@ steps: artifact_paths: - junit-*.xml - "coverage-*/**" + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - *cache_v2_role - tests#v1.0.0: suite-slug: buildkite-agent test-runner: gotest @@ -98,7 +125,13 @@ steps: config: .buildkite/docker-compose.yml cli-version: 2 propagate-environment: true + mount-buildkite-agent: true run: agent + env: + - BUILDKITE_AGENT_CACHE_STORE_URL + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - name: ":linux: Linux ARM64 Tests" key: test-linux-arm64 @@ -109,7 +142,10 @@ steps: - "coverage-*/**" agents: queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - *cache_v2_role - tests#v1.0.0: suite-slug: buildkite-agent test-runner: gotest @@ -125,8 +161,19 @@ steps: config: .buildkite/docker-compose.yml cli-version: 2 propagate-environment: true + mount-buildkite-agent: true run: agent - + env: + - BUILDKITE_AGENT_CACHE_STORE_URL + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + + # No docker-compose here (bare Windows host) -- buildkite-agent is + # already on PATH natively, so no mount-buildkite-agent equivalent is + # needed. aws-assume-role-with-web-identity's hooks are bash scripts, + # same as the command itself; not yet verified end-to-end on this + # queue specifically. - name: ":windows: Windows AMD64 Tests" key: test-windows command: "bash .buildkite\\steps\\tests.sh" @@ -136,7 +183,10 @@ steps: - "coverage-*/**" agents: queue: $AGENT_RUNNERS_WINDOWS_QUEUE + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - *cache_v2_role - tests#v1.0.0: suite-slug: buildkite-agent test-runner: gotest @@ -159,7 +209,10 @@ steps: - "coverage-*/**" agents: queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - *cache_v2_role - tests#v1.0.0: suite-slug: buildkite-agent test-runner: gotest @@ -175,7 +228,13 @@ steps: config: .buildkite/docker-compose.yml cli-version: 2 propagate-environment: true + mount-buildkite-agent: true run: agent + env: + - BUILDKITE_AGENT_CACHE_STORE_URL + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - name: ":coverage: Test coverage report Linux ARM64" key: test-coverage-linux-arm64 @@ -248,11 +307,20 @@ steps: - test-linux-amd64 - test-linux-arm64 artifact_paths: "pkg/*" + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - *cache_v2_role - docker-compose#v4.14.0: config: .buildkite/docker-compose.yml cli-version: 2 + mount-buildkite-agent: true run: agent + env: + - BUILDKITE_AGENT_CACHE_STORE_URL + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN - generate-provenance-attestation#v1.1.0: artifacts: "pkg/*" attestation_name: "buildkite-agent-{{matrix.os}}-{{matrix.arch}}.build-attestation.json" @@ -312,7 +380,10 @@ steps: key: test-bk-cli depends_on: build-binary command: ".buildkite/steps/test-bk.sh" + env: + BUILDKITE_AGENT_CACHE_STORE_URL: "s3://bk-agent-cache-v2-dogfood-445615400570-us-east-1?region=us-east-1" plugins: + - *cache_v2_role - docker-compose#v4.14.0: config: .buildkite/docker-compose.yml cli-version: 2 @@ -321,7 +392,13 @@ steps: - BUILDKITE_AGENT_ACCESS_TOKEN - BUILDKITE_BUILD_ID - BUILDKITE_JOB_ID + - BUILDKITE_AGENT_CACHE_STORE_URL + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN volumes: + # buildkite-agent comes in via this volume mount, so no + # mount-buildkite-agent is needed for the cache CLI calls. - "/usr/bin/buildkite-agent:/usr/bin/buildkite-agent" - name: ":mag: Extract Agent Version Metadata" diff --git a/.buildkite/steps/build-binary.sh b/.buildkite/steps/build-binary.sh index 3981591cb4..4f03731920 100755 --- a/.buildkite/steps/build-binary.sh +++ b/.buildkite/steps/build-binary.sh @@ -2,6 +2,15 @@ set -euo pipefail +# $HOME-anchored, matching cache.yml's ~-anchored target_paths. +export GOMODCACHE="$HOME/.gomodcache" + +# gomodcache ONLY. We deliberately do NOT restore gocache: Go keys compiled objects by +# the *target* GOOS/GOARCH, so the linux/amd64 gocache holds nothing a +# cross-build can reuse. +echo --- :inbox_tray: cache restore +buildkite-agent cache restore --name gomodcache + echo "--- :${1}: Building ${1}/${2}" rm -rf pkg diff --git a/.buildkite/steps/check-code-committed.sh b/.buildkite/steps/check-code-committed.sh index 90099c4194..6e720ae363 100755 --- a/.buildkite/steps/check-code-committed.sh +++ b/.buildkite/steps/check-code-committed.sh @@ -2,6 +2,12 @@ set -euf +export GOCACHE="$HOME/.gocache" +export GOMODCACHE="$HOME/.gomodcache" + +echo --- :inbox_tray: cache restore +buildkite-agent cache restore --name gomodcache --name gocache + echo --- :go: Checking go mod tidyness go mod tidy if ! git diff --no-ext-diff --exit-code; then @@ -65,3 +71,16 @@ EOF fi echo +++ Everything is clean and tidy! 🎉 + +# Ensure the module cache holds the full dependency graph before saving. + +# `go mod download` fetches every module's source (including deps imported +# only under other-platform build tags) so the single, platform-independent +# gomodcache key we save serves arm64 and windows restores too. +echo --- :arrow_down: go mod download +go mod download + +# Writer for two keys: the single platform-independent gomodcache, and the +# linux/amd64 gocache. This step is unparallelized, so it never races itself. +echo --- :outbox_tray: cache save +buildkite-agent cache save --name gomodcache --name gocache diff --git a/.buildkite/steps/check-protobuf-generation.sh b/.buildkite/steps/check-protobuf-generation.sh index 590c374b58..b0caa5fc20 100755 --- a/.buildkite/steps/check-protobuf-generation.sh +++ b/.buildkite/steps/check-protobuf-generation.sh @@ -2,6 +2,12 @@ set -euf +export GOCACHE="$HOME/.gocache" +export GOMODCACHE="$HOME/.gomodcache" + +echo --- :inbox_tray: cache restore +buildkite-agent cache restore --name gomodcache --name gocache + cd api/proto echo --- :buf: Installing buf... diff --git a/.buildkite/steps/test-bk.sh b/.buildkite/steps/test-bk.sh index 79b32c0caf..c2d663377e 100755 --- a/.buildkite/steps/test-bk.sh +++ b/.buildkite/steps/test-bk.sh @@ -1,6 +1,13 @@ #!/usr/bin/env bash set -euo pipefail +# $HOME-anchored, matching cache.yml's ~-anchored target_paths. +export GOCACHE="$HOME/.gocache" +export GOMODCACHE="$HOME/.gomodcache" + +# Restore only (Lint owns the linux/amd64 keys). +echo --- :inbox_tray: cache restore +buildkite-agent cache restore --name gomodcache --name gocache echo "--- :package: Downloading bk binary" go install github.com/buildkite/cli/v2/cmd/bk@cdcc5fa4b6e209f5ffa79469dad04938d6eed0cd diff --git a/.buildkite/steps/tests.sh b/.buildkite/steps/tests.sh index 859d539350..363004e9c3 100755 --- a/.buildkite/steps/tests.sh +++ b/.buildkite/steps/tests.sh @@ -4,11 +4,38 @@ set -euo pipefail go version echo arch is "$(uname -m)" +# Detect race mode first -- it selects a SEPARATE gocache key + dir, because +# -race compiles to a different build hash than normal objects, so the two +# can't share one cache. RACE='' if [[ $* == *-race* ]] ; then RACE='-race' fi +goos="$(go env GOOS)" +goarch="$(go env GOARCH)" + +# The cache dir must resolve to the SAME location the agent expands. +if [[ "${goos}" == "windows" ]]; then + cache_home="${USERPROFILE}" +else + cache_home="${HOME}" +fi + +# Non-race builds use gocache (~/.gocache). Race builds use a dedicated +# gocache_race (~/.gocache_race). +if [[ -n "${RACE}" ]]; then + gocache_name="gocache_race" + export GOCACHE="${cache_home}/.gocache_race" +else + gocache_name="gocache" + export GOCACHE="${cache_home}/.gocache" +fi +export GOMODCACHE="${cache_home}/.gomodcache" + +echo --- :inbox_tray: cache restore +buildkite-agent cache restore --name gomodcache --name "${gocache_name}" + export BUILDKITE_TEST_ENGINE_SUITE_SLUG=buildkite-agent export BUILDKITE_TEST_ENGINE_TEST_RUNNER=gotest export BUILDKITE_TEST_ENGINE_RESULT_PATH="junit-${BUILDKITE_JOB_ID}.xml" @@ -24,3 +51,9 @@ else fi go tool test-engine-client run + +# One writer per key, gated to shard 0 so parallel shards don't race it. +if [[ "${BUILDKITE_PARALLEL_JOB:-0}" == "0" ]] && ! [[ "${goos}/${goarch}" == "linux/amd64" && -z "${RACE}" ]]; then + echo --- :outbox_tray: cache save + buildkite-agent cache save --name "${gocache_name}" +fi