From b9ff893c7ed519e0af996764237118f5547a1cbe Mon Sep 17 00:00:00 2001 From: "jesse.johnson" Date: Fri, 10 Jul 2026 07:38:43 -0700 Subject: [PATCH 1/2] buildkit caching on ecr --- .github/workflows/README.md | 17 +++++++ .github/workflows/build-ot3-actions.yml | 67 +++++++++++++++++++++---- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c1ce15b0..babf61d8 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -25,3 +25,20 @@ Builds are triggered: - Cache is stored on the S3 bucket as a big zip (since OE git cache needs to store empty directories and S3 doesn't do that on its own). We fetch cache before every build, and update `LOCAL_CACHE` with it. If the build succeeds, we update the zip and write it back. - Build results get sent to an artifact bucket identified by `S3_ARTIFACT_ARN`. - We have to be a little more careful with removing working directories here than in normal github actions. + +## Docker image BuildKit cache (ECR) + +The `run-build` job builds the oe-core `Dockerfile` on each ephemeral runner. When configured, BuildKit stores reusable image layers in ECR so repeated builds skip the expensive `apt-get` stack when `Dockerfile` is unchanged. + +Set these repository variables (per AWS account / infra stage): + +| Variable | Example value | +| --- | --- | +| `OT3_OE_DOCKER_ECR_REPOSITORY_PROD` | `123456789.dkr.ecr.us-east-2.amazonaws.com/ot3-oe-ci-image` | +| `OT3_OE_DOCKER_ECR_REPOSITORY_DEV` | `123456789.dkr.ecr.us-east-2.amazonaws.com/ot3-oe-ci-image` | + +Cache tags look like `buildcache-` on that repository. The workflow creates the ECR repository if it does not exist. + +The `ROBOT_STACK_AWS_OIDC_ROLE_ARN_*` role used for the build needs ECR permissions: `ecr:GetAuthorizationToken`, `ecr:CreateRepository`, `ecr:DescribeRepositories`, `ecr:BatchCheckLayerAvailability`, `ecr:GetDownloadUrlForLayer`, `ecr:BatchGetImage`, `ecr:PutImage`, `ecr:InitiateLayerUpload`, `ecr:UploadLayerPart`, `ecr:CompleteLayerUpload`. + +If the variables are unset, CI falls back to a plain `docker build` with no layer cache. diff --git a/.github/workflows/build-ot3-actions.yml b/.github/workflows/build-ot3-actions.yml index 3ffeb15f..ea28e754 100644 --- a/.github/workflows/build-ot3-actions.yml +++ b/.github/workflows/build-ot3-actions.yml @@ -125,6 +125,8 @@ jobs: # Set these repo variables (Settings → Actions → Variables): # - RUNNER_API_URL_PROD # - RUNNER_API_URL_DEV + # - OT3_OE_DOCKER_ECR_REPOSITORY_PROD (e.g. 123456789.dkr.ecr.us-east-2.amazonaws.com/ot3-oe-ci-image) + # - OT3_OE_DOCKER_ECR_REPOSITORY_DEV API_URL: ${{ inputs.infra-stage == 'stage-dev' && vars.RUNNER_API_URL_DEV || vars.RUNNER_API_URL_PROD }} INFRA_STAGE: ${{ inputs.infra-stage }} VARIANT: ${{ needs.decide-refs.outputs.variant }} @@ -333,6 +335,25 @@ jobs: esac echo "S3_ARTIFACT_ARN=${bucket}" >> "$GITHUB_ENV" echo "Uploading artifacts to ${bucket/arn:aws:s3:::/}" + - name: Set ECR repository for Docker build cache + shell: bash + run: | + if [[ "${{ matrix.build_env }}" == "stage-dev" ]]; then + ecr_repo="${{ vars.OT3_OE_DOCKER_ECR_REPOSITORY_DEV }}" + else + ecr_repo="${{ vars.OT3_OE_DOCKER_ECR_REPOSITORY_PROD }}" + fi + if [[ -z "$ecr_repo" ]]; then + echo "OT3_OE_DOCKER_ECR_REPOSITORY_* not set for ${{ matrix.build_env }}; Docker build will run without ECR layer cache." + else + echo "Using ECR repository ${ecr_repo} for BuildKit layer cache" + fi + echo "ECR_REPOSITORY=${ecr_repo}" >> "$GITHUB_ENV" + - name: Login to Amazon ECR + if: ${{ env.ECR_REPOSITORY != '' }} + uses: aws-actions/amazon-ecr-login@4625ce35226a7557230889aae2f52eb50ec3dcda # v2.0.1 + with: + mask-password: false - name: Apply CI runner customizations run: | # Ephemeral runners may not permit writing /etc/sysctl*; apply runtime settings only. @@ -349,18 +370,46 @@ jobs: sudo mount -o remount,exec /tmp 2>/dev/null || true sudo mount | grep tmp || true - name: Build container + shell: bash run: | + set -euo pipefail cd oe-core tmp_dir=$(mktemp -d -t ci-XXXXXXX) - cp start.sh $tmp_dir/ - # Must match host runner uid/gid so bind-mounted oe-core/cache are writable (see Dockerfile). - # Defaults in Dockerfile are 1001:1001; ec2-user is often 1000 — without this, cleanup/gather hit Permission denied. - docker build -f ./Dockerfile \ - --build-arg "username=$(whoami)" \ - --build-arg "host_uid=$(id -u)" \ - --build-arg "host_gid=$(id -g)" \ - --tag "ot3-image:latest" \ - $tmp_dir + cp start.sh "$tmp_dir/" + dockerfile_cache_key=$(sha256sum Dockerfile | awk '{print substr($1, 1, 12)}') + build_args=( + -f ./Dockerfile + --build-arg "username=$(whoami)" + --build-arg "host_uid=$(id -u)" + --build-arg "host_gid=$(id -g)" + --tag "ot3-image:latest" + ) + + if [[ -n "${ECR_REPOSITORY:-}" ]]; then + repo_name="${ECR_REPOSITORY#*/}" + aws ecr describe-repositories --repository-names "$repo_name" >/dev/null 2>&1 || \ + aws ecr create-repository \ + --repository-name "$repo_name" \ + --image-tag-mutability MUTABLE \ + --encryption-configuration encryptionType=AES256 + + cache_ref="${ECR_REPOSITORY}:buildcache-${dockerfile_cache_key}" + echo "Building with BuildKit ECR layer cache: ${cache_ref}" + + docker buildx rm oe-ci-builder >/dev/null 2>&1 || true + docker buildx create --name oe-ci-builder --driver docker-container --use --bootstrap + + docker buildx build \ + "${build_args[@]}" \ + --cache-from "type=registry,ref=${cache_ref}" \ + --cache-to "type=registry,ref=${cache_ref},mode=max" \ + --load \ + "$tmp_dir" + else + echo "Building without ECR layer cache" + # Must match host runner uid/gid so bind-mounted oe-core/cache are writable (see Dockerfile). + docker build "${build_args[@]}" "$tmp_dir" + fi cd .. - name: Apply unconditional CI config overrides run: | From d2b931e1a7056a19fe052f736641dbfb343888a9 Mon Sep 17 00:00:00 2001 From: "jesse.johnson" Date: Sat, 11 Jul 2026 11:43:03 -0700 Subject: [PATCH 2/2] fix(ci): pin amazon-ecr-login to commit SHA Use the peeled v2.0.1 commit instead of the annotated tag object so zizmor impostor-commit and ref-version-mismatch checks pass. Co-authored-by: Cursor --- .github/workflows/build-ot3-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ot3-actions.yml b/.github/workflows/build-ot3-actions.yml index ea28e754..d87cd6c0 100644 --- a/.github/workflows/build-ot3-actions.yml +++ b/.github/workflows/build-ot3-actions.yml @@ -351,7 +351,7 @@ jobs: echo "ECR_REPOSITORY=${ecr_repo}" >> "$GITHUB_ENV" - name: Login to Amazon ECR if: ${{ env.ECR_REPOSITORY != '' }} - uses: aws-actions/amazon-ecr-login@4625ce35226a7557230889aae2f52eb50ec3dcda # v2.0.1 + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 with: mask-password: false - name: Apply CI runner customizations