From 461f485f4c29b88b2eea9bf18855ee3c0372ebe8 Mon Sep 17 00:00:00 2001 From: Pete Crocker Date: Thu, 20 Aug 2026 08:19:11 +0000 Subject: [PATCH] fix(ci): retry cosign transparency-log writes and upload SBOMs first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1.10.7 release failed in publish-docker-image / sbom (run 31521732131) when `cosign attest` could not write to the Sigstore transparency log: Post "https://rekor.sigstore.dev/api/v1/log/entries": giving up after 2 attempt(s) Rekor was only briefly unavailable — the sign job had succeeded against the same infrastructure five minutes earlier, and the identical sbom job passed on the next nightly run with no code change. But cosign retries a failed log write only twice internally and exposes no knob to raise that, so a blip that short was enough to fail the release. The fallout was disproportionate: the SBOM upload step never ran, so v1.10.7 is the only release since v1.10.0 with no SBOM artifacts, and repository-dispatch (which needs publish-docker-image) was skipped, so downstream repos were never notified of the release. Three changes: - Wrap cosign sign and both cosign attest calls in a retry() helper: five attempts 60 seconds apart, covering a ~4-minute outage. The helper takes the command as arguments ("$@", no string/eval layer) and is pasted verbatim into the sign and sbom jobs — they run on separate runners, so it cannot be shared without a checkout or a third-party action in the signing path. - Upload the SBOM artifacts before attesting them, so a transparency-log outage can no longer cost us the SBOMs themselves. Attestation failures still fail the job, which is the correct outcome for a supply-chain step. - Set overwrite: true on the SBOM upload. Artifacts are scoped to the run rather than the attempt, so with the upload now preceding a step that can fail, a re-run would otherwise die at the upload step because the artifact name already exists from the earlier attempt. Every caller passes a version unique to its invocation, so the only artifact this can replace is the same SBOM from a previous attempt of the same run. Verified by extracting both run scripts from the parsed workflow and executing them against a stubbed cosign: immediate success (1 call, no sleeps), recovery after 3 failures (exit 0, 3 sleeps), budget exhaustion (5 calls, 4 sleeps, exit 1), and a terminally failing first attest stops the script before the second attest runs. The two retry() bodies are asserted byte-identical, and actionlint passes. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci-docker-image.yml | 86 ++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-docker-image.yml b/.github/workflows/ci-docker-image.yml index 7d07688feb1..6d36a94e809 100644 --- a/.github/workflows/ci-docker-image.yml +++ b/.github/workflows/ci-docker-image.yml @@ -202,9 +202,33 @@ jobs: password: ${{ secrets.HARBOR_PASSWORD }} - name: Sign manifest + env: + IMAGE: ${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }} run: | - cosign sign --yes --recursive --new-bundle-format=false --use-signing-config=false \ - "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" + # cosign gives up after two internal attempts when a Sigstore transparency + # log write fails, so a brief rekor.sigstore.dev blip is enough to fail a + # release. Retry the whole command instead; five attempts 60s apart cover a + # ~4-minute outage. The sbom job carries a verbatim copy of this function: + # the jobs run on separate runners, so sharing it would take a checkout or a + # third-party action in the signing path. Keep the two copies identical. + retry() { + local attempt + for attempt in 1 2 3 4 5; do + if "$@"; then + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + echo "::warning::${1} ${2} failed (attempt ${attempt}/5), retrying in 60s" + sleep 60 + fi + done + echo "::error::${1} ${2} failed after 5 attempts" + return 1 + } + + retry cosign sign --yes --recursive \ + --new-bundle-format=false --use-signing-config=false \ + "${IMAGE}" sbom: needs: merge @@ -236,20 +260,16 @@ jobs: "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" \ --output cyclonedx-json=infrahub-sbom.cdx.json - - name: Attest SBOM (SPDX) - run: | - cosign attest --yes \ - --type spdxjson \ - --predicate infrahub-sbom.spdx.json \ - "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" - - - name: Attest SBOM (CycloneDX) - run: | - cosign attest --yes \ - --type cyclonedx \ - --predicate infrahub-sbom.cdx.json \ - "${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }}" - + # Uploaded before the attestations so that a transparency-log outage cannot cost + # us the SBOMs themselves: v1.10.7 shipped without any because the attest step + # failed first and skipped this one. + # + # overwrite is required precisely because this now runs before a step that can + # fail. Artifacts are scoped to the run rather than the attempt, so on a re-run + # this name already exists from the earlier attempt and the default overwrite: + # false would fail the upload, breaking the recovery path this ordering exists to + # protect. Every caller passes a version unique to its invocation, so the only + # artifact this can replace is the same SBOM from a previous attempt. - name: Upload SBOM artifacts uses: actions/upload-artifact@v7 with: @@ -258,3 +278,37 @@ jobs: infrahub-sbom.spdx.json infrahub-sbom.cdx.json retention-days: 90 + overwrite: true + + - name: Attest SBOMs + env: + IMAGE: ${{ vars.HARBOR_HOST }}/${{ github.repository }}@${{ needs.merge.outputs.digest }} + run: | + # Same transparency-log flakiness the sign job guards against; this is a + # verbatim copy of that job's retry function (separate runners, so it cannot + # be shared without a checkout or a third-party action in the signing path). + # Keep the two copies identical. + retry() { + local attempt + for attempt in 1 2 3 4 5; do + if "$@"; then + return 0 + fi + if [ "${attempt}" -lt 5 ]; then + echo "::warning::${1} ${2} failed (attempt ${attempt}/5), retrying in 60s" + sleep 60 + fi + done + echo "::error::${1} ${2} failed after 5 attempts" + return 1 + } + + retry cosign attest --yes \ + --type spdxjson \ + --predicate infrahub-sbom.spdx.json \ + "${IMAGE}" + + retry cosign attest --yes \ + --type cyclonedx \ + --predicate infrahub-sbom.cdx.json \ + "${IMAGE}"