diff --git a/.github/scripts/prepare-release_test.sh b/.github/scripts/prepare-release_test.sh index fb2195840d..fb60c0df81 100644 --- a/.github/scripts/prepare-release_test.sh +++ b/.github/scripts/prepare-release_test.sh @@ -369,6 +369,8 @@ test_subsequent_rc_rejects_historical_form() { test_final() { setup_repo "v0.60.0-rc.3" + cp "${SCRIPT_DIR}/../../docs/release-notes/template.md" \ + "${REPO}/docs/release-notes/template.md" make_release_branch 0.60 git -C "${REPO}" tag v0.60.0-rc.3 run_prepare final 0.60 @@ -378,6 +380,8 @@ test_final() { assert_file_contains "${REPO}/out/requires-backport.txt" 'true' || return assert_file_contains "${REPO}/docs/release-notes/v0.60.0.md" \ '## Upgrading to Radius v0.60.0' || return + assert_file_contains "${REPO}/docs/release-notes/v0.60.0.md" \ + 'Restarting pods no longer picks up later patches implicitly' || return ((++PASS)) } @@ -466,11 +470,15 @@ test_version_only_does_not_mutate_files() { test_patch() { setup_repo "v0.60.2" + cp "${SCRIPT_DIR}/../../docs/release-notes/template_patch.md" \ + "${REPO}/docs/release-notes/template_patch.md" make_release_branch 0.60 git -C "${REPO}" tag v0.60.2 git -C "${REPO}" tag v0.61.0 run_prepare patch 0.60 assert_version "v0.60.3" || return + assert_file_contains "${REPO}/docs/release-notes/v0.60.3.md" \ + 'Restarting pods no longer picks up later patches implicitly' || return assert_file_contains "${REPO}/CHANGELOG.md" \ 'compare/v0.60.2...v0.60.3' || return ((++PASS)) diff --git a/.github/scripts/release-cutover_test.sh b/.github/scripts/release-cutover_test.sh index 910f86056b..9fd9004ecc 100644 --- a/.github/scripts/release-cutover_test.sh +++ b/.github/scripts/release-cutover_test.sh @@ -25,6 +25,7 @@ readonly REPO_ROOT readonly RELEASE_WORKFLOW="${REPO_ROOT}/.github/workflows/build-release.yaml" readonly CLI_WORKFLOW="${REPO_ROOT}/.github/workflows/__build-cli.yaml" readonly IMAGE_WORKFLOW="${REPO_ROOT}/.github/workflows/__build-images.yaml" +readonly HELM_WORKFLOW="${REPO_ROOT}/.github/workflows/__build-helm-chart.yaml" readonly CONFIG="${REPO_ROOT}/.goreleaser.yaml" readonly ARTIFACTS_MAKEFILE="${REPO_ROOT}/build/artifacts.mk" PASS=0 @@ -106,6 +107,21 @@ test_publication_gate() { fail_test "failed verification or missing approval can reach publication" return fi + if ! yq -o=json '.jobs."build-and-push-helm-chart".steps' \ + "${HELM_WORKFLOW}" | jq -e ' + (map(.name) | index("Pin external chart images") as $pin | + $pin != null and $pin < index("Package Helm chart") and + $pin < index("Push helm chart to GHCR")) and + any(.[]; .name == "Pin external chart images" and + (.if | contains("refs/tags/v")) and + (.run | contains("--names dashboard")) and + (.run | contains("--names deployment-engine")) and + (.run | contains("--expected-digest")) and + (.run | contains("--source-sha"))) + ' > /dev/null; then + fail_test "chart publication can precede verified external version tags" + return + fi ((++PASS)) } diff --git a/.github/scripts/release-oci-artifacts.sh b/.github/scripts/release-oci-artifacts.sh index 8c5139d655..200bc45b50 100644 --- a/.github/scripts/release-oci-artifacts.sh +++ b/.github/scripts/release-oci-artifacts.sh @@ -44,9 +44,14 @@ VERIFY_ALIASES=false VERIFY_SBOMS=false PROMOTE_LATEST="${RELEASE_PROMOTE_LATEST:-true}" SOURCE_SHA="${RELEASE_SOURCE_SHA:-}" +EXPECTED_DIGEST="" TEMP_DIR="" readonly RETRY_ATTEMPTS="${RELEASE_RETRY_ATTEMPTS:-5}" readonly RETRY_MAX_DELAY_SECONDS="${RELEASE_RETRY_MAX_DELAY_SECONDS:-15}" +# External images are published by their own repositories from the sibling +# tags the controller creates, in parallel with the Radius tag build. +readonly EXTERNAL_IMAGE_WAIT_SECONDS="${RELEASE_EXTERNAL_IMAGE_WAIT_SECONDS:-600}" +readonly EXTERNAL_IMAGE_POLL_SECONDS="${RELEASE_EXTERNAL_IMAGE_POLL_SECONDS:-30}" readonly SOURCE_ANNOTATION="org.opencontainers.image.source=" readonly SOURCE_URL="https://github.com/radius-project/radius" @@ -141,6 +146,9 @@ retry_read() { usage() { cat >&2 << 'EOF' Usage: + release-oci-artifacts.sh pin-image --registry --names \ + --version --channel --source-sha \ + [--expected-digest ] release-oci-artifacts.sh stage-cli --registry \ --version --artifacts --output release-oci-artifacts.sh stage-cli --registry \ @@ -211,6 +219,10 @@ parse_args() { SOURCE_SHA="${2:-}" shift 2 ;; + --expected-digest) + EXPECTED_DIGEST="${2:-}" + shift 2 + ;; --aliases) VERIFY_ALIASES=true shift @@ -608,6 +620,120 @@ image_reference_state() { done } +external_image_matches() { + local raw="$1" + local expected_platforms="$2" + + jq -e --arg source "${SOURCE_SHA}" \ + --argjson platforms "${expected_platforms}" ' + def platform_name: + .os + "/" + .architecture + + (if (.variant // "") == "" then "" else "/" + .variant end); + ([if .manifest.manifests then + .manifest.manifests[] | select(.platform.os != "unknown") | .platform + else .image end | platform_name] | sort) == $platforms and + ([if .manifest.manifests then + .image | to_entries[] | select(.key != "unknown/unknown") | .value + else .image end | .config.Labels."org.opencontainers.image.revision"] | + unique) == [$source] + ' <<< "${raw}" > /dev/null +} + +# Prints the inspection of an external image reference once it exists and +# carries the planned source and platform set. The publisher that produces the +# reference runs in parallel with this build, so a missing reference or one +# still serving an earlier source is awaited for a bounded time. An existing +# immutable version tag is never awaited: a mismatch there is a conflict. +await_external_image() { + local reference="$1" + local expected_platforms="$2" + local immutable="$3" + local deadline=$((SECONDS + EXTERNAL_IMAGE_WAIT_SECONDS)) + local raw problem + + while true; do + if [[ "$(image_reference_state "${reference}")" == "absent" ]]; then + problem="external image is not published: ${reference}" + else + raw="$(retry_read "external image inspection" \ + docker buildx imagetools inspect --format '{{json .}}' \ + "${reference}")" + if external_image_matches "${raw}" "${expected_platforms}"; then + printf '%s\n' "${raw}" + return + fi + problem="external image source or platforms differ from the plan: ${reference}" + fi + if [[ "${immutable}" == "true" ]] || ((SECONDS >= deadline)); then + fail "${problem}" + fi + echo "Waiting for ${reference} to carry the planned source; its publisher may still be running." >&2 + if [[ "${RELEASE_RETRY_NO_SLEEP:-}" != "true" ]]; then + sleep "${EXTERNAL_IMAGE_POLL_SECONDS}" + fi + done +} + +pin_image() { + local repository reference target raw digest expected_platforms state + local attempt output status immutable + + require_command docker + require_command oras + require_command jq + validate_version + validate_source_sha + [[ -n "${REGISTRY}" ]] || fail "registry is required" + [[ "${CHANNEL}" =~ ^[0-9]+\.[0-9]+$ ]] || + is_radius_release_version "${CHANNEL}" || fail "invalid published tag" + [[ -z "${EXPECTED_DIGEST}" || + "${EXPECTED_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] || + fail "invalid expected digest" + expected_platforms="$(jq -ce --arg name "${NAMES}" ' + [.images[] | select(.name == $name and .radiusBuild == false)] | + if length == 1 then .[0].requiredPlatforms | sort + else error("select exactly one external image") end + ' "${TARGETS_FILE}")" + repository="${REGISTRY%/}/${NAMES}" + target="${repository}:${VERSION}" + reference="${repository}:${CHANNEL}" + state="$(image_reference_state "${target}")" + immutable=false + if [[ "${state}" == "exists" ]]; then + reference="${target}" + immutable=true + fi + raw="$(await_external_image "${reference}" "${expected_platforms}" \ + "${immutable}")" + digest="$(jq -er '.manifest.digest | + select(test("^sha256:[0-9a-f]{64}$"))' <<< "${raw}")" + [[ -z "${EXPECTED_DIGEST}" || "${digest}" == "${EXPECTED_DIGEST}" ]] || + fail "external image differs from its locked digest: ${reference}" + [[ "${state}" == "exists" ]] && return + + for ((attempt = 1; attempt <= RETRY_ATTEMPTS; attempt++)); do + if [[ "$(image_reference_state "${target}")" == "exists" ]]; then + verify_image_alias "${target}" "${digest}" + return + fi + if output="$(oras tag "${repository}@${digest}" "${VERSION}" 2>&1)"; then + verify_image_alias "${target}" "${digest}" + return + else + status=$? + fi + if [[ "$(image_reference_state "${target}")" == "exists" ]]; then + verify_image_alias "${target}" "${digest}" + return + fi + if ((attempt == RETRY_ATTEMPTS)) || ! is_retryable_error "${output}"; then + echo "${output}" >&2 + return "${status}" + fi + wait_before_retry "immutable external image tag" "${attempt}" + done +} + assert_images_absent() { local name local reference @@ -1045,6 +1171,7 @@ promote_aliases() { main() { parse_args "$@" case "${COMMAND}" in + pin-image) pin_image ;; stage-cli) stage_cli ;; promote) promote_aliases ;; verify) verify_locks ;; diff --git a/.github/scripts/release-oci-artifacts_test.sh b/.github/scripts/release-oci-artifacts_test.sh index cc4f36339b..7a9d921916 100644 --- a/.github/scripts/release-oci-artifacts_test.sh +++ b/.github/scripts/release-oci-artifacts_test.sh @@ -270,7 +270,17 @@ JSON printf '}\n' exit 0 fi - printf '{"manifest":{"digest":"%s"}}\n' "${digest}" + if [[ -n "${FAKE_IMAGE_INSPECTION_STALE:-}" ]] && + (( $(grep -c -F -- "inspect --format {{json .}} ${reference}" "${calls}") <= + ${FAKE_IMAGE_INSPECTION_STALE_CALLS:-0} )); then + jq --arg digest "${digest}" '.manifest.digest = $digest' "${FAKE_IMAGE_INSPECTION_STALE}" + exit 0 + fi + if [[ -n "${FAKE_IMAGE_INSPECTION:-}" ]]; then + jq --arg digest "${digest}" '.manifest.digest = $digest' "${FAKE_IMAGE_INSPECTION}" + else + printf '{"manifest":{"digest":"%s"}}\n' "${digest}" + fi exit 0 fi @@ -284,6 +294,7 @@ run_script() { FAKE_REGISTRY_STATE="${TEST_ROOT}/registry-state" \ FAKE_REGISTRY_CALLS="${TEST_ROOT}/calls" \ RELEASE_RETRY_NO_SLEEP=true \ + RELEASE_EXTERNAL_IMAGE_WAIT_SECONDS="${RELEASE_EXTERNAL_IMAGE_WAIT_SECONDS:-0}" \ RELEASE_SOURCE_SHA="${SOURCE_SHA}" \ GORELEASER_PARITY_TARGETS="${TEST_ROOT}/targets.json" \ bash "${SCRIPT}" "$@" @@ -361,6 +372,7 @@ test_real_oras_preserves_basename() { local layout local pull_dir local release_dir + local digest setup_fixture layout="${TEST_ROOT}/layout" @@ -398,6 +410,16 @@ EOF fail_test "real ORAS push did not preserve the CLI basename" return fi + digest="$("${real_oras}" resolve --oci-layout "${layout}/rad/linux-amd64:0.61.0")" + "${real_oras}" tag --oci-layout "${layout}/rad/linux-amd64@${digest}" 0.61.1 > /dev/null + [[ "$("${real_oras}" resolve --oci-layout "${layout}/rad/linux-amd64:0.61.1")" == "${digest}" ]] || { + fail_test "native ORAS tagging changed the content digest" + return + } + [[ "$("${real_oras}" resolve --oci-layout "${layout}/rad/linux-amd64:0.61.0")" == "${digest}" ]] || { + fail_test "native ORAS tagging changed the source reference" + return + } ((++PASS)) } @@ -706,6 +728,143 @@ test_image_preflight_fails_closed_on_lookup_errors() { ((++PASS)) } +test_pins_external_images_without_rebuilding_or_moving_aliases() { + local digest source_reference target_reference + setup_fixture + digest="sha256:$(digest_for dashboard)" + source_reference=example.test/radius/dashboard:0.61 + target_reference=example.test/radius/dashboard:0.61.2 + jq '.images += [{name:"dashboard",radiusBuild:false,category:"downstream", + requiredPlatforms:["linux/amd64"]}]' "${TEST_ROOT}/targets.json" \ + > "${TEST_ROOT}/external-targets.json" + mv "${TEST_ROOT}/external-targets.json" "${TEST_ROOT}/targets.json" + jq -n --arg source "${SOURCE_SHA}" '{manifest:{digest:""},image:{ + os:"linux",architecture:"amd64",config:{Labels:{ + "org.opencontainers.image.revision":$source}}}}' \ + > "${TEST_ROOT}/inspection.json" + printf '%s\t%s\n' "${source_reference}" "${digest}" >> "${TEST_ROOT}/registry-state" + FAKE_IMAGE_INSPECTION="${TEST_ROOT}/inspection.json" FAKE_FAIL_TAG_ONCE=true \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.2 --channel 0.61 --expected-digest "${digest}" + [[ "$(awk -F '\t' -v ref="${target_reference}" '$1 == ref {print $2}' \ + "${TEST_ROOT}/registry-state")" == "${digest}" ]] || { + fail_test "external version tag does not preserve the manifest digest" + return + } + : > "${TEST_ROOT}/calls" + FAKE_IMAGE_INSPECTION="${TEST_ROOT}/inspection.json" \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.2 --channel 0.61 --expected-digest "${digest}" + if grep -Eq '^oras tag|imagetools create|docker build ' "${TEST_ROOT}/calls"; then + fail_test "retry rebuilt or retagged an existing immutable image" + return + fi + if FAKE_IMAGE_INSPECTION="${TEST_ROOT}/inspection.json" \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.2 --channel 0.61 --expected-digest "sha256:$(digest_for conflict)" \ + > /dev/null 2>&1; then + fail_test "pinning accepted a conflicting locked digest" + return + fi + if FAKE_IMAGE_INSPECTION="${TEST_ROOT}/inspection.json" \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.3 --channel 0.61 --source-sha bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb \ + > /dev/null 2>&1; then + fail_test "pinning accepted a source mismatch" + return + fi + if FAKE_IMAGE_LOOKUP_ERROR=credentials \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.3 --channel 0.61 > /dev/null 2>&1; then + fail_test "pinning accepted a registry lookup failure" + return + fi + jq 'del(.image.config.Labels)' "${TEST_ROOT}/inspection.json" \ + > "${TEST_ROOT}/unlabeled.json" + if FAKE_IMAGE_INSPECTION="${TEST_ROOT}/unlabeled.json" \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.3 --channel 0.61 > /dev/null 2>&1; then + fail_test "pinning accepted an external image without source evidence" + return + fi + jq '{manifest:{manifests:[{platform:{os:"linux",architecture:"amd64"}}]}, + image:{"linux/amd64":.image}}' "${TEST_ROOT}/inspection.json" \ + > "${TEST_ROOT}/index.json" + FAKE_IMAGE_INSPECTION="${TEST_ROOT}/index.json" \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.3 --channel 0.61 --expected-digest "${digest}" + jq '.manifest.manifests[0].platform.architecture = "arm64"' \ + "${TEST_ROOT}/index.json" > "${TEST_ROOT}/wrong-platform.json" + if FAKE_IMAGE_INSPECTION="${TEST_ROOT}/wrong-platform.json" \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.4 --channel 0.61 > /dev/null 2>&1; then + fail_test "pinning accepted an unexpected platform set" + return + fi + [[ "$(awk -F '\t' -v ref="${source_reference}" '$1 == ref {print $2}' \ + "${TEST_ROOT}/registry-state")" == "${digest}" ]] || { + fail_test "pinning modified the original channel alias" + return + } + ((++PASS)) +} + +test_waits_for_external_image_publisher() { + local digest + setup_fixture + digest="sha256:$(digest_for dashboard)" + jq '.images += [{name:"dashboard",radiusBuild:false,category:"downstream", + requiredPlatforms:["linux/amd64"]}]' "${TEST_ROOT}/targets.json" \ + > "${TEST_ROOT}/external-targets.json" + mv "${TEST_ROOT}/external-targets.json" "${TEST_ROOT}/targets.json" + jq -n --arg source "${SOURCE_SHA}" '{manifest:{digest:""},image:{ + os:"linux",architecture:"amd64",config:{Labels:{ + "org.opencontainers.image.revision":$source}}}}' \ + > "${TEST_ROOT}/inspection.json" + jq '.image.config.Labels."org.opencontainers.image.revision" = + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"' \ + "${TEST_ROOT}/inspection.json" > "${TEST_ROOT}/stale.json" + printf '%s\t%s\n' example.test/radius/dashboard:0.61 "${digest}" \ + >> "${TEST_ROOT}/registry-state" + # The channel reference still serves the previous build for the first + # inspections; the publisher finishes while the pin waits. + if ! FAKE_IMAGE_INSPECTION="${TEST_ROOT}/inspection.json" \ + FAKE_IMAGE_INSPECTION_STALE="${TEST_ROOT}/stale.json" \ + FAKE_IMAGE_INSPECTION_STALE_CALLS=4 \ + RELEASE_EXTERNAL_IMAGE_WAIT_SECONDS=60 \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.2 --channel 0.61 2> "${TEST_ROOT}/wait.log"; then + fail_test "pinning did not wait for the external publisher" + return + fi + if ! grep -q 'Waiting for example.test/radius/dashboard:0.61 ' \ + "${TEST_ROOT}/wait.log"; then + fail_test "the wait for the external publisher was not reported" + return + fi + [[ "$(awk -F '\t' -v ref=example.test/radius/dashboard:0.61.2 \ + '$1 == ref {print $2}' "${TEST_ROOT}/registry-state")" == "${digest}" ]] || { + fail_test "the awaited image was not pinned to its digest" + return + } + # An existing full-version tag that differs is a conflict, never awaited. + : > "${TEST_ROOT}/calls" + if FAKE_IMAGE_INSPECTION="${TEST_ROOT}/inspection.json" \ + FAKE_IMAGE_INSPECTION_STALE="${TEST_ROOT}/stale.json" \ + FAKE_IMAGE_INSPECTION_STALE_CALLS=99 \ + RELEASE_EXTERNAL_IMAGE_WAIT_SECONDS=60 \ + run_script pin-image --registry example.test/radius --names dashboard \ + --version 0.61.2 --channel 0.61 > /dev/null 2>&1; then + fail_test "an existing version tag with another source was accepted" + return + fi + if (($(grep -c 'imagetools inspect' "${TEST_ROOT}/calls") > 3)); then + fail_test "an immutable version tag was awaited instead of rejected" + return + fi + ((++PASS)) +} + main() { export RELEASE_SOURCE_SHA="${SOURCE_SHA}" test_stages_cli_artifacts @@ -723,6 +882,8 @@ main() { test_rejects_stale_cli_tag_without_overwriting test_requires_image_lock_before_reusing_version_tag test_image_preflight_fails_closed_on_lookup_errors + test_pins_external_images_without_rebuilding_or_moving_aliases + test_waits_for_external_image_publisher if ((FAIL > 0)); then echo "release OCI artifact tests failed: ${PASS} passed, ${FAIL} failed" diff --git a/.github/scripts/release-parity-manifest.sh b/.github/scripts/release-parity-manifest.sh index 7b530b8b58..7022f944ff 100644 --- a/.github/scripts/release-parity-manifest.sh +++ b/.github/scripts/release-parity-manifest.sh @@ -536,7 +536,7 @@ collect_images() { category="$(jq -r '.category' <<<"${target}")" radius_build="$(jq -r '.radiusBuild' <<<"${target}")" reference="${registry}/${name}:${channel}" - if [[ "${STAGED}" == "true" && "${radius_build}" == "true" ]]; then + if [[ "${STAGED}" == "true" ]]; then reference="${registry}/${name}:${VERSION}" fi raw_path="${TEMP_DIR}/image-${name}-raw.json" diff --git a/.github/scripts/release-parity-manifest_test.sh b/.github/scripts/release-parity-manifest_test.sh index 5b73ddcfae..ab83bf51ec 100644 --- a/.github/scripts/release-parity-manifest_test.sh +++ b/.github/scripts/release-parity-manifest_test.sh @@ -468,6 +468,14 @@ jq -e ' ' "${OUTPUT}" >/dev/null || fail "staged collection used mutable Radius images" [[ -f "${TEST_ROOT}/staged-assets/rad_linux_amd64" ]] || fail "staged collector did not retain the installation binary" +jq '.images[0].radiusBuild = false' "${TARGETS}" >"${FIXTURES}/external-targets.json" +mv "${FIXTURES}/external-targets.json" "${TARGETS}" +RELEASE_PARITY_STAGED=true \ + RELEASE_PARITY_ASSETS_DIR="${TEST_ROOT}/external-assets" run_collector +jq -e '.images[0].reference == "ghcr.io/radius-project/ucpd:0.60.0"' \ + "${OUTPUT}" >/dev/null || fail "staged external inspection used a channel tag" +jq '.images[0].radiusBuild = true' "${TARGETS}" >"${FIXTURES}/radius-targets.json" +mv "${FIXTURES}/radius-targets.json" "${TARGETS}" jq '.draft = false' "${FIXTURES}/release.json" >"${FIXTURES}/final.json" mv "${FIXTURES}/final.json" "${FIXTURES}/release.json" diff --git a/.github/scripts/release-verification.sh b/.github/scripts/release-verification.sh index 0ada6afe29..e97ff6755f 100755 --- a/.github/scripts/release-verification.sh +++ b/.github/scripts/release-verification.sh @@ -156,12 +156,6 @@ readonly DOWNLOAD_URL="${DOWNLOAD_BASE}/v${RELEASE_VERSION_NUMBER}/${RADIUS_CLI_ EXPECTED_CLI_VERSION=$RELEASE_VERSION_NUMBER EXPECTED_TAG_VERSION="$RELEASE_VERSION_NUMBER" -# if RELEASE_VERSION_NUMBER contains -rc, then it is a prerelease. -# In that case, we need to set expected tag version to the major.minor of the -# release version number -if [[ "$RELEASE_VERSION_NUMBER" != *"rc"* ]]; then - EXPECTED_TAG_VERSION=$(echo "$RELEASE_VERSION_NUMBER" | cut -d '.' -f 1,2) -fi echo "RELEASE_VERSION_NUMBER: ${RELEASE_VERSION_NUMBER}" echo "OS: ${OS}" diff --git a/.github/scripts/release-verification_test.sh b/.github/scripts/release-verification_test.sh index 316f8b69b6..26fa11cb03 100644 --- a/.github/scripts/release-verification_test.sh +++ b/.github/scripts/release-verification_test.sh @@ -27,6 +27,10 @@ echo "$*" >>"${TEST_ROOT}/helm-calls" EOF cat > "${TEST_ROOT}/bin/curl" << 'EOF' #!/bin/bash +if [[ "${FAKE_PUBLIC_INSTALL:-false}" == "true" ]]; then + cp "${TEST_ROOT}/rad" "${@: -1}" + exit +fi echo "Staged verification must not download a public binary" >&2 exit 1 EOF @@ -42,10 +46,10 @@ case "$*" in bicep-de) name=deployment-engine ;; ucp) name=ucpd ;; esac - jq -r --arg name "${name}" '.observed.images[] | select(.name == $name) | .reference + "@" + .digest' "${TEST_ROOT}/manifest.json" + jq -r --arg name "${name}" '.observed.images[] | select(.name == $name) | .reference + (if env.FAKE_PUBLIC_INSTALL == "true" then "" else "@" + .digest end)' "${TEST_ROOT}/manifest.json" ;; "get job pre-upgrade "*) - jq '{spec:{template:{spec:{containers:[{image:(.observed.images[] | select(.name == "pre-upgrade") | .reference + "@" + .digest)}]}}}}' "${TEST_ROOT}/manifest.json" + jq '{spec:{template:{spec:{containers:[{image:(.observed.images[] | select(.name == "pre-upgrade") | .reference + (if env.FAKE_PUBLIC_INSTALL == "true" then "" else "@" + .digest end))}]}}}}' "${TEST_ROOT}/manifest.json" ;; "get pods "*) jq '{items:[{spec:{initContainers:[{image:(.observed.images[] | select(.name == "bicep") | .reference + "@" + .digest)}]}}]}' "${TEST_ROOT}/manifest.json" @@ -98,6 +102,8 @@ jq -e '.checks.installation == "verified"' "${RELEASE_VERIFY_MANIFEST}" > /dev/n grep -Fq -- "--chart ${TEST_ROOT}/radius.tgz" "${TEST_ROOT}/rad-calls" [[ "$(grep -o '@sha256:' "${TEST_ROOT}/rad-calls" | wc -l)" == "8" ]] grep -Fq -- 'delete cluster --name radius-verification-' "${TEST_ROOT}/kind-calls" +FAKE_PUBLIC_INSTALL=true RELEASE_VERIFY_CLI="" RELEASE_VERIFY_MANIFEST="" \ + bash "${ROOT}/.github/scripts/release-verification.sh" 0.61.0 > /dev/null if FAIL_INSTALL=true bash "${ROOT}/.github/scripts/release-verification.sh" 0.61.0 > /dev/null 2>&1; then echo "A failed installation was accepted" >&2 exit 1 @@ -115,4 +121,4 @@ if bash "${ROOT}/.github/scripts/release-verification.sh" 0.61.0 > /dev/null 2>& echo "A changed staged binary was accepted" >&2 exit 1 fi -echo "Staged installation tests passed (5 tests)" +echo "Release installation tests passed (6 tests)" diff --git a/.github/scripts/verify-release-manifest.mjs b/.github/scripts/verify-release-manifest.mjs index d4388659d6..e7a1fc0e10 100644 --- a/.github/scripts/verify-release-manifest.mjs +++ b/.github/scripts/verify-release-manifest.mjs @@ -211,7 +211,7 @@ export function verifyReleaseManifest({ compare( "images", `${target.name} reference`, - `${targets.imageRegistry}/${target.name}:${target.radiusBuild ? version : artifactTag}`, + `${targets.imageRegistry}/${target.name}:${version}`, image?.reference ); if (target.radiusBuild) { @@ -250,7 +250,7 @@ export function verifyReleaseManifest({ `${target.name} chart reference`, true, observed.helm?.renderedImages?.includes( - `${targets.imageRegistry}/${target.name}:${artifactTag}` + `${targets.imageRegistry}/${target.name}:${version}` ) ); } diff --git a/.github/scripts/verify-release-manifest_test.mjs b/.github/scripts/verify-release-manifest_test.mjs index c9b463f596..791c7ccf09 100644 --- a/.github/scripts/verify-release-manifest_test.mjs +++ b/.github/scripts/verify-release-manifest_test.mjs @@ -79,7 +79,7 @@ function fixture(releaseType = "final") { }, images: targets.images.map((image) => ({ name: image.name, - reference: `${targets.imageRegistry}/${image.name}:${image.radiusBuild ? version : channel}`, + reference: `${targets.imageRegistry}/${image.name}:${version}`, digest, platforms: image.requiredPlatforms.map((platform) => ({ platform, @@ -91,7 +91,7 @@ function fixture(releaseType = "final") { descriptor: { digest }, renderedImages: targets.images .filter((image) => targets.helm.expectedImages.includes(image.name)) - .map((image) => `${targets.imageRegistry}/${image.name}:${channel}`) + .map((image) => `${targets.imageRegistry}/${image.name}:${version}`) }, downstream: { repositories: targets.siblingRepositories.map((repository) => ({ @@ -256,6 +256,23 @@ test("reports expected and observed mismatches without weakening other checks", } }); +test("stable chart gates reject channel tags while preserving non-chart channel contracts", () => { + for (const releaseType of ["final", "patch"]) { + const input = fixture(releaseType); + for (const reference of input.observed.helm.renderedImages) { + const changed = structuredClone(input); + changed.observed.helm.renderedImages = + changed.observed.helm.renderedImages.map((image) => + image === reference ? image.replace(/:[^:]+$/, ":0.61") : image + ); + const report = verifyReleaseManifest(changed); + assert.equal(report.checks.helm, "failed", reference); + assert.equal(report.checks.metadata, "verified"); + assert.equal(report.checks.external, "verified"); + } + } +}); + test("approval waits cannot replace verified digests or bypass installation", () => { const approved = verifyReleaseManifest(fixture()); assert.throws( diff --git a/.github/scripts/verify-release-publication.sh b/.github/scripts/verify-release-publication.sh index 93f300edc6..7d0dbb9672 100644 --- a/.github/scripts/verify-release-publication.sh +++ b/.github/scripts/verify-release-publication.sh @@ -48,7 +48,7 @@ gh api -H 'Accept: application/vnd.github.raw+json' \ "repos/radius-project/radius/contents/.github/release-state/deployment-engine.json?ref=automation/release-state-${VERSION}" \ > "${DIRECTORY}/controller-lock.json" bash "${ROOT}/.github/scripts/verify-deployment-engine-image.sh" \ - --tag "$(jq -er '.deploymentEngine.imageTag' "${DIRECTORY}/controller-lock.json")" \ + --tag "${VERSION}" \ --signed-tag "v${VERSION}" \ --source-commit "$(jq -er '.deploymentEngine.sourceCommit' "${DIRECTORY}/controller-lock.json")" \ --expected-digest "$(jq -er '.deploymentEngine.digest' "${DIRECTORY}/controller-lock.json")" \ diff --git a/.github/workflows/__build-helm-chart.yaml b/.github/workflows/__build-helm-chart.yaml index f902cc48b1..03414c033b 100644 --- a/.github/workflows/__build-helm-chart.yaml +++ b/.github/workflows/__build-helm-chart.yaml @@ -27,7 +27,7 @@ jobs: build-and-push-helm-chart: name: Helm chart build runs-on: ubuntu-24.04 - timeout-minutes: 5 + timeout-minutes: 30 # includes a bounded wait for sibling image publishers permissions: packages: write # Required for uploading the package contents: read # Required for actions/checkout @@ -57,6 +57,43 @@ jobs: - name: Run Helm linter run: helm lint "${HELM_CHARTS_DIR}" + - name: Install image pinning tools + if: startsWith(github.ref, 'refs/tags/v') + run: make install-jq install-yq install-oras + + - name: Setup Docker Buildx + if: startsWith(github.ref, 'refs/tags/v') + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + - name: Authenticate external image pinning + if: startsWith(github.ref, 'refs/tags/v') + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Pin external chart images + if: startsWith(github.ref, 'refs/tags/v') + env: + GH_TOKEN: ${{ github.token }} + run: | + plan=".github/release-plans/v${REL_VERSION}.yaml" + lock="${RUNNER_TEMP}/deployment-engine-lock.json" + gh api -H 'Accept: application/vnd.github.raw+json' \ + "repos/${GITHUB_REPOSITORY}/contents/.github/release-state/deployment-engine.json?ref=automation/release-state-${REL_VERSION}" >"${lock}" + jq -e --arg version "v${REL_VERSION}" --arg source "$(git rev-parse HEAD)" \ + '.version == $version and .releaseSourceCommit == $source and .deploymentEngine.signedTag == $version' "${lock}" + bash ./.github/scripts/release-oci-artifacts.sh pin-image \ + --registry ghcr.io/radius-project --names deployment-engine \ + --version "${REL_VERSION}" --channel "$(jq -er '.deploymentEngine.imageTag' "${lock}")" \ + --source-sha "$(jq -er '.deploymentEngine.sourceCommit' "${lock}")" \ + --expected-digest "$(jq -er '.deploymentEngine.digest' "${lock}")" + bash ./.github/scripts/release-oci-artifacts.sh pin-image \ + --registry ghcr.io/radius-project --names dashboard \ + --version "${REL_VERSION}" --channel "${REL_CHANNEL}" \ + --source-sha "$(yq -er '.siblingRepositories[] | select(.name == "dashboard") | .sourceCommit' "${plan}")" + - name: Package Helm chart run: | mkdir -p "${ARTIFACT_DIR}/${HELM_PACKAGE_DIR}" diff --git a/deploy/Chart/README.md b/deploy/Chart/README.md index 7e9475abf4..be3a0d7cbc 100644 --- a/deploy/Chart/README.md +++ b/deploy/Chart/README.md @@ -38,6 +38,12 @@ By default, Radius pulls container images from GitHub Container Registry (ghcr.i ### Custom Image Tag +**Default image tags are version-pinned.** Final and patch charts use the full `Chart.AppVersion` (for example, `0.61.0` or `0.61.2`), including the Deployment Engine, dashboard, Bicep, readiness and pre-upgrade images. RC charts continue to use their full RC version. Historical charts keep their original tag behavior. + +**Patch updates require a chart upgrade.** Restarting pods no longer picks up a later patch through the moving `major.minor` alias. Upgrade the Radius CLI and run `rad upgrade kubernetes`, or upgrade to the desired chart version with Helm. Channel aliases remain published for existing consumers. + +An upgrade adopts the new defaults when the previous installation used chart defaults. Explicit `global.imageTag`, component tags, and tagged or digest-pinned image paths remain respected. To remove a stored global channel override while preserving other user settings, use `rad upgrade kubernetes --set global.imageTag=`. Clear any component-specific tag or image overrides separately; `--reset-values` also discards unrelated stored settings. + You can specify a custom tag for all Radius images using the `global.imageTag` parameter. This is useful when you want to deploy a specific version across all components or use custom-built images. Main-branch Radius images use the mutable `edge` tag. The `latest` tag points to the most recent stable release. Use `edge` when you need builds from `main`. diff --git a/deploy/Chart/templates/_helpers.tpl b/deploy/Chart/templates/_helpers.tpl index 9382b6ed3a..df8e08e78a 100644 --- a/deploy/Chart/templates/_helpers.tpl +++ b/deploy/Chart/templates/_helpers.tpl @@ -1,12 +1,5 @@ -{{/* Parse version and extract major and manor version from Appversion for image tag. */}} -{{- define "radius.versiontag" }} -{{- $version := .Chart.AppVersion }} -{{- /* Tag version will be 'major.minor' unless version is edge, latest, or rc release */}} -{{- if and (ne $version "edge") (ne $version "latest") (not (contains "rc" $version)) }} - {{- $ver := split "." $version }} - {{- $version = printf "%s.%s" $ver._0 $ver._1 }} -{{- end -}} -{{- print $version }} +{{- define "radius.versiontag" -}} +{{- .Chart.AppVersion -}} {{- end -}} {{/* External images retain their independently published latest tag for edge charts. */}} diff --git a/deploy/Chart/tests/helpers_test.yaml b/deploy/Chart/tests/helpers_test.yaml index 45a1b8e1e2..3d453b72e0 100644 --- a/deploy/Chart/tests/helpers_test.yaml +++ b/deploy/Chart/tests/helpers_test.yaml @@ -249,6 +249,50 @@ tests: value: ghcr.io/radius-project/controller:edge template: controller/deployment.yaml + - it: should pin final images to the full chart appVersion + chart: + appVersion: 0.61.0 + version: 0.61.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/radius-project/controller:0.61.0 + template: controller/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/radius-project/applications-rp:0.61.0 + template: rp/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/radius-project/deployment-engine:0.61.0 + template: de/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/radius-project/dashboard:0.61.0 + template: dashboard/deployment.yaml + + - it: should pin patch images to the full chart appVersion + chart: + appVersion: 0.61.2 + version: 0.61.2 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/radius-project/controller:0.61.2 + template: controller/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/radius-project/ucpd:0.61.2 + template: ucp/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/radius-project/dynamic-rp:0.61.2 + template: dynamic-rp/deployment.yaml + - equal: + path: spec.template.spec.initContainers[0].image + value: ghcr.io/radius-project/bicep:0.61.2 + template: controller/deployment.yaml + - it: should work with custom registry and default tag set: global.imageRegistry: custom.registry.io diff --git a/docs/contributing/contributing-releases/README.md b/docs/contributing/contributing-releases/README.md index c0624ce293..f43974634b 100644 --- a/docs/contributing/contributing-releases/README.md +++ b/docs/contributing/contributing-releases/README.md @@ -22,6 +22,8 @@ Before starting a release, ensure you have: - **Required release checks configured**: The `Validate release plan` check is required for generated release pull requests to `main`. The `release/*` ruleset requires `Validate release branch commits` with **Require branches to be up to date before merging** enabled; this makes the backport's recorded base SHA fail closed if the release branch advances. Backport pull requests use rebase merge; ordinary `main` pull requests continue to use squash merge. - **Publisher App access to Deployment Engine**: Prepare Release verifies the signed Deployment Engine tag with the publisher App (`RADIUS_PUBLISHER_BOT`). Its installation on `azure-octo` must include `deployment-engine` with Contents read, because neither `GITHUB_TOKEN` nor the release App can read that private repository. - **Publication approval configured**: The `release` environment has required reviewers and allows release tags. Final and patch publication waits there after verification; RCs need no environment approval. +- **External image package access configured**: Grant the `radius` repository Actions write access to the GHCR `dashboard` and `deployment-engine` packages. Helm publication adds full-version tags to their source-verified digests using `GITHUB_TOKEN` with `packages: write`; no images are rebuilt and existing channel aliases are retained. A conflicting digest, source commit, platform set, or registry lookup stops chart publication. Because the dashboard publisher builds from its sibling tag in parallel with the Radius tag build, Helm publication waits up to ten minutes for the channel image to carry the planned source before treating the mismatch as a conflict; an existing full-version tag is never awaited. +- **Dashboard source provenance available**: The dashboard publisher must stamp `org.opencontainers.image.revision` with the exact built commit SHA. Its `Build Image` step (`yarn run build-image`) and its Dockerfile set no OCI labels today, so the image build must add the label from the built commit before the release build's verification gate can pass; the gate has required it since the release manifest was introduced, not only since the chart cutover. Both publication verification and external pinning reject images without this evidence. - **Coordination identity configured**: The `RADIUS_RELEASE_BOT` installation must include `docs` and `samples` with at least Actions write, Contents read, Pull requests read, and Metadata read; their own release and upmerge workflows already require Contents and Pull requests write. The coordination job requests only those repositories and only those permissions: Actions write dispatches and reruns workflows, Contents read verifies the published channel branches, and Pull requests read verifies that each generated upmerge pull request merged. The Radius `GITHUB_TOKEN` records same-repository deployment receipts with `deployments: write`; every receipt lives in the `release-coordination` environment, which must stay free of protection rules. - **Release notifications configured**: Set the repository secret `RELEASE_TEAMS_WEBHOOK` to an HTTPS Teams incoming webhook that accepts Adaptive Cards. Stage summaries remain available in GitHub when the webhook is missing or unavailable; a notification failure cannot weaken the publication gate. @@ -87,7 +89,9 @@ Seven GitHub Actions workflows drive release preparation, validation, reconcilia Finalization is serialized across release versions. A patch to an older supported channel updates that channel without replacing global `latest`; an older version finishing after a newer version in the same channel changes neither alias. Builds remain parallel, and alias promotion uses the recorded digests without rebuilding artifacts. - `release-manifest.json` is retained as verification evidence and attached to the draft before publication. It binds expected and observed outputs to the approved plan and source commit. A failed verification leaves the release draft and does not advance Radius aliases. The staged installation overrides do not change the chart's published image-tag defaults; the immutable-chart migration is a separate phase. + `release-manifest.json` is retained as verification evidence and attached to the draft before publication. It binds expected and observed outputs to the approved plan and source commit. A failed verification leaves the release draft and does not advance Radius aliases. The published chart defaults to full-version tags for every Radius image; the gate rejects stable charts that truncate those tags to a channel. Its installation check uses digest overrides for the same verified images. + + **Patch-pickup change:** Pods installed with chart defaults no longer pick up later patches on restart. A chart upgrade is required. Explicit image and tag overrides remain respected, including channel tags for consumers that opt into them. Final and patch release-note templates carry this notice permanently, because it describes the standing chart behavior, and the preparation tests assert it. The installation check exercises the pre-upgrade hook against the same newly installed version with only its version-transition check disabled. All other enabled preflight checks must pass. This verifies the hook image and health checks, not an upgrade between different Radius versions. diff --git a/docs/release-notes/template.md b/docs/release-notes/template.md index 219db0563f..3e7bb13aff 100644 --- a/docs/release-notes/template.md +++ b/docs/release-notes/template.md @@ -26,6 +26,8 @@ Welcome to our new contributors who have merged their first PR in this release! ## Upgrading to Radius vX.Y.Z +**Helm image pinning:** This release's Helm chart pins Radius component images to the full release version. Restarting pods no longer picks up later patches implicitly; upgrade the chart to receive patched images. Existing channel aliases remain available, and explicitly configured `global.imageTag`, component tags, or image references keep overriding chart defaults. Clear those overrides to adopt version-pinned defaults while retaining other settings. + You can upgrade to this release by upgrading your Radius CLI then running `rad upgrade kubernetes`. Only incremental version upgrades are supported. Consult the [upgrade documentation](https://docs.radapp.io/guides/operations/kubernetes/kubernetes-upgrade/) for full details. ## Full changelog diff --git a/docs/release-notes/template_patch.md b/docs/release-notes/template_patch.md index cd4de2d696..28a43b54d7 100644 --- a/docs/release-notes/template_patch.md +++ b/docs/release-notes/template_patch.md @@ -3,6 +3,8 @@ This patch release includes the fixes listed in the [changelog](#changelog). +**Helm image pinning:** This release's Helm chart pins Radius component images to the full patch version. Restarting pods no longer picks up later patches implicitly; upgrade your CLI and run `rad upgrade kubernetes` to update the chart and images. Explicit channel-tag or image overrides remain respected; clear them to adopt version-pinned defaults. + ## Changelog diff --git a/eng/design-notes/tools/2026-03-goreleaser-release-lifecycle-implementation-plan.md b/eng/design-notes/tools/2026-03-goreleaser-release-lifecycle-implementation-plan.md index 17bb018492..dce5ba94ff 100644 --- a/eng/design-notes/tools/2026-03-goreleaser-release-lifecycle-implementation-plan.md +++ b/eng/design-notes/tools/2026-03-goreleaser-release-lifecycle-implementation-plan.md @@ -223,6 +223,10 @@ For docs and samples, the GitHub REST API version `2026-03-10` returns `workflow Change `deploy/Chart/templates/_helpers.tpl` so final and patch charts reference full-version image tags instead of the truncated `major.minor` channel tag (RC behavior is already full-version). Channel aliases continue to be published for backward compatibility. Call out the patch-pickup behavior change prominently in the release notes of the release that ships it. +The current dashboard and Deployment Engine publishers expose stable channel tags. Before publishing the chart, add full-version references to those verified digests with native ORAS tagging, without rebuilding or moving channel aliases. Verify the dashboard source commit against the plan and the Deployment Engine source and digest against its controller lock. The Radius workflow needs Actions package-write access to both GHCR packages; this uses `GITHUB_TOKEN`, not an expanded App identity. Staged verification inspects the full-version references while the historical parity collector and checked-in baselines retain the pre-migration contract. Chart upgrades use Helm's existing value-merging behavior: defaults advance to the chart version, explicit user overrides remain, and clearing an override adopts the new defaults. The dashboard publisher builds from its sibling tag in parallel with the Radius tag build, so the pin waits a bounded ten minutes for the channel image to carry the planned source before treating a mismatch as a conflict; an existing full-version tag is never awaited. + +- **Publisher prerequisite**: `radius-project/dashboard` must stamp `org.opencontainers.image.revision` with its built commit SHA. Its current build does not emit this label; a companion publisher change and a source-labeled release image are required before this draft can roll out. PR 16's manifest already compares this label with the plan, so the companion change gates PR 16's rollout as well. Do not waive the source check to adopt an unproven channel image. +- **Cleanup**: delete the helper's stable-version truncation and update chart verification expectations in the same PR. Keep channel publication and historical parity baselines unchanged. - **Exit criteria**: chart install from a final release pins full-version images; upgrade path from a channel-tag install verified. - **Rollback**: revert the template; both tag forms exist in the registry. diff --git a/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md b/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md index 41c3fb6bf1..9c1b01f0cf 100644 --- a/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md +++ b/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md @@ -22,20 +22,22 @@ This directory records the review of the 18-pull-request stack that implements t | 14 | `dp/goreleaser-sboms` | [pr-14-goreleaser-sboms.md](./pr-14-goreleaser-sboms.md) | | 15 | `dp/release-controller-trigger-swap` | [pr-15-release-controller.md](./pr-15-release-controller.md) | | 16 | `dp/release-publication-gate` | [pr-16-release-publication-gate.md](./pr-16-release-publication-gate.md) | +| 17 | `dp/helm-immutable-image-tags` | [pr-17-helm-immutable-image-tags.md](./pr-17-helm-immutable-image-tags.md) | Later notes are added as the review progresses up the stack. ## Cross-layer findings -| # | Finding | Layers | Resolution | -|----|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1 | Checksum sidecars: GoReleaser's split checksum files contain only the hash, while the published contract is ` *`. PR 2 and PR 7 encode the bare-hash form as an explained difference; PR 13 normalizes the staged draft assets back to the published contract, and the parity collector keeps validating the published contract. | 2, 7, 13 | Consistent end state; no change required. The verifier in PR 2 accepts both forms from PR 13 onward. | -| 2 | Release state is spread over three stores: JSON lock assets on the GitHub Release, `automation/release-state-` branches for the Deployment Engine lock, and GitHub deployments for coordination receipts. The state branches accumulate one per release and never get deleted, and every release page exposes about a dozen internal JSON assets to end users. | 13, 15, 16 | Proposed: record the Deployment Engine lock as a deployment receipt like the coordination receipts, and fold the per-stage lock files into one `radius-release-manifest.json` asset. PR 13 keeps its eleven per-stage assets as the durable idempotency store, which is sound; consolidation into one merge-immutable manifest is evaluated at PR 16, where the release manifest is introduced. PR 15 adds the Deployment Engine lock as a signed App commit on a per-version state branch, written before any release exists; it is kept, and pruning the branch after publication is evaluated at PR 16. Closed in the PR 16 note: the release manifest consolidates the verification evidence into one immutable asset, the per-stage assets stay as the staging idempotency store because they exist before a manifest can, and the state branch stays because every re-verification after publication reads the lock from it. PR 16's coordination receipts move into one `release-coordination` environment so releases stop creating environments. | -| 3 | Sibling repository commits are frozen at preparation time and the release-plan check fails at merge time when `recipes`, `dashboard`, or `bicep-types-aws` `main` moves, which forces a re-run of Prepare Release for unrelated activity. | 15 | Resolved in the PR 15 note: sibling commits stay frozen at preparation, as the design's plan record requires, and the plan check verifies that each frozen commit is still reachable from its branch instead of requiring equality with the live head, so unrelated sibling activity no longer forces a rerun. | -| 4 | The generated release pull request body ends with a hard-coded "Generated for #12814." line. | 12 | Resolved in the PR 12 note: the line is removed. | -| 5 | `cliff.toml` still carried a `deps` commit parser after PR 8 removed `deps` from the accepted types. | 8 | Resolved in the PR 8 note: the parser is removed and the configuration test covers the new `style` exclusion instead. | -| 6 | The CLI export matrix in the final layer runs seven runners that each download the same snapshot and copy one file. | 18 | Proposed: one job that uploads the seven artifacts. Evaluated when the review reaches PR 18. | -| 7 | Each production image keeps two Dockerfiles (`Dockerfile` for the developer Make path, `Dockerfile.goreleaser` for releases). | 2, 13, 18 | PR 2 carries the static parity guard that PR 13 introduces, so the pairs cannot drift from the first merge. Unifying them is deferred to PR 18: the `Dockerfile` copies produce the production images through the Make path until PR 13, and PR 18 rewrites the Make image targets. | -| 8 | `main` moved after the stack was based: the split of `build.yaml` in PR 5 conflicts with an action bump made on `main`, and the QEMU and Buildx pins in the stack's own workflows trail the versions on `main`. | 2, 5 | Resolved: the bump is carried into the split workflows during the stack rebase, and the PR 2 note pins the snapshot workflow to the same versions. | -| 9 | Mixed RC forms within one version invert SemVer ordering: `0.61.0-rc.2` sorts before `0.61.0-rc1`, and the `rad upgrade` preflight rejects that move as a downgrade. | 11, 12 | PR 11 states the rule in the policy helper, the runbook, and the design, and pins the preflight behavior in a test. Resolved in the PR 12 note: preparation refuses a dotted RC when the channel's current RC or highest RC tag uses the historical form, with a test. | -| 10 | The signed Deployment Engine tag check reads a private repository in `azure-octo` with tokens scoped to `radius-project`: the release App in PR 12 and `GITHUB_TOKEN` in PR 15's controller. Both receive a 404 and the script reported the tag as missing. | 12, 15 | PR 12 mints a publisher App token for `deployment-engine` with Contents read and the script now distinguishes no access from a missing tag. Resolved in the PR 15 note: both controller verifications mint the same publisher App token. | +| # | Finding | Layers | Resolution | +|----|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | Checksum sidecars: GoReleaser's split checksum files contain only the hash, while the published contract is ` *`. PR 2 and PR 7 encode the bare-hash form as an explained difference; PR 13 normalizes the staged draft assets back to the published contract, and the parity collector keeps validating the published contract. | 2, 7, 13 | Consistent end state; no change required. The verifier in PR 2 accepts both forms from PR 13 onward. | +| 2 | Release state is spread over three stores: JSON lock assets on the GitHub Release, `automation/release-state-` branches for the Deployment Engine lock, and GitHub deployments for coordination receipts. The state branches accumulate one per release and never get deleted, and every release page exposes about a dozen internal JSON assets to end users. | 13, 15, 16 | Proposed: record the Deployment Engine lock as a deployment receipt like the coordination receipts, and fold the per-stage lock files into one `radius-release-manifest.json` asset. PR 13 keeps its eleven per-stage assets as the durable idempotency store, which is sound; consolidation into one merge-immutable manifest is evaluated at PR 16, where the release manifest is introduced. PR 15 adds the Deployment Engine lock as a signed App commit on a per-version state branch, written before any release exists; it is kept, and pruning the branch after publication is evaluated at PR 16. Closed in the PR 16 note: the release manifest consolidates the verification evidence into one immutable asset, the per-stage assets stay as the staging idempotency store because they exist before a manifest can, and the state branch stays because every re-verification after publication reads the lock from it. PR 16's coordination receipts move into one `release-coordination` environment so releases stop creating environments. | +| 3 | Sibling repository commits are frozen at preparation time and the release-plan check fails at merge time when `recipes`, `dashboard`, or `bicep-types-aws` `main` moves, which forces a re-run of Prepare Release for unrelated activity. | 15 | Resolved in the PR 15 note: sibling commits stay frozen at preparation, as the design's plan record requires, and the plan check verifies that each frozen commit is still reachable from its branch instead of requiring equality with the live head, so unrelated sibling activity no longer forces a rerun. | +| 4 | The generated release pull request body ends with a hard-coded "Generated for #12814." line. | 12 | Resolved in the PR 12 note: the line is removed. | +| 5 | `cliff.toml` still carried a `deps` commit parser after PR 8 removed `deps` from the accepted types. | 8 | Resolved in the PR 8 note: the parser is removed and the configuration test covers the new `style` exclusion instead. | +| 6 | The CLI export matrix in the final layer runs seven runners that each download the same snapshot and copy one file. | 18 | Proposed: one job that uploads the seven artifacts. Evaluated when the review reaches PR 18. | +| 7 | Each production image keeps two Dockerfiles (`Dockerfile` for the developer Make path, `Dockerfile.goreleaser` for releases). | 2, 13, 18 | PR 2 carries the static parity guard that PR 13 introduces, so the pairs cannot drift from the first merge. Unifying them is deferred to PR 18: the `Dockerfile` copies produce the production images through the Make path until PR 13, and PR 18 rewrites the Make image targets. | +| 8 | `main` moved after the stack was based: the split of `build.yaml` in PR 5 conflicts with an action bump made on `main`, and the QEMU and Buildx pins in the stack's own workflows trail the versions on `main`. | 2, 5 | Resolved: the bump is carried into the split workflows during the stack rebase, and the PR 2 note pins the snapshot workflow to the same versions. | +| 9 | Mixed RC forms within one version invert SemVer ordering: `0.61.0-rc.2` sorts before `0.61.0-rc1`, and the `rad upgrade` preflight rejects that move as a downgrade. | 11, 12 | PR 11 states the rule in the policy helper, the runbook, and the design, and pins the preflight behavior in a test. Resolved in the PR 12 note: preparation refuses a dotted RC when the channel's current RC or highest RC tag uses the historical form, with a test. | +| 10 | The signed Deployment Engine tag check reads a private repository in `azure-octo` with tokens scoped to `radius-project`: the release App in PR 12 and `GITHUB_TOKEN` in PR 15's controller. Both receive a 404 and the script reported the tag as missing. | 12, 15 | PR 12 mints a publisher App token for `deployment-engine` with Contents read and the script now distinguishes no access from a missing tag. Resolved in the PR 15 note: both controller verifications mint the same publisher App token. | +| 11 | The published dashboard images (`0.59`, `0.60`, `latest`) carry no OCI labels and are single-manifest images, while PR 16's manifest compares `org.opencontainers.image.revision` on every dashboard platform with the plan's dashboard commit and PR 17's chart pin requires the same label. Until `radius-project/dashboard` stamps the label, every release stops at verification. | 16, 17 | Confirmed with oras. PR 17's runbook and plan state the prerequisite; the PR 17 note names the exact companion change, the runbook and plan now say it gates PR 16's manifest as well, and PR 16's rollout gates list it. The pin waits a bounded time for the parallel dashboard build so timing alone cannot fail chart publication. | diff --git a/eng/design-notes/tools/2026-09-goreleaser-stack-review/pr-17-helm-immutable-image-tags.md b/eng/design-notes/tools/2026-09-goreleaser-stack-review/pr-17-helm-immutable-image-tags.md new file mode 100644 index 0000000000..08de00fe0a --- /dev/null +++ b/eng/design-notes/tools/2026-09-goreleaser-stack-review/pr-17-helm-immutable-image-tags.md @@ -0,0 +1,47 @@ +# Review note: PR 17 - Helm chart immutable image tags + +- **Pull request**: [#12953](https://github.com/radius-project/radius/pull/12953) +- **Plan phase**: [PR 17](../2026-03-goreleaser-release-lifecycle-implementation-plan.md#pr-17-helm-chart-immutable-image-tags) +- **Stack index**: [README](./README.md) + +## Verdict + +The layer does what the plan asks with little code. The chart helper returns the chart's app version unchanged, so final and patch charts pin every component image to the release version while RC and edge charts keep their behavior. The external Deployment Engine and dashboard images receive full-version tags before the chart is packaged, by retagging their verified digests with ORAS rather than rebuilding anything or moving a channel alias: the Deployment Engine tag is checked against the controller's lock, the dashboard tag against the plan's frozen source commit, and both against the expected platform set. The verification gate from PR 16 expects full-version references everywhere and rejects a stable chart that renders a channel tag. The release-note templates carry the patch-pickup notice, and the Helm client test drives a real Helm install and upgrade with the actual chart templates to show that defaults advance to the new version, explicit overrides survive, and a cleared override adopts the defaults while other stored values remain. Chart unit tests, the Go Helm client tests, and every release suite pass. The rollout prerequisites the pull request lists are real: the repository needs Actions write on the two GHCR packages, and the dashboard image carries no provenance label today. + +Two things needed work: the pin step raced the dashboard publisher, and the dashboard-label prerequisite already gates the previous layer. + +## Changes made in this review + +### 1. The external-image pin waits for a publisher that is still running + +- **What changed**: `pin-image` awaits the channel reference for a bounded time, ten minutes by default polled every thirty seconds, when the reference is absent or still serves another source, and reports the mismatch only when the wait runs out. An existing full-version tag is never awaited, because a mismatch there is a conflict. The helm job's timeout allows for the wait. A test covers a publisher that finishes during the wait and the immediate rejection of a differing immutable tag. +- **Why**: the controller creates the dashboard's sibling tag and the Radius tag within seconds of each other, and the dashboard's build workflow then publishes the channel image from its tag. In the last patch release the dashboard build took four minutes and the Radius image build twelve, so the pin would normally find the image ready, but dashboard builds have taken eleven minutes, and the pin failed closed the moment the channel image still carried the previous source. Resume Release would have recovered it, but as a routine failure rather than an exceptional one. +- **Value**: chart publication no longer depends on which of two parallel builds finishes first. +- **Impact**: a genuine source conflict on the channel image is reported after the wait instead of at once; the message is unchanged, and the conflict on an existing full-version tag is still immediate. + +### 2. The dashboard provenance prerequisite is recorded where it first applies + +- **What changed**: cross-layer finding 11 in the index, a sentence in the runbook and the plan that PR 16's manifest already requires the label, a rollout gate on PR 16's description, and the exact companion change named in the runbook: the dashboard's `Build Image` step and Dockerfile set no OCI labels, so the image build must add `org.opencontainers.image.revision` from the built commit. +- **Why**: PR 16's manifest compares that label on every dashboard platform with the plan's dashboard commit. The published `0.59`, `0.60`, and `latest` dashboard images carry no OCI labels at all, checked with oras, so the gate fails for every release until `radius-project/dashboard` labels its image. PR 17 stated the prerequisite only for the chart cutover. +- **Value**: the rollout order is visible from the stack itself: the dashboard change must land before PR 16 is enabled, not before PR 17. +- **Impact**: none on code; a companion change in the dashboard repository remains required. + +### 3. Small items + +- The helm workflow's Buildx pin matches the other workflows (v4.3.0). +- The gate contract test's new block is indented like the rest of its function. +- The runbook says the release-note notice is permanent. The templates and the preparation tests already treat it that way, while the runbook said to retain it only for the first release that ships the policy. + +## Findings left as-is + +- **Package access**: retagging with `GITHUB_TOKEN` requires the `dashboard` and `deployment-engine` packages to grant the repository write access; the runbook and the rollout gates say so, and no App scope is widened. +- **Single-manifest dashboard image**: the published dashboard image is a single `linux/amd64` manifest rather than an index; the parity targets expect exactly that, and the pin and the manifest handle both forms. +- **RC charts**: the publishers produce the RC tags themselves (`0.61.0-rc.1` for both images), so the pin verifies them without retagging. +- **Shell style**: `release-cutover_test.sh` matches neither shfmt profile, as before this layer. + +## Verification + +- Chart: helm unittest passes (136 tests, including the new final and patch pinning cases); `go test ./pkg/cli/helm` passes with the three real Helm upgrade cases. +- Shell: OCI artifacts (17, one added), cutover (9), parity, installation (6), preparation (17), and SBOM suites pass; ShellCheck is clean for the changed scripts. +- Node: manifest suite (5) passes. actionlint and Prettier pass for the helm workflow and the changed Node and chart test files; markdownlint and cspell pass for the chart README, the release-note templates, the runbook, the plan, and these notes. +- Facts checked outside the repository: the dashboard image labels and manifest form with oras; the `Build Image` step of the dashboard's `build.yaml`; build timings of the `v0.60.2` dashboard and Radius runs; the plan's `linux/amd64` platform expectation for the dashboard. diff --git a/pkg/cli/helm/helmclient_test.go b/pkg/cli/helm/helmclient_test.go index 7e114389b5..a805a48d0b 100644 --- a/pkg/cli/helm/helmclient_test.go +++ b/pkg/cli/helm/helmclient_test.go @@ -17,13 +17,115 @@ limitations under the License. package helm import ( + "bytes" + "io" + "slices" + "strings" "testing" "time" "github.com/stretchr/testify/require" helm "helm.sh/helm/v4/pkg/action" + "helm.sh/helm/v4/pkg/chart/common" + "helm.sh/helm/v4/pkg/chart/v2/loader" + kubefake "helm.sh/helm/v4/pkg/kube/fake" + "helm.sh/helm/v4/pkg/storage" + "helm.sh/helm/v4/pkg/storage/driver" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/yaml" ) +func TestHelmClientImpl_UpgradeImmutableImageDefaults(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + storedTag string + clearTag bool + expectedTag string + }{ + {name: "chart defaults move from channel to patch", expectedTag: "0.61.1"}, + {name: "explicit channel override survives", storedTag: "0.61", expectedTag: "0.61"}, + {name: "cleared channel override adopts patch", storedTag: "0.61", clearTag: true, expectedTag: "0.61.1"}, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + previousChart, err := loader.Load("../../../deploy/Chart") + require.NoError(t, err) + previousChart.Metadata.Version = "0.61.0" + previousChart.Metadata.AppVersion = "0.61.0" + previousChart.Files = slices.DeleteFunc(previousChart.Files, func(file *common.File) bool { + return strings.HasPrefix(file.Name, "crds/") + }) + for _, template := range previousChart.Templates { + if template.Name == "templates/_helpers.tpl" { + require.Contains(t, string(template.Data), "{{- .Chart.AppVersion -}}") + template.Data = bytes.Replace(template.Data, []byte("{{- .Chart.AppVersion -}}"), []byte(`{{- $parts := splitList "." .Chart.AppVersion -}}{{- printf "%s.%s" (index $parts 0) (index $parts 1) -}}`), 1) + } + } + configuration := &helm.Configuration{ + Releases: storage.Init(driver.NewMemory()), + KubeClient: &kubefake.PrintingKubeClient{Out: io.Discard}, + Capabilities: common.DefaultCapabilities, + } + client := NewHelmClient() + values := map[string]any{ + "global": map[string]any{"imageTag": testCase.storedTag}, + "rp": map[string]any{"publicEndpointOverride": "retained.example.test"}, + "preupgrade": map[string]any{"enabled": true}, + } + installed, err := client.RunHelmInstall(configuration, previousChart, values, "radius", "radius-system", false) + require.NoError(t, err) + require.Contains(t, helmWorkloadImages(t, installed.Manifest), "ghcr.io/radius-project/controller:0.61") + + nextChart, err := loader.Load("../../../deploy/Chart") + require.NoError(t, err) + nextChart.Metadata.Version = "0.61.1" + nextChart.Metadata.AppVersion = "0.61.1" + nextChart.Files = slices.DeleteFunc(nextChart.Files, func(file *common.File) bool { + return strings.HasPrefix(file.Name, "crds/") + }) + overrides := map[string]any{} + if testCase.clearTag { + overrides["global"] = map[string]any{"imageTag": ""} + } + upgraded, err := client.RunHelmUpgrade(configuration, nextChart, overrides, "radius", "radius-system", false, true) + require.NoError(t, err) + require.Equal(t, 2, upgraded.Version) + manifest := upgraded.Manifest + for _, hook := range upgraded.Hooks { + manifest += "\n---\n" + hook.Manifest + } + images := helmWorkloadImages(t, manifest) + for _, image := range []string{"applications-rp", "controller", "dynamic-rp", "ucpd", "pre-upgrade", "bicep", "dashboard", "deployment-engine"} { + require.Contains(t, images, "ghcr.io/radius-project/"+image+":"+testCase.expectedTag) + } + require.Equal(t, "retained.example.test", upgraded.Config["rp"].(map[string]any)["publicEndpointOverride"]) + }) + } +} + +func helmWorkloadImages(t *testing.T, manifest string) []string { + t.Helper() + decoder := yaml.NewYAMLOrJSONDecoder(strings.NewReader(manifest), 4096) + var images []string + for { + var workload struct { + Spec struct { + Template corev1.PodTemplateSpec `json:"template"` + } `json:"spec"` + } + err := decoder.Decode(&workload) + if err == io.EOF { + return images + } + require.NoError(t, err) + for _, container := range append(workload.Spec.Template.Spec.Containers, workload.Spec.Template.Spec.InitContainers...) { + images = append(images, container.Image) + } + } +} + func TestHelmClientImpl_RunHelmHistory(t *testing.T) { client := &HelmClientImpl{}