From 72281dcad2a799751e5662b7d747148a34555ce7 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 03:26:18 -0400 Subject: [PATCH 01/14] feat(pusher): add digest-pinned chart-only recovery Failure-Class: FC-runtime-image-boundary Co-authored-by: multica-agent --- .../scripts/check-deployment-concurrency.py | 2 +- .github/workflows/gcp_backend_pusher.yml | 146 ++++++- .../charts/pusher/dev_omi_pusher_values.yaml | 3 + .../charts/pusher/prod_omi_pusher_values.yaml | 3 + .../charts/pusher/templates/deployment.yaml | 18 +- .../verify_pusher_chart_only_recovery.py | 401 ++++++++++++++++++ backend/testing/workflow_contracts.json | 8 +- .../test_verify_pusher_chart_only_recovery.py | 183 ++++++++ backend/tests/unit/test_workflow_contracts.py | 1 + 9 files changed, 757 insertions(+), 8 deletions(-) create mode 100644 backend/scripts/verify_pusher_chart_only_recovery.py create mode 100644 backend/tests/unit/test_verify_pusher_chart_only_recovery.py diff --git a/.github/scripts/check-deployment-concurrency.py b/.github/scripts/check-deployment-concurrency.py index 58728bba0d0..08818d56367 100644 --- a/.github/scripts/check-deployment-concurrency.py +++ b/.github/scripts/check-deployment-concurrency.py @@ -427,7 +427,7 @@ def pusher_preflight_step_is_valid(name: str, step: list[str]) -> bool: nonfatal = any( candidate.strip().startswith("continue-on-error:") or candidate.strip() == "set +e" for candidate in step ) - allowed_conditions = ["if: env.SERVICE == 'pusher'"] if name == "gcp_backend_pusher.yml" else [] + allowed_conditions = ["if: env.SERVICE == 'pusher' && env.CHART_ONLY != 'true'"] if name == "gcp_backend_pusher.yml" else [] if nonfatal or conditions != allowed_conditions: return False diff --git a/.github/workflows/gcp_backend_pusher.yml b/.github/workflows/gcp_backend_pusher.yml index f1fd0977e6a..e8de5a26336 100644 --- a/.github/workflows/gcp_backend_pusher.yml +++ b/.github/workflows/gcp_backend_pusher.yml @@ -23,6 +23,22 @@ on: description: 'Service to deploy' required: false default: 'pusher' + deployment_mode: + description: 'build-and-push (normal) or chart-only (Pusher recovery only)' + required: false + default: 'build-and-push' + image_repository: + description: 'Exact approved Pusher repository for chart-only recovery' + required: false + default: '' + image_digest: + description: 'Exact sha256 digest for chart-only recovery' + required: false + default: '' + ready_baseline: + description: 'Minimum already-Ready Pusher replicas to preserve during chart-only recovery' + required: false + default: '' # The LLM Gateway path reapplies shared backend-secrets; standalone pusher only # mutates its own GKE release and therefore keeps a separate environment lock. @@ -32,6 +48,8 @@ concurrency: env: SERVICE: ${{ github.event.inputs.service || 'pusher' }} + DEPLOYMENT_MODE: ${{ github.event.inputs.deployment_mode || 'build-and-push' }} + CHART_ONLY: ${{ github.event.inputs.deployment_mode == 'chart-only' }} REGION: us-central1 jobs: @@ -57,6 +75,28 @@ jobs: echo "LLM Gateway deploys are dev-only until the prod launch checklist is explicitly completed." exit 1 fi + if [[ "${{ env.DEPLOYMENT_MODE }}" != "build-and-push" && "${{ env.DEPLOYMENT_MODE }}" != "chart-only" ]]; then + echo "deployment_mode must be build-and-push or chart-only." + exit 1 + fi + if [[ "${{ env.DEPLOYMENT_MODE }}" == "chart-only" ]]; then + if [[ "${{ env.SERVICE }}" != "pusher" ]]; then + echo "chart-only recovery is Pusher-only." + exit 1 + fi + if [[ "${{ github.event.inputs.image_repository }}" != "gcr.io/${{ vars.GCP_PROJECT_ID }}/pusher" ]]; then + echo "chart-only recovery repository is not the selected environment's approved Pusher repository." + exit 1 + fi + if [[ ! "${{ github.event.inputs.image_digest }}" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "chart-only recovery requires an exact sha256 digest." + exit 1 + fi + if [[ ! "${{ github.event.inputs.ready_baseline }}" =~ ^[1-9][0-9]*$ ]]; then + echo "chart-only recovery requires a positive Ready containment baseline." + exit 1 + fi + fi # To workaround "no space left on device" issue of GitHub-hosted runner - name: Delete huge unnecessary tools folder @@ -71,6 +111,7 @@ jobs: - name: Admit checked-out production source and image identity run: | set -euo pipefail + python3 -m pip install -q pyyaml CHECKED_OUT_SHA=$(git rev-parse HEAD) if [[ "${{ github.event.inputs.environment }}" == prod ]]; then git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main @@ -126,12 +167,15 @@ jobs: } - name: Login to GCR + if: env.CHART_ONLY != 'true' run: gcloud auth configure-docker - name: Set up Docker Buildx + if: env.CHART_ONLY != 'true' uses: docker/setup-buildx-action@v4 - name: Build and Push Docker image + if: env.CHART_ONLY != 'true' run: | if [[ "${{ env.SERVICE }}" == "llm-gateway" ]]; then docker build -t gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:${IMAGE_TAG} -f backend/Dockerfile . @@ -154,7 +198,7 @@ jobs: project_id: ${{ vars.GCP_PROJECT_ID }} - name: Resolve production pusher runtime targets - if: env.SERVICE == 'pusher' && vars.ENV == 'prod' + if: env.SERVICE == 'pusher' && vars.ENV == 'prod' && env.CHART_ONLY != 'true' id: pusher-runtime-targets run: | set -euo pipefail @@ -180,7 +224,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Apply non-secret pusher runtime config - if: env.SERVICE == 'pusher' + if: env.SERVICE == 'pusher' && env.CHART_ONLY != 'true' env: ACCOUNT_DELETION_HANDLER_URL: ${{ steps.pusher-runtime-targets.outputs.account_deletion_handler_url }} CONVERSATION_SUMMARIZED_APP_IDS: ${{ vars.CONVERSATION_SUMMARIZED_APP_IDS }} @@ -210,12 +254,64 @@ jobs: run: backend/scripts/deploy-backend-config.sh - name: Preflight pusher ConfigMap and Secret references - if: env.SERVICE == 'pusher' + if: env.SERVICE == 'pusher' && env.CHART_ONLY != 'true' run: | python3 -m pip install -q pyyaml python3 backend/scripts/verify_pusher_config_references.py \ --environment ${{ vars.ENV }} --namespace ${{ vars.ENV }}-omi-backend + - name: Capture and validate chart-only Pusher recovery + if: env.SERVICE == 'pusher' && env.CHART_ONLY == 'true' + run: | + set -euo pipefail + NAMESPACE="${{ vars.ENV }}-omi-backend" + RELEASE="${{ vars.ENV }}-omi-pusher" + DEPLOYMENT="$RELEASE" + CONFIGMAP="${{ vars.ENV }}-omi-backend-config" + mkdir -p .pusher-recovery-snapshot + # Capture identity and readiness evidence only. ConfigMap contents are + # never printed or uploaded; the verifier reads its key names locally. + helm history "$RELEASE" -n "$NAMESPACE" -o json > .pusher-recovery-snapshot/helm-history.json + kubectl -n "$NAMESPACE" get deployment "$DEPLOYMENT" -o json > .pusher-recovery-snapshot/deployment.json + kubectl -n "$NAMESPACE" get configmap "$CONFIGMAP" -o json > .pusher-recovery-snapshot/configmap.json + kubectl -n "$NAMESPACE" get service "$RELEASE" -o json > .pusher-recovery-snapshot/service.json + kubectl -n "$NAMESPACE" get hpa "$RELEASE" -o json > .pusher-recovery-snapshot/hpa.json + kubectl -n "$NAMESPACE" get pdb "$RELEASE" -o json > .pusher-recovery-snapshot/pdb.json + kubectl -n "$NAMESPACE" get ingress "$RELEASE" -o json > .pusher-recovery-snapshot/ingress.json + kubectl -n "$NAMESPACE" get serviceaccount "$RELEASE" -o json > .pusher-recovery-snapshot/serviceaccount.json + kubectl -n "$NAMESPACE" get backendconfig "${{ vars.ENV }}-pusher-backend-config" -o json > .pusher-recovery-snapshot/backendconfig.json + kubectl -n "$NAMESPACE" get rs -l app.kubernetes.io/instance="$RELEASE" -o json > .pusher-recovery-snapshot/replicasets.json + kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/instance="$RELEASE" -o json > .pusher-recovery-snapshot/pods.json + helm template "$RELEASE" ./backend/charts/pusher \ + -f "./backend/charts/pusher/${{ vars.ENV }}_omi_pusher_values.yaml" \ + --set-string "image.repository=${{ github.event.inputs.image_repository }}" \ + --set-string 'image.tag=' \ + --set-string "image.digest=${{ github.event.inputs.image_digest }}" \ + > .pusher-recovery-snapshot/rendered.yaml + python3 backend/scripts/verify_pusher_chart_only_recovery.py preflight \ + --environment "${{ vars.ENV }}" --namespace "$NAMESPACE" --release "$RELEASE" --deployment "$DEPLOYMENT" \ + --repository "${{ github.event.inputs.image_repository }}" --expected-repository "gcr.io/${{ vars.GCP_PROJECT_ID }}/pusher" \ + --digest "${{ github.event.inputs.image_digest }}" --ready-baseline "${{ github.event.inputs.ready_baseline }}" \ + --live-deployment .pusher-recovery-snapshot/deployment.json \ + --live-configmap .pusher-recovery-snapshot/configmap.json \ + --live-service .pusher-recovery-snapshot/service.json \ + --live-hpa .pusher-recovery-snapshot/hpa.json \ + --live-pdb .pusher-recovery-snapshot/pdb.json \ + --live-ingress .pusher-recovery-snapshot/ingress.json \ + --live-serviceaccount .pusher-recovery-snapshot/serviceaccount.json \ + --live-backendconfig .pusher-recovery-snapshot/backendconfig.json \ + --live-pods .pusher-recovery-snapshot/pods.json \ + --rendered .pusher-recovery-snapshot/rendered.yaml \ + --evidence pusher-recovery-evidence.json + + - name: Upload redacted chart-only recovery evidence + if: always() && env.SERVICE == 'pusher' && env.CHART_ONLY == 'true' + uses: actions/upload-artifact@v4 + with: + name: pusher-chart-only-recovery-evidence-${{ github.run_id }} + path: pusher-recovery-evidence.json + if-no-files-found: ignore + - name: Deploy Pusher to GKE cluster using Helm env: LLM_GATEWAY_GSA: ${{ vars.LLM_GATEWAY_GSA }} @@ -250,13 +346,55 @@ jobs: fi python3 backend/scripts/deploy_status_report.py --env ${{ vars.ENV }} --include-gke --gke-service llm-gateway else - helm -n ${{ vars.ENV }}-omi-backend upgrade --install ${{ vars.ENV }}-omi-pusher ./backend/charts/pusher -f ./backend/charts/pusher/${{ vars.ENV }}_omi_pusher_values.yaml --set-string "image.tag=${IMAGE_TAG}" + if [[ "${{ env.CHART_ONLY }}" == "true" ]]; then + # Re-read the live objects immediately before mutation. The + # verifier fails closed if the digest, ConfigMap key reference, + # ready baseline, or target identity can no longer be proven. + NAMESPACE="${{ vars.ENV }}-omi-backend" + RELEASE="${{ vars.ENV }}-omi-pusher" + kubectl -n "$NAMESPACE" get deployment "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-deployment.json + kubectl -n "$NAMESPACE" get configmap "${{ vars.ENV }}-omi-backend-config" -o json > .pusher-recovery-snapshot/preapply-configmap.json + kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/instance="$RELEASE" -o json > .pusher-recovery-snapshot/preapply-pods.json + kubectl -n "$NAMESPACE" get ingress "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-ingress.json + kubectl -n "$NAMESPACE" get serviceaccount "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-serviceaccount.json + kubectl -n "$NAMESPACE" get backendconfig "${{ vars.ENV }}-pusher-backend-config" -o json > .pusher-recovery-snapshot/preapply-backendconfig.json + python3 backend/scripts/verify_pusher_chart_only_recovery.py preflight \ + --environment "${{ vars.ENV }}" --namespace "$NAMESPACE" --release "$RELEASE" --deployment "$RELEASE" \ + --repository "${{ github.event.inputs.image_repository }}" --expected-repository "gcr.io/${{ vars.GCP_PROJECT_ID }}/pusher" \ + --digest "${{ github.event.inputs.image_digest }}" --ready-baseline "${{ github.event.inputs.ready_baseline }}" \ + --live-deployment .pusher-recovery-snapshot/preapply-deployment.json \ + --live-configmap .pusher-recovery-snapshot/preapply-configmap.json \ + --live-service .pusher-recovery-snapshot/service.json \ + --live-hpa .pusher-recovery-snapshot/hpa.json \ + --live-pdb .pusher-recovery-snapshot/pdb.json \ + --live-ingress .pusher-recovery-snapshot/preapply-ingress.json \ + --live-serviceaccount .pusher-recovery-snapshot/preapply-serviceaccount.json \ + --live-backendconfig .pusher-recovery-snapshot/preapply-backendconfig.json \ + --live-pods .pusher-recovery-snapshot/preapply-pods.json \ + --rendered .pusher-recovery-snapshot/rendered.yaml \ + --expected-evidence pusher-recovery-evidence.json + helm -n "$NAMESPACE" upgrade "$RELEASE" ./backend/charts/pusher \ + -f "./backend/charts/pusher/${{ vars.ENV }}_omi_pusher_values.yaml" \ + --set-string "image.repository=${{ github.event.inputs.image_repository }}" \ + --set-string 'image.tag=' \ + --set-string "image.digest=${{ github.event.inputs.image_digest }}" + else + helm -n ${{ vars.ENV }}-omi-backend upgrade --install ${{ vars.ENV }}-omi-pusher ./backend/charts/pusher -f ./backend/charts/pusher/${{ vars.ENV }}_omi_pusher_values.yaml --set-string "image.tag=${IMAGE_TAG}" + fi # Must cover the chart-derived Pusher rollout budget (fourteen # availability waves at the current production HPA ceiling). if ! kubectl -n ${{ vars.ENV }}-omi-backend rollout status deploy/${{ vars.ENV }}-omi-pusher --timeout=9600s; then python3 backend/scripts/deploy_status_report.py --env ${{ vars.ENV }} --include-gke --gke-service pusher || true exit 1 fi + if [[ "${{ env.CHART_ONLY }}" == "true" ]]; then + kubectl -n ${{ vars.ENV }}-omi-backend get deployment ${{ vars.ENV }}-omi-pusher -o json > .pusher-recovery-snapshot/postrollout-deployment.json + python3 backend/scripts/verify_pusher_chart_only_recovery.py postverify \ + --environment "${{ vars.ENV }}" --namespace "${{ vars.ENV }}-omi-backend" --deployment "${{ vars.ENV }}-omi-pusher" \ + --repository "${{ github.event.inputs.image_repository }}" --expected-repository "gcr.io/${{ vars.GCP_PROJECT_ID }}/pusher" \ + --digest "${{ github.event.inputs.image_digest }}" --ready-baseline "${{ github.event.inputs.ready_baseline }}" \ + --live-deployment .pusher-recovery-snapshot/postrollout-deployment.json + fi python3 backend/scripts/deploy_status_report.py --env ${{ vars.ENV }} --include-gke --gke-service pusher fi diff --git a/backend/charts/pusher/dev_omi_pusher_values.yaml b/backend/charts/pusher/dev_omi_pusher_values.yaml index 83a3d2db599..90dda94668e 100644 --- a/backend/charts/pusher/dev_omi_pusher_values.yaml +++ b/backend/charts/pusher/dev_omi_pusher_values.yaml @@ -18,6 +18,9 @@ image: pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: "" + # An exact immutable digest for chart-only recovery. It cannot be combined + # with tag; ordinary development deployments continue to supply image.tag. + digest: "" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: [] diff --git a/backend/charts/pusher/prod_omi_pusher_values.yaml b/backend/charts/pusher/prod_omi_pusher_values.yaml index 85a7700cabe..54164d6aa28 100644 --- a/backend/charts/pusher/prod_omi_pusher_values.yaml +++ b/backend/charts/pusher/prod_omi_pusher_values.yaml @@ -19,6 +19,9 @@ image: pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: "" + # An exact immutable digest for chart-only recovery. It cannot be combined + # with tag; ordinary production deployments continue to supply image.tag. + digest: "" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: [] diff --git a/backend/charts/pusher/templates/deployment.yaml b/backend/charts/pusher/templates/deployment.yaml index fd9926564e1..cbf480cdb63 100644 --- a/backend/charts/pusher/templates/deployment.yaml +++ b/backend/charts/pusher/templates/deployment.yaml @@ -54,7 +54,23 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - image: "{{ required "image.repository is required" .Values.image.repository }}:{{ required "image.tag is required" .Values.image.tag }}" + {{- $repository := required "image.repository is required" .Values.image.repository }} + {{- $tag := default "" .Values.image.tag }} + {{- $digest := default "" .Values.image.digest }} + {{- if or (contains "@" $repository) (not (regexMatch "^[a-z0-9][a-z0-9.-]*(?::[0-9]+)?(?:/[a-z0-9][a-z0-9._-]*)+$" $repository)) }} + {{- fail "image.repository must not include a tag or digest" }} + {{- end }} + {{- if and $tag $digest }} + {{- fail "image.tag and image.digest are mutually exclusive" }} + {{- end }} + {{- if $digest }} + {{- if not (regexMatch "^sha256:[a-f0-9]{64}$" $digest) }} + {{- fail "image.digest must be sha256:<64 lowercase hex characters>" }} + {{- end }} + image: "{{ $repository }}@{{ $digest }}" + {{- else }} + image: "{{ $repository }}:{{ required "image.tag is required when image.digest is empty" $tag }}" + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.envFrom }} envFrom: diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py new file mode 100644 index 00000000000..f2236471482 --- /dev/null +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +"""Validate the narrow, digest-pinned Pusher chart-only recovery profile. + +The workflow supplies only Kubernetes object JSON/YAML to this program. It +never reads ConfigMap values, Secret values, or pod logs. The profile permits +only the known REDIS_DB_HOST source repair and an image identity that is already +an exact digest; any other chart-owned workload drift fails before Helm writes. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import re +from pathlib import Path +from typing import Any + +import yaml + +DIGEST = re.compile(r"^sha256:[a-f0-9]{64}$") +ENVIRONMENTS = {"dev", "prod"} +VOLATILE_METADATA = { + "creationTimestamp", + "generation", + "managedFields", + "resourceVersion", + "selfLink", + "uid", +} + + +def load_object(path: str) -> dict[str, Any]: + """Load exactly one Kubernetes object without ever serializing its data.""" + documents = [doc for doc in yaml.safe_load_all(Path(path).read_text(encoding="utf-8")) if isinstance(doc, dict)] + if len(documents) != 1: + raise ValueError(f"{path} must contain exactly one Kubernetes object") + return documents[0] + + +def load_rendered_deployment(path: str) -> dict[str, Any]: + documents = [doc for doc in yaml.safe_load_all(Path(path).read_text(encoding="utf-8")) if isinstance(doc, dict)] + deployments = [doc for doc in documents if doc.get("kind") == "Deployment"] + if len(deployments) != 1: + raise ValueError("rendered chart must contain exactly one Deployment") + return deployments[0] + + +def rendered_resource(path: str, kind: str, name: str) -> dict[str, Any]: + documents = [doc for doc in yaml.safe_load_all(Path(path).read_text(encoding="utf-8")) if isinstance(doc, dict)] + matches = [doc for doc in documents if doc.get("kind") == kind and doc.get("metadata", {}).get("name") == name] + if len(matches) != 1: + raise ValueError(f"rendered chart must contain exactly one {kind}/{name}") + return matches[0] + + +def expected_targets(environment: str) -> tuple[str, str, str, str]: + if environment not in ENVIRONMENTS: + raise ValueError("environment must be dev or prod") + return ( + f"{environment}-omi-backend", + f"{environment}-omi-pusher", + f"{environment}-omi-pusher", + f"{environment}-omi-backend-config", + ) + + +def validate_identity(repository: str, digest: str, expected_repository: str | None = None) -> list[str]: + failures: list[str] = [] + leaf = repository.rsplit("/", 1)[-1] + if not repository or "@" in repository or "://" in repository or repository.endswith(":") or ":" in leaf: + failures.append("image repository must be a non-empty repository without a tag or digest") + if not DIGEST.fullmatch(digest): + failures.append("image digest must be sha256:<64 lowercase hex characters>") + if expected_repository and repository != expected_repository: + failures.append("image repository does not match the selected environment's approved Pusher repository") + return failures + + +def primary_container(deployment: dict[str, Any]) -> dict[str, Any]: + containers = deployment.get("spec", {}).get("template", {}).get("spec", {}).get("containers", []) + matches = [item for item in containers if isinstance(item, dict) and item.get("name") == "pusher"] + if len(matches) != 1: + raise ValueError("Pusher Deployment must contain exactly one pusher container") + return matches[0] + + +def redis_entry(deployment: dict[str, Any]) -> dict[str, Any]: + entries = [entry for entry in primary_container(deployment).get("env", []) if entry.get("name") == "REDIS_DB_HOST"] + if len(entries) != 1: + raise ValueError("Pusher Deployment must contain exactly one REDIS_DB_HOST entry") + return entries[0] + + +def validate_redis_source(deployment: dict[str, Any], configmap: str) -> list[str]: + try: + entry = redis_entry(deployment) + except ValueError as exc: + return [str(exc)] + value_from = entry.get("valueFrom") + expected = {"name": configmap, "key": "REDIS_DB_HOST"} + if not isinstance(value_from, dict) or value_from.get("configMapKeyRef") != expected: + return ["REDIS_DB_HOST must use the selected backend ConfigMap key"] + if value_from.get("secretKeyRef", object()) is not None: + return ["REDIS_DB_HOST must not retain a Secret source"] + return [] + + +def image_of(deployment: dict[str, Any]) -> str: + image = primary_container(deployment).get("image") + if not isinstance(image, str): + raise ValueError("Pusher container image is missing") + return image + + +def validate_target(obj: dict[str, Any], kind: str, name: str, namespace: str) -> list[str]: + metadata = obj.get("metadata", {}) + if obj.get("kind") != kind: + return [f"expected {kind}, found {obj.get('kind')!r}"] + failures = [] + if metadata.get("name") != name: + failures.append(f"expected {kind} name {name}") + if metadata.get("namespace") not in (None, namespace): + failures.append(f"expected {kind} namespace {namespace}") + return failures + + +def is_concurrent_rollout(deployment: dict[str, Any]) -> bool: + metadata = deployment.get("metadata", {}) + status = deployment.get("status", {}) + generation = metadata.get("generation") + observed = status.get("observedGeneration") + return generation is None or observed is None or generation != observed + + +def ready_replicas(deployment: dict[str, Any]) -> int: + value = deployment.get("status", {}).get("readyReplicas", 0) + return value if isinstance(value, int) else 0 + + +def normalize(obj: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(obj) + result.pop("status", None) + metadata = result.get("metadata") + if isinstance(metadata, dict): + for key in VOLATILE_METADATA: + metadata.pop(key, None) + metadata.pop("namespace", None) + if result.get("kind") == "Service": + spec = result.get("spec", {}) + for key in ( + "clusterIP", + "clusterIPs", + "healthCheckNodePort", + "ipFamilies", + "ipFamilyPolicy", + "sessionAffinityConfig", + ): + spec.pop(key, None) + return remove_nulls(result) + + +def remove_nulls(value: Any) -> Any: + if isinstance(value, dict): + return {key: remove_nulls(child) for key, child in value.items() if child is not None} + if isinstance(value, list): + return [remove_nulls(child) for child in value] + return value + + +def replace_env_by_name(entries: list[dict[str, Any]], desired: dict[str, Any]) -> list[dict[str, Any]]: + """Model Kubernetes named-list strategic merge, including explicit null deletion.""" + result = copy.deepcopy(entries) + for index, entry in enumerate(result): + if entry.get("name") == desired.get("name"): + result[index] = strategic_merge(entry, desired) + return result + result.append(copy.deepcopy(desired)) + return result + + +def strategic_merge(live: Any, desired: Any) -> Any: + if desired is None: + return None + if not isinstance(live, dict) or not isinstance(desired, dict): + return copy.deepcopy(desired) + merged = copy.deepcopy(live) + for key, value in desired.items(): + if value is None: + merged.pop(key, None) + else: + merged[key] = strategic_merge(merged.get(key), value) + return merged + + +def allowed_recovery_drift(live: dict[str, Any], rendered: dict[str, Any]) -> list[str]: + """Compare chart-owned objects after replacing only image and REDIS source.""" + adapted = copy.deepcopy(live) + try: + live_container = primary_container(adapted) + desired_container = primary_container(rendered) + live_container["image"] = desired_container["image"] + live_container["env"] = replace_env_by_name(live_container.get("env", []), redis_entry(rendered)) + except ValueError as exc: + return [str(exc)] + return ( + [] + if normalize(adapted) == normalize(rendered) + else ["recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition"] + ) + + +def validate_chart_owned_resource_drift(live: dict[str, Any], rendered: dict[str, Any], kind: str) -> list[str]: + """Require Service/HPA/PDB identity and spec to remain unchanged.""" + return ( + [] + if normalize(live) == normalize(rendered) + else [f"recovery profile would change {kind} outside the allowlist"] + ) + + +def configmap_has_key(configmap: dict[str, Any], name: str, namespace: str) -> list[str]: + failures = validate_target(configmap, "ConfigMap", name, namespace) + data = configmap.get("data") + if not isinstance(data, dict) or "REDIS_DB_HOST" not in data: + failures.append("required ConfigMap key REDIS_DB_HOST is unavailable") + return failures + + +def serving_pusher_digests(pods: dict[str, Any]) -> set[str]: + """Read image IDs from Ready Pusher containers, never resolving a mutable tag.""" + digests: set[str] = set() + ready_containers = 0 + for pod in pods.get("items", []): + for status in pod.get("status", {}).get("containerStatuses", []): + if status.get("name") != "pusher" or not status.get("ready"): + continue + ready_containers += 1 + image_id = status.get("imageID") + match = re.search(r"sha256:[a-f0-9]{64}", image_id or "") + if not match: + raise ValueError("a Ready Pusher container has no immutable image digest status") + digests.add(match.group(0)) + if not ready_containers: + raise ValueError("no Ready Pusher container image status is available") + return digests + + +def serving_pusher_images(pods: dict[str, Any]) -> set[str]: + """Return full repository@digest identities from Ready serving containers.""" + images: set[str] = set() + for pod in pods.get("items", []): + for status in pod.get("status", {}).get("containerStatuses", []): + if status.get("name") != "pusher" or not status.get("ready"): + continue + image_id = status.get("imageID") or "" + identity = image_id.split("://", 1)[-1] + if not re.fullmatch(r"[a-z0-9][a-z0-9./_-]*@sha256:[a-f0-9]{64}", identity): + raise ValueError("a Ready Pusher container has no full immutable repository and digest status") + images.add(identity) + if not images: + raise ValueError("no Ready Pusher container image status is available") + return images + + +def redact(obj: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(obj) + for key in ("data", "stringData", "binaryData"): + result.pop(key, None) + return result + + +def evidence(deployment: dict[str, Any], configmap: dict[str, Any], baseline: int) -> dict[str, Any]: + return { + "deployment_image": image_of(deployment), + "deployment_generation": deployment.get("metadata", {}).get("generation"), + "configmap_resource_version": configmap.get("metadata", {}).get("resourceVersion"), + "ready_baseline": baseline, + "ready_replicas": ready_replicas(deployment), + "configmap": redact(configmap), + } + + +def preflight(args: argparse.Namespace) -> list[str]: + namespace, release, deployment_name, configmap_name = expected_targets(args.environment) + failures = validate_identity(args.repository, args.digest, args.expected_repository) + if args.namespace != namespace or args.release != release or args.deployment != deployment_name: + failures.append("selected namespace, release, or Deployment is not the selected environment's Pusher target") + live = load_object(args.live_deployment) + rendered = load_rendered_deployment(args.rendered) + configmap = load_object(args.live_configmap) + failures += validate_target(live, "Deployment", deployment_name, namespace) + failures += validate_target(rendered, "Deployment", deployment_name, namespace) + failures += configmap_has_key(configmap, configmap_name, namespace) + expected_image = f"{args.repository}@{args.digest}" + try: + if image_of(rendered) != expected_image: + failures.append("rendered Pusher image is not the admitted exact digest") + except ValueError as exc: + failures.append(str(exc)) + failures += validate_redis_source(rendered, configmap_name) + failures += allowed_recovery_drift(live, rendered) + if not args.live_pods: + failures.append("recovery preflight is missing live Pusher pod image-status evidence") + else: + try: + images = serving_pusher_images(load_object(args.live_pods)) + if images != {f"{args.repository}@{args.digest}"}: + failures.append("serving Ready Pusher repository and digest do not exactly match the admitted image") + except ValueError as exc: + failures.append(str(exc)) + for kind, name, supplied in ( + ("Service", release, args.live_service), + ("HorizontalPodAutoscaler", release, args.live_hpa), + ("PodDisruptionBudget", release, args.live_pdb), + ("Ingress", release, args.live_ingress), + ("ServiceAccount", release, args.live_serviceaccount), + ("BackendConfig", f"{args.environment}-pusher-backend-config", args.live_backendconfig), + ): + if not supplied: + failures.append(f"recovery preflight is missing live {kind} evidence") + continue + live_resource = load_object(supplied) + failures += validate_target(live_resource, kind, name, namespace) + try: + failures += validate_chart_owned_resource_drift( + live_resource, rendered_resource(args.rendered, kind, name), kind + ) + except ValueError as exc: + failures.append(str(exc)) + if args.expected_evidence: + previous = json.loads(Path(args.expected_evidence).read_text(encoding="utf-8")) + if previous.get("deployment_image") != image_of(live): + failures.append("Pusher image identity changed after the captured preflight") + if previous.get("configmap_resource_version") != configmap.get("metadata", {}).get("resourceVersion"): + failures.append("Pusher ConfigMap identity changed after the captured preflight") + if is_concurrent_rollout(live): + failures.append("concurrent Pusher rollout detected") + if ready_replicas(live) < args.ready_baseline: + failures.append("Pusher Ready count is below the declared containment baseline") + if not failures and args.evidence: + Path(args.evidence).write_text( + json.dumps(evidence(live, configmap, args.ready_baseline), indent=2) + "\n", encoding="utf-8" + ) + return failures + + +def postverify(args: argparse.Namespace) -> list[str]: + namespace, _release, deployment_name, configmap_name = expected_targets(args.environment) + deployment = load_object(args.live_deployment) + failures = validate_target(deployment, "Deployment", deployment_name, namespace) + try: + if image_of(deployment) != f"{args.repository}@{args.digest}": + failures.append("serving Pusher image is not the admitted exact digest") + except ValueError as exc: + failures.append(str(exc)) + failures += validate_redis_source(deployment, configmap_name) + if ready_replicas(deployment) < args.ready_baseline: + failures.append("Pusher Ready count is below the declared containment baseline") + return failures + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("preflight", "postverify")) + parser.add_argument("--environment", required=True, choices=sorted(ENVIRONMENTS)) + parser.add_argument("--namespace", required=True) + parser.add_argument("--release", default="") + parser.add_argument("--deployment", required=True) + parser.add_argument("--repository", required=True) + parser.add_argument("--expected-repository") + parser.add_argument("--digest", required=True) + parser.add_argument("--ready-baseline", type=int, required=True) + parser.add_argument("--live-deployment", required=True) + parser.add_argument("--live-configmap") + parser.add_argument("--rendered") + parser.add_argument("--evidence") + parser.add_argument("--expected-evidence") + parser.add_argument("--live-service") + parser.add_argument("--live-hpa") + parser.add_argument("--live-pdb") + parser.add_argument("--live-pods") + parser.add_argument("--live-ingress") + parser.add_argument("--live-serviceaccount") + parser.add_argument("--live-backendconfig") + args = parser.parse_args() + if args.ready_baseline <= 0: + parser.error("--ready-baseline must be positive") + if args.mode == "preflight" and (not args.live_configmap or not args.rendered): + parser.error("preflight requires --live-configmap and --rendered") + failures = preflight(args) if args.mode == "preflight" else postverify(args) + if failures: + for failure in failures: + print(f"FAIL: {failure}") + return 1 + print(f"Pusher chart-only {args.mode} passed without exposing ConfigMap values.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/testing/workflow_contracts.json b/backend/testing/workflow_contracts.json index a63c9ddc4f4..43addee5c41 100644 --- a/backend/testing/workflow_contracts.json +++ b/backend/testing/workflow_contracts.json @@ -585,9 +585,13 @@ "backend/charts/pusher/**", ".github/workflows/gcp_backend_pusher.yml", ".github/workflows/gcp_backend_pusher_auto_deploy.yml", - "backend/scripts/verify_pusher_rollout_budget.py" + "backend/scripts/verify_pusher_rollout_budget.py", + "backend/scripts/verify_pusher_chart_only_recovery.py" + ], + "tests": [ + "tests/unit/test_verify_pusher_rollout_budget.py", + "tests/unit/test_verify_pusher_chart_only_recovery.py" ], - "tests": ["tests/unit/test_verify_pusher_rollout_budget.py"], "checks": [], "invariants": [ "Pusher's Kubernetes progress deadline and CI rollout wait both cover every configured healthy-availability wave at the committed HPA ceiling", diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py new file mode 100644 index 00000000000..8a225a6d704 --- /dev/null +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -0,0 +1,183 @@ +"""Regression coverage for the digest-pinned Pusher recovery profile.""" + +from __future__ import annotations + +import copy +from pathlib import Path +import runpy +import subprocess +from types import SimpleNamespace + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "verify_pusher_chart_only_recovery.py" +DIGEST = "sha256:" + "a" * 64 + + +@pytest.fixture +def recovery() -> SimpleNamespace: + return SimpleNamespace(**runpy.run_path(str(SCRIPT))) + + +def deployment(image: str, redis_source: dict | None = None) -> dict: + return { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": "prod-omi-pusher", "namespace": "prod-omi-backend", "generation": 2}, + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "pusher", + "image": image, + "env": [ + { + "name": "REDIS_DB_HOST", + "valueFrom": redis_source + or { + "configMapKeyRef": {"name": "prod-omi-backend-config", "key": "REDIS_DB_HOST"}, + "secretKeyRef": None, + }, + }, + {"name": "KEEP", "value": "unchanged"}, + ], + "resources": {"requests": {"cpu": "1"}}, + "readinessProbe": {"httpGet": {"path": "/health"}}, + } + ] + } + } + }, + "status": {"observedGeneration": 2, "readyReplicas": 7}, + } + + +def test_exact_digest_identity_rejects_tag_digest_ambiguity_and_repository_mutation(recovery: SimpleNamespace): + assert recovery.validate_identity("gcr.io/project/pusher", DIGEST, "gcr.io/project/pusher") == [] + assert recovery.validate_identity("gcr.io/project/pusher:tag", DIGEST) + assert recovery.validate_identity("gcr.io/other/pusher", DIGEST, "gcr.io/project/pusher") + assert recovery.validate_identity("gcr.io/project/pusher", "sha256:abc") + + +def test_historical_named_env_strategic_merge_removes_secret_source(recovery: SimpleNamespace): + live = { + "name": "REDIS_DB_HOST", + "valueFrom": {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}}, + } + desired = { + "name": "REDIS_DB_HOST", + "valueFrom": { + "configMapKeyRef": {"name": "prod-omi-backend-config", "key": "REDIS_DB_HOST"}, + "secretKeyRef": None, + }, + } + merged = recovery.replace_env_by_name([live], desired)[0] + assert merged == { + "name": "REDIS_DB_HOST", + "valueFrom": {"configMapKeyRef": {"name": "prod-omi-backend-config", "key": "REDIS_DB_HOST"}}, + } + + +def test_recovery_profile_allows_only_exact_image_and_redis_transition(recovery: SimpleNamespace): + live = deployment( + "gcr.io/project/pusher:2ae7f78", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}} + ) + rendered = deployment(f"gcr.io/project/pusher@{DIGEST}") + assert recovery.allowed_recovery_drift(live, rendered) == [] + changed = copy.deepcopy(rendered) + changed["spec"]["template"]["spec"]["containers"][0]["resources"]["requests"]["cpu"] = "2" + assert recovery.allowed_recovery_drift(live, changed) == [ + "recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition" + ] + + +def test_redis_validation_rejects_secret_and_configmap_errors(recovery: SimpleNamespace): + assert recovery.validate_redis_source(deployment("x").copy(), "prod-omi-backend-config") == [] + secret = deployment("x", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}}) + assert recovery.validate_redis_source(secret, "prod-omi-backend-config") == [ + "REDIS_DB_HOST must use the selected backend ConfigMap key" + ] + + +def test_target_and_readiness_guards_fail_closed(recovery: SimpleNamespace): + live = deployment("x") + live["status"]["observedGeneration"] = 1 + assert recovery.is_concurrent_rollout(live) + assert recovery.ready_replicas(live) == 7 + assert recovery.expected_targets("prod") == ( + "prod-omi-backend", + "prod-omi-pusher", + "prod-omi-pusher", + "prod-omi-backend-config", + ) + + +@pytest.mark.parametrize("kind", ["Service", "HorizontalPodAutoscaler", "PodDisruptionBudget"]) +def test_chart_owned_resources_reject_unexpected_drift(recovery: SimpleNamespace, kind: str): + live = {"kind": kind, "metadata": {"name": "prod-omi-pusher"}, "spec": {"guard": "unchanged"}} + rendered = copy.deepcopy(live) + assert recovery.validate_chart_owned_resource_drift(live, rendered, kind) == [] + rendered["spec"]["guard"] = "changed" + assert recovery.validate_chart_owned_resource_drift(live, rendered, kind) == [ + f"recovery profile would change {kind} outside the allowlist" + ] + + +def test_serving_digest_is_read_only_and_rejects_mixed_or_missing_status(recovery: SimpleNamespace): + pods = { + "items": [ + { + "status": { + "containerStatuses": [ + {"name": "pusher", "ready": True, "imageID": f"docker://gcr.io/project/pusher@{DIGEST}"} + ] + } + } + ] + } + assert recovery.serving_pusher_digests(pods) == {DIGEST} + mixed = copy.deepcopy(pods) + mixed["items"].append( + { + "status": { + "containerStatuses": [ + {"name": "pusher", "ready": True, "imageID": "docker://gcr.io/project/pusher@sha256:" + "b" * 64} + ] + } + } + ) + assert recovery.serving_pusher_digests(mixed) == {DIGEST, "sha256:" + "b" * 64} + with pytest.raises(ValueError, match="no immutable image digest"): + recovery.serving_pusher_digests( + {"items": [{"status": {"containerStatuses": [{"name": "pusher", "ready": True}]}}]} + ) + assert recovery.serving_pusher_images(pods) == {f"gcr.io/project/pusher@{DIGEST}"} + + +def test_chart_only_workflow_skips_build_push_and_normal_paths_stay_available(): + workflow = (SCRIPT.parents[2] / ".github/workflows/gcp_backend_pusher.yml").read_text(encoding="utf-8") + assert "deployment_mode:" in workflow + assert "if: env.CHART_ONLY != 'true'" in workflow + assert "Build and Push Docker image" in workflow + assert "if: env.SERVICE == 'pusher' && env.CHART_ONLY != 'true'" in workflow + assert "if: env.SERVICE == 'pusher' && env.CHART_ONLY == 'true'" in workflow + assert "--expected-evidence pusher-recovery-evidence.json" in workflow + + +def test_chart_renders_exact_digest_and_rejects_ambiguous_or_mutated_repository(): + chart = SCRIPT.parents[1] / "charts" / "pusher" + values = chart / "prod_omi_pusher_values.yaml" + base = ["helm", "template", "prod-omi-pusher", str(chart), "-f", str(values)] + valid = subprocess.run( + [*base, "--set-string", "image.tag=", "--set-string", f"image.digest={DIGEST}"], capture_output=True, text=True + ) + assert valid.returncode == 0, valid.stderr + assert f"gcr.io/based-hardware/pusher@{DIGEST}" in valid.stdout + for arguments in ( + ["--set-string", "image.tag=tag", "--set-string", f"image.digest={DIGEST}"], + ["--set-string", "image.tag=", "--set-string", "image.digest=sha256:bad"], + ["--set-string", "image.repository=gcr.io/based-hardware/pusher:tag", "--set-string", f"image.digest={DIGEST}"], + ): + result = subprocess.run([*base, *arguments], capture_output=True, text=True) + assert result.returncode != 0 diff --git a/backend/tests/unit/test_workflow_contracts.py b/backend/tests/unit/test_workflow_contracts.py index 09d586948e8..de12e0d3925 100644 --- a/backend/tests/unit/test_workflow_contracts.py +++ b/backend/tests/unit/test_workflow_contracts.py @@ -60,6 +60,7 @@ def test_workflow_contract_sources_select_adjacent_tests(selector_and_all_tests) "backend/charts/pusher/templates/deployment.yaml": "tests/unit/test_rendered_deployment_contract.py", ".github/workflows/gcp_backend_pusher.yml": "tests/unit/test_verify_pusher_rollout_budget.py", "backend/scripts/verify_pusher_rollout_budget.py": "tests/unit/test_verify_pusher_rollout_budget.py", + "backend/scripts/verify_pusher_chart_only_recovery.py": "tests/unit/test_verify_pusher_chart_only_recovery.py", "backend/scripts/validate_rendered_deployment_contract.py": "tests/unit/test_rendered_deployment_contract.py", ".github/workflows/gcp_backend_auto_dev.yml": "tests/unit/test_verify_backend_release_vector.py", ".github/workflows/gcp_llm_gateway.yml": "tests/unit/test_preflight_cloud_run_deploy.py", From b4bcdac1a0bdba0eb380126c026700b3940513cd Mon Sep 17 00:00:00 2001 From: David Zhang <9387252+Git-on-my-level@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:10:21 +0700 Subject: [PATCH 02/14] fix(pusher): upgrade upload-artifact to v7 in chart-only recovery workflow The Hygiene CI check rejects deprecated Node-20 action versions. actions/upload-artifact@v4 was added by this PR; upgrade to v7 to match the repo-wide standard. --- .github/workflows/gcp_backend_pusher.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gcp_backend_pusher.yml b/.github/workflows/gcp_backend_pusher.yml index e8de5a26336..92bc35e89bf 100644 --- a/.github/workflows/gcp_backend_pusher.yml +++ b/.github/workflows/gcp_backend_pusher.yml @@ -306,7 +306,7 @@ jobs: - name: Upload redacted chart-only recovery evidence if: always() && env.SERVICE == 'pusher' && env.CHART_ONLY == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: pusher-chart-only-recovery-evidence-${{ github.run_id }} path: pusher-recovery-evidence.json From 1c4f7dffa7891784e96211fd7d2cb513749014ec Mon Sep 17 00:00:00 2001 From: David Zhang <9387252+Git-on-my-level@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:11:18 +0700 Subject: [PATCH 03/14] =?UTF-8?q?fix:=20address=20Codex=20review=20?= =?UTF-8?q?=E2=80=94=20normalize=20k8s=20defaults=20and=20omit-after-repai?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1: normalize() did not strip API-server/controller-populated Deployment fields (revisionHistoryLimit, restartPolicy, dnsPolicy, schedulerName, terminationMessagePath/Policy, deployment.kubernetes.io/revision annotation) that are absent from helm template output. This caused allowed_recovery_drift() to always fail when comparing a live Deployment against a rendered chart. P2: validate_redis_source() used a sentinel object() default for the missing secretKeyRef key, so a post-repair env entry (where Kubernetes omits the key entirely after Helm applies secretKeyRef: null) was incorrectly flagged as a retained Secret source. Failure-Class: none --- .../verify_pusher_chart_only_recovery.py | 47 ++++++++++++++++++- .../test_verify_pusher_chart_only_recovery.py | 25 ++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index f2236471482..83b858e33f3 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -29,6 +29,25 @@ "uid", } +# Kubernetes API-server/controller-populated Deployment fields that are absent +# from ``helm template`` output. ``normalize()`` strips them so the drift +# comparison does not spuriously fail on a healthy live Deployment. +VOLATILE_DEPLOYMENT_ANNOTATIONS = { + "deployment.kubernetes.io/revision", +} +VOLATILE_DEPLOYMENT_SPEC = { + "revisionHistoryLimit", +} +VOLATILE_POD_TEMPLATE_SPEC = { + "restartPolicy", + "dnsPolicy", + "schedulerName", +} +VOLATILE_CONTAINER_FIELDS = { + "terminationMessagePath", + "terminationMessagePolicy", +} + def load_object(path: str) -> dict[str, Any]: """Load exactly one Kubernetes object without ever serializing its data.""" @@ -101,7 +120,7 @@ def validate_redis_source(deployment: dict[str, Any], configmap: str) -> list[st expected = {"name": configmap, "key": "REDIS_DB_HOST"} if not isinstance(value_from, dict) or value_from.get("configMapKeyRef") != expected: return ["REDIS_DB_HOST must use the selected backend ConfigMap key"] - if value_from.get("secretKeyRef", object()) is not None: + if value_from.get("secretKeyRef") is not None: return ["REDIS_DB_HOST must not retain a Secret source"] return [] @@ -157,9 +176,35 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: "sessionAffinityConfig", ): spec.pop(key, None) + if result.get("kind") == "Deployment": + _strip_deployment_defaults(result) return remove_nulls(result) +def _strip_deployment_defaults(deployment: dict[str, Any]) -> None: + """Remove API-server/controller-populated fields absent from helm template.""" + metadata = deployment.get("metadata", {}) + annotations = metadata.get("annotations") + if isinstance(annotations, dict): + for key in VOLATILE_DEPLOYMENT_ANNOTATIONS: + annotations.pop(key, None) + if not annotations: + metadata.pop("annotations", None) + + spec = deployment.get("spec", {}) + for key in VOLATILE_DEPLOYMENT_SPEC: + spec.pop(key, None) + + template = spec.get("template", {}) + template_spec = template.get("spec", {}) + for key in VOLATILE_POD_TEMPLATE_SPEC: + template_spec.pop(key, None) + + for container in template_spec.get("containers", []): + for key in VOLATILE_CONTAINER_FIELDS: + container.pop(key, None) + + def remove_nulls(value: Any) -> Any: if isinstance(value, dict): return {key: remove_nulls(child) for key, child in value.items() if child is not None} diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index 8a225a6d704..49912e774bb 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -92,6 +92,25 @@ def test_recovery_profile_allows_only_exact_image_and_redis_transition(recovery: ] +def test_recovery_profile_ignores_api_server_deployment_defaults(recovery: SimpleNamespace): + """Live Deployments include controller-populated fields absent from helm template output.""" + live = deployment( + "gcr.io/project/pusher:2ae7f78", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}} + ) + # Simulate API-server/controller defaults present in kubectl get -o json but + # absent from helm template output. + live["metadata"]["annotations"] = {"deployment.kubernetes.io/revision": "3"} + live["spec"]["revisionHistoryLimit"] = 10 + live["spec"]["template"]["spec"]["restartPolicy"] = "Always" + live["spec"]["template"]["spec"]["dnsPolicy"] = "ClusterFirst" + live["spec"]["template"]["spec"]["schedulerName"] = "default-scheduler" + live["spec"]["template"]["spec"]["containers"][0]["terminationMessagePath"] = "/dev/termination-log" + live["spec"]["template"]["spec"]["containers"][0]["terminationMessagePolicy"] = "File" + + rendered = deployment(f"gcr.io/project/pusher@{DIGEST}") + assert recovery.allowed_recovery_drift(live, rendered) == [] + + def test_redis_validation_rejects_secret_and_configmap_errors(recovery: SimpleNamespace): assert recovery.validate_redis_source(deployment("x").copy(), "prod-omi-backend-config") == [] secret = deployment("x", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}}) @@ -100,6 +119,12 @@ def test_redis_validation_rejects_secret_and_configmap_errors(recovery: SimpleNa ] +def test_redis_validation_accepts_post_repair_env_without_secret_key(recovery: SimpleNamespace): + """After Helm applies secretKeyRef: null, Kubernetes stores the env without the key at all.""" + repaired = deployment("x", {"configMapKeyRef": {"name": "prod-omi-backend-config", "key": "REDIS_DB_HOST"}}) + assert recovery.validate_redis_source(repaired, "prod-omi-backend-config") == [] + + def test_target_and_readiness_guards_fail_closed(recovery: SimpleNamespace): live = deployment("x") live["status"]["observedGeneration"] = 1 From 55e9f607bfce850a25b79749bea54cceb15690cf Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 07:46:29 -0400 Subject: [PATCH 04/14] fix(pusher): normalize Helm service defaults Failure-Class: FC-runtime-image-boundary Co-authored-by: multica-agent --- .../verify_pusher_chart_only_recovery.py | 17 ++++++++++ .../test_verify_pusher_chart_only_recovery.py | 31 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index 83b858e33f3..6cfde002012 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -28,6 +28,10 @@ "selfLink", "uid", } +HELM_RELEASE_ANNOTATIONS = { + "meta.helm.sh/release-name", + "meta.helm.sh/release-namespace", +} # Kubernetes API-server/controller-populated Deployment fields that are absent # from ``helm template`` output. ``normalize()`` strips them so the drift @@ -165,6 +169,12 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: for key in VOLATILE_METADATA: metadata.pop(key, None) metadata.pop("namespace", None) + annotations = metadata.get("annotations") + if isinstance(annotations, dict): + for key in HELM_RELEASE_ANNOTATIONS: + annotations.pop(key, None) + if not annotations: + metadata.pop("annotations", None) if result.get("kind") == "Service": spec = result.get("spec", {}) for key in ( @@ -176,6 +186,13 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: "sessionAffinityConfig", ): spec.pop(key, None) + # These fields are defaulted by the Service API when the chart omits + # them. Strip only their documented defaults; non-default policies are + # meaningful drift and must remain visible to the recovery profile. + if spec.get("internalTrafficPolicy") == "Cluster": + spec.pop("internalTrafficPolicy", None) + if spec.get("sessionAffinity") == "None": + spec.pop("sessionAffinity", None) if result.get("kind") == "Deployment": _strip_deployment_defaults(result) return remove_nulls(result) diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index 49912e774bb..ed650840360 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -149,6 +149,37 @@ def test_chart_owned_resources_reject_unexpected_drift(recovery: SimpleNamespace ] +def test_service_recovery_profile_ignores_helm_and_api_defaults_but_not_real_drift(recovery: SimpleNamespace): + rendered = { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "prod-omi-pusher", + "annotations": {"cloud.google.com/neg": '{"ingress": true}'}, + }, + "spec": {"type": "ClusterIP", "ports": [{"name": "http", "port": 8080}], "selector": {"app": "pusher"}}, + } + live = copy.deepcopy(rendered) + live["metadata"]["annotations"].update( + {"meta.helm.sh/release-name": "prod-omi-pusher", "meta.helm.sh/release-namespace": "prod-omi-backend"} + ) + live["spec"].update({"clusterIP": "10.0.0.7", "internalTrafficPolicy": "Cluster", "sessionAffinity": "None"}) + + assert recovery.validate_chart_owned_resource_drift(live, rendered, "Service") == [] + + annotation_drift = copy.deepcopy(live) + annotation_drift["metadata"]["annotations"]["cloud.google.com/neg"] = '{"ingress": false}' + assert recovery.validate_chart_owned_resource_drift(annotation_drift, rendered, "Service") == [ + "recovery profile would change Service outside the allowlist" + ] + + policy_drift = copy.deepcopy(live) + policy_drift["spec"]["internalTrafficPolicy"] = "Local" + assert recovery.validate_chart_owned_resource_drift(policy_drift, rendered, "Service") == [ + "recovery profile would change Service outside the allowlist" + ] + + def test_serving_digest_is_read_only_and_rejects_mixed_or_missing_status(recovery: SimpleNamespace): pods = { "items": [ From ca5342a15a3bf9641a44156625720e8b5b69dca5 Mon Sep 17 00:00:00 2001 From: David Zhang <9387252+Git-on-my-level@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:47:44 +0700 Subject: [PATCH 05/14] fix: strip Helm annotations and Service cluster defaults in normalize() Follow-up to the k8s defaults fix: normalize() still included Helm-managed annotations (meta.helm.sh/release-*) and cluster-added Service fields (internalTrafficPolicy) that are absent from helm template output. This would cause the drift comparison to fail on any existing Helm-managed release. Moved annotation stripping to normalize() for all kinds, added internalTrafficPolicy to VOLATILE_SERVICE_SPEC, and consolidated the Service volatile set. Failure-Class: none --- .../verify_pusher_chart_only_recovery.py | 38 ++++++++++--------- .../test_verify_pusher_chart_only_recovery.py | 31 +++++++++++++++ 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index 83b858e33f3..88c0dc137e6 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -34,6 +34,8 @@ # comparison does not spuriously fail on a healthy live Deployment. VOLATILE_DEPLOYMENT_ANNOTATIONS = { "deployment.kubernetes.io/revision", + "meta.helm.sh/release-name", + "meta.helm.sh/release-namespace", } VOLATILE_DEPLOYMENT_SPEC = { "revisionHistoryLimit", @@ -48,6 +50,17 @@ "terminationMessagePolicy", } +# Cluster-added Service fields absent from ``helm template`` output. +VOLATILE_SERVICE_SPEC = { + "clusterIP", + "clusterIPs", + "healthCheckNodePort", + "internalTrafficPolicy", + "ipFamilies", + "ipFamilyPolicy", + "sessionAffinityConfig", +} + def load_object(path: str) -> dict[str, Any]: """Load exactly one Kubernetes object without ever serializing its data.""" @@ -165,16 +178,15 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: for key in VOLATILE_METADATA: metadata.pop(key, None) metadata.pop("namespace", None) + annotations = metadata.get("annotations") + if isinstance(annotations, dict): + for key in VOLATILE_DEPLOYMENT_ANNOTATIONS: + annotations.pop(key, None) + if not annotations: + metadata.pop("annotations", None) if result.get("kind") == "Service": spec = result.get("spec", {}) - for key in ( - "clusterIP", - "clusterIPs", - "healthCheckNodePort", - "ipFamilies", - "ipFamilyPolicy", - "sessionAffinityConfig", - ): + for key in VOLATILE_SERVICE_SPEC: spec.pop(key, None) if result.get("kind") == "Deployment": _strip_deployment_defaults(result) @@ -182,15 +194,7 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: def _strip_deployment_defaults(deployment: dict[str, Any]) -> None: - """Remove API-server/controller-populated fields absent from helm template.""" - metadata = deployment.get("metadata", {}) - annotations = metadata.get("annotations") - if isinstance(annotations, dict): - for key in VOLATILE_DEPLOYMENT_ANNOTATIONS: - annotations.pop(key, None) - if not annotations: - metadata.pop("annotations", None) - + """Remove Deployment-specific API-server/controller defaults absent from helm template.""" spec = deployment.get("spec", {}) for key in VOLATILE_DEPLOYMENT_SPEC: spec.pop(key, None) diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index 49912e774bb..0c3e6ee18dd 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -111,6 +111,37 @@ def test_recovery_profile_ignores_api_server_deployment_defaults(recovery: Simpl assert recovery.allowed_recovery_drift(live, rendered) == [] +def test_recovery_profile_ignores_helm_annotations_and_service_defaults(recovery: SimpleNamespace): + """Helm-managed annotations and cluster-added Service fields are absent from helm template.""" + live = deployment( + "gcr.io/project/pusher:2ae7f78", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}} + ) + live["metadata"]["annotations"] = { + "deployment.kubernetes.io/revision": "3", + "meta.helm.sh/release-name": "prod-omi-pusher", + "meta.helm.sh/release-namespace": "prod-omi-backend", + } + rendered = deployment(f"gcr.io/project/pusher@{DIGEST}") + assert recovery.allowed_recovery_drift(live, rendered) == [] + + # Service drift comparison should also ignore cluster-added defaults. + live_svc = { + "kind": "Service", + "metadata": { + "name": "prod-omi-pusher", + "annotations": {"meta.helm.sh/release-name": "prod-omi-pusher"}, + }, + "spec": { + "clusterIP": "10.0.0.1", + "clusterIPs": ["10.0.0.1"], + "internalTrafficPolicy": "Cluster", + "sessionAffinityConfig": {}, + }, + } + rendered_svc = {"kind": "Service", "metadata": {"name": "prod-omi-pusher"}, "spec": {}} + assert recovery.validate_chart_owned_resource_drift(live_svc, rendered_svc, "Service") == [] + + def test_redis_validation_rejects_secret_and_configmap_errors(recovery: SimpleNamespace): assert recovery.validate_redis_source(deployment("x").copy(), "prod-omi-backend-config") == [] secret = deployment("x", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}}) From c185d68cd7c3a9155c2154652f8624b366aa8c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Carter=20=E7=A5=81=E6=98=8E=E6=80=9D?= Date: Wed, 22 Jul 2026 04:55:27 -0400 Subject: [PATCH 06/14] fix(backend): update workflow contract triggers and smoke_what_matters_now assertion - Add missing desktop_rollback_beta.yml and desktop_breakglass_rollout_beta.yml to backend-unit-tests.yml paths triggers. - Update test_smoke_what_matters_now.py assertion to match $DEPLOY_CONTROL_SCRIPTS path prefix in gcp_backend.yml. Failure-Class: none Co-authored-by: multica-agent --- .github/workflows/backend-unit-tests.yml | 4 ++++ backend/tests/unit/test_smoke_what_matters_now.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backend-unit-tests.yml b/.github/workflows/backend-unit-tests.yml index 3e830a6d4ec..8000c2cbff9 100644 --- a/.github/workflows/backend-unit-tests.yml +++ b/.github/workflows/backend-unit-tests.yml @@ -23,6 +23,8 @@ on: - '.github/workflows/repo-checks.yml' - '.github/workflows/release-eligibility.yml' - '.github/workflows/desktop_promote_beta.yml' + - '.github/workflows/desktop_rollback_beta.yml' + - '.github/workflows/desktop_breakglass_rollout_beta.yml' - '.github/workflows/desktop_promote_prod.yml' - '.github/workflows/gcp_backend.yml' - '.github/workflows/gcp_backend_auto_dev.yml' @@ -63,6 +65,8 @@ on: - '.github/workflows/repo-checks.yml' - '.github/workflows/release-eligibility.yml' - '.github/workflows/desktop_promote_beta.yml' + - '.github/workflows/desktop_rollback_beta.yml' + - '.github/workflows/desktop_breakglass_rollout_beta.yml' - '.github/workflows/desktop_promote_prod.yml' - '.github/workflows/gcp_backend.yml' - '.github/workflows/gcp_backend_auto_dev.yml' diff --git a/backend/tests/unit/test_smoke_what_matters_now.py b/backend/tests/unit/test_smoke_what_matters_now.py index 19a49bd6e27..1dc0ea568c3 100644 --- a/backend/tests/unit/test_smoke_what_matters_now.py +++ b/backend/tests/unit/test_smoke_what_matters_now.py @@ -127,4 +127,7 @@ def test_manual_development_smoke_keeps_its_existing_external_hostname_path(): assert 'OMI_TASK_INTELLIGENCE_SMOKE_UID' not in workflow assert '--secret=ADMIN_KEY' in workflow - assert 'smoke_what_matters_now.py --base-url https://api.omi.dev' in workflow + assert ( + 'smoke_what_matters_now.py" --base-url https://api.omi.dev' in workflow + or 'smoke_what_matters_now.py --base-url https://api.omi.dev' in workflow + ) From 8f759f921e31eb9e6edef825f47624d69bf594dc Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 08:12:05 -0400 Subject: [PATCH 07/14] fix(pusher): close chart-only recovery race gates Failure-Class: FC-runtime-image-boundary Co-authored-by: multica-agent --- .github/workflows/gcp_backend_pusher.yml | 9 +++-- .../verify_pusher_chart_only_recovery.py | 29 +++++++++++++- .../test_verify_pusher_chart_only_recovery.py | 40 +++++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gcp_backend_pusher.yml b/.github/workflows/gcp_backend_pusher.yml index 92bc35e89bf..70ea87fb164 100644 --- a/.github/workflows/gcp_backend_pusher.yml +++ b/.github/workflows/gcp_backend_pusher.yml @@ -354,6 +354,9 @@ jobs: RELEASE="${{ vars.ENV }}-omi-pusher" kubectl -n "$NAMESPACE" get deployment "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-deployment.json kubectl -n "$NAMESPACE" get configmap "${{ vars.ENV }}-omi-backend-config" -o json > .pusher-recovery-snapshot/preapply-configmap.json + kubectl -n "$NAMESPACE" get service "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-service.json + kubectl -n "$NAMESPACE" get hpa "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-hpa.json + kubectl -n "$NAMESPACE" get pdb "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-pdb.json kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/instance="$RELEASE" -o json > .pusher-recovery-snapshot/preapply-pods.json kubectl -n "$NAMESPACE" get ingress "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-ingress.json kubectl -n "$NAMESPACE" get serviceaccount "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-serviceaccount.json @@ -364,9 +367,9 @@ jobs: --digest "${{ github.event.inputs.image_digest }}" --ready-baseline "${{ github.event.inputs.ready_baseline }}" \ --live-deployment .pusher-recovery-snapshot/preapply-deployment.json \ --live-configmap .pusher-recovery-snapshot/preapply-configmap.json \ - --live-service .pusher-recovery-snapshot/service.json \ - --live-hpa .pusher-recovery-snapshot/hpa.json \ - --live-pdb .pusher-recovery-snapshot/pdb.json \ + --live-service .pusher-recovery-snapshot/preapply-service.json \ + --live-hpa .pusher-recovery-snapshot/preapply-hpa.json \ + --live-pdb .pusher-recovery-snapshot/preapply-pdb.json \ --live-ingress .pusher-recovery-snapshot/preapply-ingress.json \ --live-serviceaccount .pusher-recovery-snapshot/preapply-serviceaccount.json \ --live-backendconfig .pusher-recovery-snapshot/preapply-backendconfig.json \ diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index b7d09b99c00..9ab1330e052 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -248,7 +248,9 @@ def strategic_merge(live: Any, desired: Any) -> Any: return merged -def allowed_recovery_drift(live: dict[str, Any], rendered: dict[str, Any]) -> list[str]: +def allowed_recovery_drift( + live: dict[str, Any], rendered: dict[str, Any], *, autoscaling_enabled: bool = False +) -> list[str]: """Compare chart-owned objects after replacing only image and REDIS source.""" adapted = copy.deepcopy(live) try: @@ -258,6 +260,11 @@ def allowed_recovery_drift(live: dict[str, Any], rendered: dict[str, Any]) -> li live_container["env"] = replace_env_by_name(live_container.get("env", []), redis_entry(rendered)) except ValueError as exc: return [str(exc)] + # The HPA is the replica-count authority. Helm intentionally omits the + # Deployment replica count when that HPA is present; only discard the + # live controller value in that exact case. Never strip a rendered value. + if autoscaling_enabled: + adapted.get("spec", {}).pop("replicas", None) return ( [] if normalize(adapted) == normalize(rendered) @@ -265,6 +272,12 @@ def allowed_recovery_drift(live: dict[str, Any], rendered: dict[str, Any]) -> li ) +def hpa_controls_deployment(hpa: dict[str, Any], deployment_name: str) -> bool: + """Return true only for an apps/v1 HPA targetting this exact Deployment.""" + target = hpa.get("spec", {}).get("scaleTargetRef") + return target == {"apiVersion": "apps/v1", "kind": "Deployment", "name": deployment_name} + + def validate_chart_owned_resource_drift(live: dict[str, Any], rendered: dict[str, Any], kind: str) -> list[str]: """Require Service/HPA/PDB identity and spec to remain unchanged.""" return ( @@ -354,7 +367,19 @@ def preflight(args: argparse.Namespace) -> list[str]: except ValueError as exc: failures.append(str(exc)) failures += validate_redis_source(rendered, configmap_name) - failures += allowed_recovery_drift(live, rendered) + autoscaling_enabled = False + if args.live_hpa: + try: + live_hpa = load_object(args.live_hpa) + rendered_hpa = rendered_resource(args.rendered, "HorizontalPodAutoscaler", release) + autoscaling_enabled = hpa_controls_deployment(live_hpa, deployment_name) and hpa_controls_deployment( + rendered_hpa, deployment_name + ) + except ValueError: + # The normal chart-owned resource validation below reports the + # malformed or missing HPA. Keep Deployment replicas fail-closed. + pass + failures += allowed_recovery_drift(live, rendered, autoscaling_enabled=autoscaling_enabled) if not args.live_pods: failures.append("recovery preflight is missing live Pusher pod image-status evidence") else: diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index 31e39bcb79e..73ab92ed244 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -92,6 +92,32 @@ def test_recovery_profile_allows_only_exact_image_and_redis_transition(recovery: ] +def test_recovery_profile_strips_only_hpa_owned_live_replicas(recovery: SimpleNamespace): + live = deployment( + "gcr.io/project/pusher:2ae7f78", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}} + ) + live["spec"]["replicas"] = 7 + rendered = deployment(f"gcr.io/project/pusher@{DIGEST}") + + assert recovery.allowed_recovery_drift(live, rendered, autoscaling_enabled=True) == [] + assert recovery.allowed_recovery_drift(live, rendered, autoscaling_enabled=False) == [ + "recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition" + ] + + rendered["spec"]["replicas"] = 8 + assert recovery.allowed_recovery_drift(live, rendered, autoscaling_enabled=True) == [ + "recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition" + ] + + +def test_hpa_replica_normalization_requires_the_exact_deployment_target(recovery: SimpleNamespace): + hpa = {"spec": {"scaleTargetRef": {"apiVersion": "apps/v1", "kind": "Deployment", "name": "prod-omi-pusher"}}} + assert recovery.hpa_controls_deployment(hpa, "prod-omi-pusher") + + hpa["spec"]["scaleTargetRef"]["name"] = "other-deployment" + assert not recovery.hpa_controls_deployment(hpa, "prod-omi-pusher") + + def test_recovery_profile_ignores_api_server_deployment_defaults(recovery: SimpleNamespace): """Live Deployments include controller-populated fields absent from helm template output.""" live = deployment( @@ -252,6 +278,20 @@ def test_chart_only_workflow_skips_build_push_and_normal_paths_stay_available(): assert "--expected-evidence pusher-recovery-evidence.json" in workflow +def test_chart_only_preapply_gate_recaptures_all_chart_owned_resources(): + workflow = (SCRIPT.parents[2] / ".github/workflows/gcp_backend_pusher.yml").read_text(encoding="utf-8") + preapply = workflow.split("# Re-read the live objects immediately before mutation.", 1)[1].split( + 'helm -n "$NAMESPACE" upgrade', 1 + )[0] + + for kind, snapshot in (("service", "service"), ("hpa", "hpa"), ("pdb", "pdb")): + assert ( + f'kubectl -n "$NAMESPACE" get {kind} "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-{snapshot}.json' + in preapply + ) + assert f"--live-{kind} .pusher-recovery-snapshot/preapply-{snapshot}.json" in preapply + + def test_chart_renders_exact_digest_and_rejects_ambiguous_or_mutated_repository(): chart = SCRIPT.parents[1] / "charts" / "pusher" values = chart / "prod_omi_pusher_values.yaml" From 6a4d6be9caaf51880e7bbbb301a07b957c213955 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 08:29:56 -0400 Subject: [PATCH 08/14] fix(pusher): verify chart-only live references Failure-Class: FC-runtime-image-boundary Co-authored-by: multica-agent --- .github/workflows/gcp_backend_pusher.yml | 4 + .../verify_pusher_chart_only_recovery.py | 57 ++++++++++++ .../verify_pusher_config_references.py | 14 ++- .../test_verify_pusher_chart_only_recovery.py | 90 +++++++++++++++++++ .../test_verify_pusher_config_references.py | 40 +++++++++ 5 files changed, 202 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gcp_backend_pusher.yml b/.github/workflows/gcp_backend_pusher.yml index 70ea87fb164..8520f0a01f8 100644 --- a/.github/workflows/gcp_backend_pusher.yml +++ b/.github/workflows/gcp_backend_pusher.yml @@ -288,6 +288,10 @@ jobs: --set-string 'image.tag=' \ --set-string "image.digest=${{ github.event.inputs.image_digest }}" \ > .pusher-recovery-snapshot/rendered.yaml + python3 -m pip install -q pyyaml + python3 backend/scripts/verify_pusher_config_references.py \ + --environment "${{ vars.ENV }}" --namespace "$NAMESPACE" \ + --rendered .pusher-recovery-snapshot/rendered.yaml python3 backend/scripts/verify_pusher_chart_only_recovery.py preflight \ --environment "${{ vars.ENV }}" --namespace "$NAMESPACE" --release "$RELEASE" --deployment "$DEPLOYMENT" \ --repository "${{ github.event.inputs.image_repository }}" --expected-repository "gcr.io/${{ vars.GCP_PROJECT_ID }}/pusher" \ diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index 9ab1330e052..e329d61b6d6 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -57,6 +57,29 @@ "ipFamilies", "ipFamilyPolicy", } +GKE_CONTROLLER_STATUS_ANNOTATIONS = { + "Service": {"cloud.google.com/neg-status"}, + "Ingress": { + "ingress.kubernetes.io/backends", + "ingress.kubernetes.io/forwarding-rule", + "ingress.kubernetes.io/target-proxy", + "ingress.kubernetes.io/url-map", + }, +} +GKE_INGRESS_FINALIZERS = { + "networking.gke.io/ingress-finalizer", + "networking.gke.io/ingress-finalizer-V2", +} +POD_SECURITY_DEFAULTS = { + "fsGroupChangePolicy": "Always", + "supplementalGroupsPolicy": "Merge", +} +CONTAINER_SECURITY_DEFAULTS = { + "allowPrivilegeEscalation": True, + "privileged": False, + "readOnlyRootFilesystem": False, + "runAsNonRoot": False, +} def load_object(path: str) -> dict[str, Any]: @@ -179,8 +202,16 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: if isinstance(annotations, dict): for key in VOLATILE_DEPLOYMENT_ANNOTATIONS: annotations.pop(key, None) + for key in GKE_CONTROLLER_STATUS_ANNOTATIONS.get(result.get("kind"), set()): + annotations.pop(key, None) if not annotations: metadata.pop("annotations", None) + if result.get("kind") == "Ingress": + finalizers = metadata.get("finalizers") + if isinstance(finalizers, list): + metadata["finalizers"] = [item for item in finalizers if item not in GKE_INGRESS_FINALIZERS] + if not metadata["finalizers"]: + metadata.pop("finalizers", None) if result.get("kind") == "Service": spec = result.get("spec", {}) for key in VOLATILE_SERVICE_SPEC: @@ -204,15 +235,41 @@ def _strip_deployment_defaults(deployment: dict[str, Any]) -> None: spec = deployment.get("spec", {}) for key in VOLATILE_DEPLOYMENT_SPEC: spec.pop(key, None) + if spec.get("minReadySeconds") == 0: + spec.pop("minReadySeconds", None) template = spec.get("template", {}) template_spec = template.get("spec", {}) for key in VOLATILE_POD_TEMPLATE_SPEC: template_spec.pop(key, None) + if template_spec.get("serviceAccount") == template_spec.get("serviceAccountName"): + template_spec.pop("serviceAccount", None) + _strip_default_security_context(template_spec, "securityContext", POD_SECURITY_DEFAULTS) for container in template_spec.get("containers", []): for key in VOLATILE_CONTAINER_FIELDS: container.pop(key, None) + _strip_default_security_context(container, "securityContext", CONTAINER_SECURITY_DEFAULTS) + for probe_name in ("livenessProbe", "readinessProbe", "startupProbe"): + probe = container.get(probe_name) + if not isinstance(probe, dict): + continue + http_get = probe.get("httpGet") + if isinstance(http_get, dict) and http_get.get("scheme") == "HTTP": + http_get.pop("scheme", None) + if probe.get("successThreshold") == 1: + probe.pop("successThreshold", None) + + +def _strip_default_security_context(parent: dict[str, Any], field: str, defaults: dict[str, Any]) -> None: + context = parent.get(field) + if not isinstance(context, dict): + return + for key, value in defaults.items(): + if context.get(key) == value: + context.pop(key, None) + if not context: + parent.pop(field, None) def remove_nulls(value: Any) -> Any: diff --git a/backend/scripts/verify_pusher_config_references.py b/backend/scripts/verify_pusher_config_references.py index c513df548e3..c0758c25fd9 100644 --- a/backend/scripts/verify_pusher_config_references.py +++ b/backend/scripts/verify_pusher_config_references.py @@ -78,8 +78,13 @@ def render(environment: str, image_tag: str = "contract-test") -> list[dict[str, return [doc for doc in yaml.safe_load_all(result.stdout) if isinstance(doc, dict)] -def rendered_pusher_deployment(environment: str) -> dict[str, Any]: - rendered = render(environment) +def rendered_pusher_deployment(environment: str, rendered_path: str | None = None) -> dict[str, Any]: + if rendered_path: + rendered = [ + doc for doc in yaml.safe_load_all(Path(rendered_path).read_text(encoding="utf-8")) if isinstance(doc, dict) + ] + else: + rendered = render(environment) deployments = [doc for doc in rendered if doc.get("kind") == "Deployment"] if len(deployments) != 1: raise RuntimeError("pusher render did not contain exactly one Deployment") @@ -292,8 +297,11 @@ def main() -> int: parser.add_argument("--environment", required=True, choices=sorted(ENVIRONMENTS)) parser.add_argument("--namespace", required=True) parser.add_argument("--render-only", action="store_true") + parser.add_argument( + "--rendered", help="exact chart-only Helm render to validate without exposing referenced values" + ) args = parser.parse_args() - deployment = rendered_pusher_deployment(args.environment) + deployment = rendered_pusher_deployment(args.environment, args.rendered) refs = pusher_references(args.environment, deployment) failures = validate_dev_pusher_binding_contract(deployment) if args.environment == "dev" else [] if not args.render_only and not failures: diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index 73ab92ed244..f78adfa8dec 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -137,6 +137,45 @@ def test_recovery_profile_ignores_api_server_deployment_defaults(recovery: Simpl assert recovery.allowed_recovery_drift(live, rendered) == [] +def test_recovery_profile_ignores_only_deployment_api_defaults(recovery: SimpleNamespace): + live = deployment( + "gcr.io/project/pusher:2ae7f78", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}} + ) + rendered = deployment(f"gcr.io/project/pusher@{DIGEST}") + rendered["spec"]["template"]["spec"]["serviceAccountName"] = "prod-omi-pusher" + live["spec"]["minReadySeconds"] = 0 + live["spec"]["template"]["spec"].update( + { + "serviceAccountName": "prod-omi-pusher", + "serviceAccount": "prod-omi-pusher", + "securityContext": {"fsGroupChangePolicy": "Always", "supplementalGroupsPolicy": "Merge"}, + } + ) + container = live["spec"]["template"]["spec"]["containers"][0] + container["securityContext"] = { + "allowPrivilegeEscalation": True, + "privileged": False, + "readOnlyRootFilesystem": False, + "runAsNonRoot": False, + } + container["readinessProbe"]["httpGet"]["scheme"] = "HTTP" + container["readinessProbe"]["successThreshold"] = 1 + + assert recovery.allowed_recovery_drift(live, rendered) == [] + + resource_drift = copy.deepcopy(live) + resource_drift["spec"]["template"]["spec"]["containers"][0]["resources"]["requests"]["cpu"] = "2" + assert recovery.allowed_recovery_drift(resource_drift, rendered) == [ + "recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition" + ] + + probe_drift = copy.deepcopy(live) + probe_drift["spec"]["template"]["spec"]["containers"][0]["readinessProbe"]["httpGet"]["path"] = "/other" + assert recovery.allowed_recovery_drift(probe_drift, rendered) == [ + "recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition" + ] + + def test_recovery_profile_ignores_helm_annotations_and_service_defaults(recovery: SimpleNamespace): """Helm-managed annotations and cluster-added Service fields are absent from helm template.""" live = deployment( @@ -168,6 +207,55 @@ def test_recovery_profile_ignores_helm_annotations_and_service_defaults(recovery assert recovery.validate_chart_owned_resource_drift(live_svc, rendered_svc, "Service") == [] +def test_chart_owned_resources_ignore_only_gke_status_metadata(recovery: SimpleNamespace): + rendered_service = { + "kind": "Service", + "metadata": { + "name": "dev-omi-pusher", + "annotations": { + "cloud.google.com/backend-config": '{"default":"dev-pusher-backend-config"}', + "cloud.google.com/neg": '{"ingress":true}', + }, + }, + "spec": {"selector": {"app": "pusher"}}, + } + live_service = copy.deepcopy(rendered_service) + live_service["metadata"]["annotations"]["cloud.google.com/neg-status"] = '{"network_endpoint_groups":{}}' + assert recovery.validate_chart_owned_resource_drift(live_service, rendered_service, "Service") == [] + + service_policy_drift = copy.deepcopy(live_service) + service_policy_drift["metadata"]["annotations"]["cloud.google.com/neg"] = '{"ingress":false}' + assert recovery.validate_chart_owned_resource_drift(service_policy_drift, rendered_service, "Service") == [ + "recovery profile would change Service outside the allowlist" + ] + + rendered_ingress = { + "kind": "Ingress", + "metadata": { + "name": "dev-omi-pusher", + "annotations": {"kubernetes.io/ingress.class": "gce-internal"}, + }, + "spec": {"rules": []}, + } + live_ingress = copy.deepcopy(rendered_ingress) + live_ingress["metadata"].update({"finalizers": ["networking.gke.io/ingress-finalizer-V2"]}) + live_ingress["metadata"]["annotations"].update( + { + "ingress.kubernetes.io/backends": "{}", + "ingress.kubernetes.io/forwarding-rule": "k8s2-fr", + "ingress.kubernetes.io/target-proxy": "k8s2-tp", + "ingress.kubernetes.io/url-map": "k8s2-um", + } + ) + assert recovery.validate_chart_owned_resource_drift(live_ingress, rendered_ingress, "Ingress") == [] + + ingress_policy_drift = copy.deepcopy(live_ingress) + ingress_policy_drift["metadata"]["annotations"]["kubernetes.io/ingress.class"] = "gce" + assert recovery.validate_chart_owned_resource_drift(ingress_policy_drift, rendered_ingress, "Ingress") == [ + "recovery profile would change Ingress outside the allowlist" + ] + + def test_redis_validation_rejects_secret_and_configmap_errors(recovery: SimpleNamespace): assert recovery.validate_redis_source(deployment("x").copy(), "prod-omi-backend-config") == [] secret = deployment("x", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}}) @@ -276,6 +364,8 @@ def test_chart_only_workflow_skips_build_push_and_normal_paths_stay_available(): assert "if: env.SERVICE == 'pusher' && env.CHART_ONLY != 'true'" in workflow assert "if: env.SERVICE == 'pusher' && env.CHART_ONLY == 'true'" in workflow assert "--expected-evidence pusher-recovery-evidence.json" in workflow + assert "verify_pusher_config_references.py" in workflow + assert "--rendered .pusher-recovery-snapshot/rendered.yaml" in workflow def test_chart_only_preapply_gate_recaptures_all_chart_owned_resources(): diff --git a/backend/tests/unit/test_verify_pusher_config_references.py b/backend/tests/unit/test_verify_pusher_config_references.py index af813b18622..552ff5991e3 100644 --- a/backend/tests/unit/test_verify_pusher_config_references.py +++ b/backend/tests/unit/test_verify_pusher_config_references.py @@ -32,6 +32,46 @@ def test_extracts_object_and_key_names_without_values(preflight: SimpleNamespace assert refs == {("secret", "safe-name", "KEY")} +def test_exact_chart_only_render_extracts_every_configmap_key(tmp_path: Path, preflight: SimpleNamespace): + rendered = tmp_path / "rendered.yaml" + rendered.write_text( + """\ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dev-omi-pusher +spec: + template: + spec: + containers: + - name: pusher + env: + - name: REDIS_DB_HOST + valueFrom: + configMapKeyRef: + name: dev-omi-backend-config + key: REDIS_DB_HOST + - name: TYPESENSE_HOST + valueFrom: + configMapKeyRef: + name: dev-omi-backend-config + key: TYPESENSE_HOST + - name: API_KEY + valueFrom: + secretKeyRef: + name: dev-omi-backend-secrets + key: API_KEY +""", + encoding="utf-8", + ) + + deployment = preflight.rendered_pusher_deployment("dev", str(rendered)) + assert { + ("configmap", "dev-omi-backend-config", "REDIS_DB_HOST"), + ("configmap", "dev-omi-backend-config", "TYPESENSE_HOST"), + }.issubset(preflight.pusher_references("dev", deployment)) + + def test_rejects_missing_required_configmap_fixture(monkeypatch, preflight: SimpleNamespace): monkeypatch.setitem( preflight.pusher_references.__globals__, From 5ca9810b6c098df10bec8cdb62ba6a2d04e4fbe3 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 08:35:56 -0400 Subject: [PATCH 09/14] fix(pusher): recheck chart references before Helm Failure-Class: FC-runtime-image-boundary Co-authored-by: multica-agent --- .github/workflows/gcp_backend_pusher.yml | 3 +++ backend/tests/unit/test_verify_pusher_chart_only_recovery.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/gcp_backend_pusher.yml b/.github/workflows/gcp_backend_pusher.yml index 8520f0a01f8..8e314b8d69f 100644 --- a/.github/workflows/gcp_backend_pusher.yml +++ b/.github/workflows/gcp_backend_pusher.yml @@ -365,6 +365,9 @@ jobs: kubectl -n "$NAMESPACE" get ingress "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-ingress.json kubectl -n "$NAMESPACE" get serviceaccount "$RELEASE" -o json > .pusher-recovery-snapshot/preapply-serviceaccount.json kubectl -n "$NAMESPACE" get backendconfig "${{ vars.ENV }}-pusher-backend-config" -o json > .pusher-recovery-snapshot/preapply-backendconfig.json + python3 backend/scripts/verify_pusher_config_references.py \ + --environment "${{ vars.ENV }}" --namespace "$NAMESPACE" \ + --rendered .pusher-recovery-snapshot/rendered.yaml python3 backend/scripts/verify_pusher_chart_only_recovery.py preflight \ --environment "${{ vars.ENV }}" --namespace "$NAMESPACE" --release "$RELEASE" --deployment "$RELEASE" \ --repository "${{ github.event.inputs.image_repository }}" --expected-repository "gcr.io/${{ vars.GCP_PROJECT_ID }}/pusher" \ diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index f78adfa8dec..ed6139d0425 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -381,6 +381,11 @@ def test_chart_only_preapply_gate_recaptures_all_chart_owned_resources(): ) assert f"--live-{kind} .pusher-recovery-snapshot/preapply-{snapshot}.json" in preapply + # The final mutation gate must re-check every rendered ConfigMap/Secret + # key reference, not only the Redis key covered by recovery preflight. + assert "verify_pusher_config_references.py" in preapply + assert "--rendered .pusher-recovery-snapshot/rendered.yaml" in preapply + def test_chart_renders_exact_digest_and_rejects_ambiguous_or_mutated_repository(): chart = SCRIPT.parents[1] / "charts" / "pusher" From e7538a31b33355423c00ba8f022ab679621cfd2e Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 08:39:34 -0400 Subject: [PATCH 10/14] fix(pusher): preserve chart annotation drift Failure-Class: FC-runtime-image-boundary Co-authored-by: multica-agent --- .../verify_pusher_chart_only_recovery.py | 11 +++----- .../test_verify_pusher_chart_only_recovery.py | 26 ++++++++++++------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index e329d61b6d6..9cf665d0873 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -31,11 +31,7 @@ # Kubernetes API-server/controller-populated Deployment fields that are absent # from ``helm template`` output. ``normalize()`` strips them so the drift # comparison does not spuriously fail on a healthy live Deployment. -VOLATILE_DEPLOYMENT_ANNOTATIONS = { - "deployment.kubernetes.io/revision", - "meta.helm.sh/release-name", - "meta.helm.sh/release-namespace", -} +VOLATILE_DEPLOYMENT_ANNOTATIONS = {"deployment.kubernetes.io/revision"} VOLATILE_DEPLOYMENT_SPEC = { "revisionHistoryLimit", } @@ -200,8 +196,9 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: metadata.pop("namespace", None) annotations = metadata.get("annotations") if isinstance(annotations, dict): - for key in VOLATILE_DEPLOYMENT_ANNOTATIONS: - annotations.pop(key, None) + if result.get("kind") == "Deployment": + for key in VOLATILE_DEPLOYMENT_ANNOTATIONS: + annotations.pop(key, None) for key in GKE_CONTROLLER_STATUS_ANNOTATIONS.get(result.get("kind"), set()): annotations.pop(key, None) if not annotations: diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index ed6139d0425..fd49e939e55 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -176,15 +176,13 @@ def test_recovery_profile_ignores_only_deployment_api_defaults(recovery: SimpleN ] -def test_recovery_profile_ignores_helm_annotations_and_service_defaults(recovery: SimpleNamespace): - """Helm-managed annotations and cluster-added Service fields are absent from helm template.""" +def test_recovery_profile_ignores_deployment_and_service_defaults(recovery: SimpleNamespace): + """Only API/controller defaults absent from helm template are ignored.""" live = deployment( "gcr.io/project/pusher:2ae7f78", {"secretKeyRef": {"name": "prod-omi-backend-secrets", "key": "REDIS_DB_HOST"}} ) live["metadata"]["annotations"] = { "deployment.kubernetes.io/revision": "3", - "meta.helm.sh/release-name": "prod-omi-pusher", - "meta.helm.sh/release-namespace": "prod-omi-backend", } rendered = deployment(f"gcr.io/project/pusher@{DIGEST}") assert recovery.allowed_recovery_drift(live, rendered) == [] @@ -194,7 +192,6 @@ def test_recovery_profile_ignores_helm_annotations_and_service_defaults(recovery "kind": "Service", "metadata": { "name": "prod-omi-pusher", - "annotations": {"meta.helm.sh/release-name": "prod-omi-pusher"}, }, "spec": { "clusterIP": "10.0.0.1", @@ -294,7 +291,7 @@ def test_chart_owned_resources_reject_unexpected_drift(recovery: SimpleNamespace ] -def test_service_recovery_profile_ignores_helm_and_api_defaults_but_not_real_drift(recovery: SimpleNamespace): +def test_service_recovery_profile_ignores_api_defaults_but_not_real_drift(recovery: SimpleNamespace): rendered = { "apiVersion": "v1", "kind": "Service", @@ -305,9 +302,6 @@ def test_service_recovery_profile_ignores_helm_and_api_defaults_but_not_real_dri "spec": {"type": "ClusterIP", "ports": [{"name": "http", "port": 8080}], "selector": {"app": "pusher"}}, } live = copy.deepcopy(rendered) - live["metadata"]["annotations"].update( - {"meta.helm.sh/release-name": "prod-omi-pusher", "meta.helm.sh/release-namespace": "prod-omi-backend"} - ) live["spec"].update({"clusterIP": "10.0.0.7", "internalTrafficPolicy": "Cluster", "sessionAffinity": "None"}) assert recovery.validate_chart_owned_resource_drift(live, rendered, "Service") == [] @@ -318,6 +312,20 @@ def test_service_recovery_profile_ignores_helm_and_api_defaults_but_not_real_dri "recovery profile would change Service outside the allowlist" ] + helm_annotation_drift = copy.deepcopy(live) + helm_annotation_drift["metadata"]["annotations"]["meta.helm.sh/release-name"] = "unexpected-release" + assert recovery.validate_chart_owned_resource_drift(helm_annotation_drift, rendered, "Service") == [ + "recovery profile would change Service outside the allowlist" + ] + + desired_annotation = copy.deepcopy(rendered) + desired_annotation["metadata"]["annotations"]["meta.helm.sh/release-name"] = "desired-release" + conflicting_annotation = copy.deepcopy(desired_annotation) + conflicting_annotation["metadata"]["annotations"]["meta.helm.sh/release-name"] = "live-release" + assert recovery.validate_chart_owned_resource_drift(conflicting_annotation, desired_annotation, "Service") == [ + "recovery profile would change Service outside the allowlist" + ] + policy_drift = copy.deepcopy(live) policy_drift["spec"]["internalTrafficPolicy"] = "Local" assert recovery.validate_chart_owned_resource_drift(policy_drift, rendered, "Service") == [ From 58432289ebb788622411b354da0df2039d3e019e Mon Sep 17 00:00:00 2001 From: David Zhang <9387252+Git-on-my-level@users.noreply.github.com> Date: Wed, 22 Jul 2026 19:57:27 +0700 Subject: [PATCH 11/14] fix(pusher): retain rollback recovery evidence Failure-Class: none --- .github/workflows/gcp_backend_pusher.yml | 42 +++++++++++++++- .../verify_pusher_chart_only_recovery.py | 50 ++++++++++++++++--- .../test_verify_pusher_chart_only_recovery.py | 20 ++++++++ 3 files changed, 104 insertions(+), 8 deletions(-) diff --git a/.github/workflows/gcp_backend_pusher.yml b/.github/workflows/gcp_backend_pusher.yml index 8e314b8d69f..656b77f3554 100644 --- a/.github/workflows/gcp_backend_pusher.yml +++ b/.github/workflows/gcp_backend_pusher.yml @@ -307,13 +307,53 @@ jobs: --live-pods .pusher-recovery-snapshot/pods.json \ --rendered .pusher-recovery-snapshot/rendered.yaml \ --evidence pusher-recovery-evidence.json + python3 - <<'PY' + import json + from pathlib import Path + + snapshot = Path(".pusher-recovery-snapshot") + + def load_json(name): + return json.loads((snapshot / name).read_text(encoding="utf-8")) + + def object_identity(obj): + metadata = obj.get("metadata", {}) + return { + "kind": obj.get("kind"), + "name": metadata.get("name"), + "namespace": metadata.get("namespace"), + "resourceVersion": metadata.get("resourceVersion"), + "uid": metadata.get("uid"), + "generation": metadata.get("generation"), + } + + pods = load_json("pods.json") + replicasets = load_json("replicasets.json") + helm_history = load_json("helm-history.json") + rollback = { + "helm_history": [ + {key: item.get(key) for key in ("revision", "updated", "status", "chart", "app_version", "description")} + for item in helm_history + ], + "deployment": object_identity(load_json("deployment.json")), + "replicasets": [object_identity(item) for item in replicasets.get("items", [])], + "ready_pods": [ + object_identity(item) + for item in pods.get("items", []) + if any(status.get("name") == "pusher" and status.get("ready") for status in item.get("status", {}).get("containerStatuses", [])) + ], + } + Path("pusher-recovery-rollback-identity.json").write_text(json.dumps(rollback, indent=2) + "\n", encoding="utf-8") + PY - name: Upload redacted chart-only recovery evidence if: always() && env.SERVICE == 'pusher' && env.CHART_ONLY == 'true' uses: actions/upload-artifact@v7 with: name: pusher-chart-only-recovery-evidence-${{ github.run_id }} - path: pusher-recovery-evidence.json + path: | + pusher-recovery-evidence.json + pusher-recovery-rollback-identity.json if-no-files-found: ignore - name: Deploy Pusher to GKE cluster using Helm diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index 9cf665d0873..dd0a434968e 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -186,7 +186,9 @@ def ready_replicas(deployment: dict[str, Any]) -> int: return value if isinstance(value, int) else 0 -def normalize(obj: dict[str, Any]) -> dict[str, Any]: +def normalize( + obj: dict[str, Any], *, helm_release: str | None = None, helm_namespace: str | None = None +) -> dict[str, Any]: result = copy.deepcopy(obj) result.pop("status", None) metadata = result.get("metadata") @@ -201,6 +203,7 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: annotations.pop(key, None) for key in GKE_CONTROLLER_STATUS_ANNOTATIONS.get(result.get("kind"), set()): annotations.pop(key, None) + _strip_expected_helm_annotations(annotations, helm_release=helm_release, helm_namespace=helm_namespace) if not annotations: metadata.pop("annotations", None) if result.get("kind") == "Ingress": @@ -227,6 +230,19 @@ def normalize(obj: dict[str, Any]) -> dict[str, Any]: return remove_nulls(result) +def _strip_expected_helm_annotations( + annotations: dict[str, Any], *, helm_release: str | None = None, helm_namespace: str | None = None +) -> None: + """Strip Helm ownership annotations only when they match the selected release.""" + expected = { + "meta.helm.sh/release-name": helm_release, + "meta.helm.sh/release-namespace": helm_namespace, + } + for key, value in expected.items(): + if value is not None and annotations.get(key) == value: + annotations.pop(key, None) + + def _strip_deployment_defaults(deployment: dict[str, Any]) -> None: """Remove Deployment-specific API-server/controller defaults absent from helm template.""" spec = deployment.get("spec", {}) @@ -303,7 +319,12 @@ def strategic_merge(live: Any, desired: Any) -> Any: def allowed_recovery_drift( - live: dict[str, Any], rendered: dict[str, Any], *, autoscaling_enabled: bool = False + live: dict[str, Any], + rendered: dict[str, Any], + *, + autoscaling_enabled: bool = False, + helm_release: str | None = None, + helm_namespace: str | None = None, ) -> list[str]: """Compare chart-owned objects after replacing only image and REDIS source.""" adapted = copy.deepcopy(live) @@ -321,7 +342,8 @@ def allowed_recovery_drift( adapted.get("spec", {}).pop("replicas", None) return ( [] - if normalize(adapted) == normalize(rendered) + if normalize(adapted, helm_release=helm_release, helm_namespace=helm_namespace) + == normalize(rendered, helm_release=helm_release, helm_namespace=helm_namespace) else ["recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition"] ) @@ -332,11 +354,19 @@ def hpa_controls_deployment(hpa: dict[str, Any], deployment_name: str) -> bool: return target == {"apiVersion": "apps/v1", "kind": "Deployment", "name": deployment_name} -def validate_chart_owned_resource_drift(live: dict[str, Any], rendered: dict[str, Any], kind: str) -> list[str]: +def validate_chart_owned_resource_drift( + live: dict[str, Any], + rendered: dict[str, Any], + kind: str, + *, + helm_release: str | None = None, + helm_namespace: str | None = None, +) -> list[str]: """Require Service/HPA/PDB identity and spec to remain unchanged.""" return ( [] - if normalize(live) == normalize(rendered) + if normalize(live, helm_release=helm_release, helm_namespace=helm_namespace) + == normalize(rendered, helm_release=helm_release, helm_namespace=helm_namespace) else [f"recovery profile would change {kind} outside the allowlist"] ) @@ -433,7 +463,9 @@ def preflight(args: argparse.Namespace) -> list[str]: # The normal chart-owned resource validation below reports the # malformed or missing HPA. Keep Deployment replicas fail-closed. pass - failures += allowed_recovery_drift(live, rendered, autoscaling_enabled=autoscaling_enabled) + failures += allowed_recovery_drift( + live, rendered, autoscaling_enabled=autoscaling_enabled, helm_release=release, helm_namespace=namespace + ) if not args.live_pods: failures.append("recovery preflight is missing live Pusher pod image-status evidence") else: @@ -458,7 +490,11 @@ def preflight(args: argparse.Namespace) -> list[str]: failures += validate_target(live_resource, kind, name, namespace) try: failures += validate_chart_owned_resource_drift( - live_resource, rendered_resource(args.rendered, kind, name), kind + live_resource, + rendered_resource(args.rendered, kind, name), + kind, + helm_release=release, + helm_namespace=namespace, ) except ValueError as exc: failures.append(str(exc)) diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index fd49e939e55..3e6920aaf63 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -7,6 +7,7 @@ import runpy import subprocess from types import SimpleNamespace +from typing import Any, cast import pytest @@ -306,6 +307,23 @@ def test_service_recovery_profile_ignores_api_defaults_but_not_real_drift(recove assert recovery.validate_chart_owned_resource_drift(live, rendered, "Service") == [] + helm_owned = copy.deepcopy(live) + helm_metadata = cast(dict[str, Any], helm_owned["metadata"]) + helm_annotations = cast(dict[str, Any], helm_metadata["annotations"]) + helm_annotations.update( + {"meta.helm.sh/release-name": "prod-omi-pusher", "meta.helm.sh/release-namespace": "prod-omi-backend"} + ) + assert ( + recovery.validate_chart_owned_resource_drift( + helm_owned, + rendered, + "Service", + helm_release="prod-omi-pusher", + helm_namespace="prod-omi-backend", + ) + == [] + ) + annotation_drift = copy.deepcopy(live) annotation_drift["metadata"]["annotations"]["cloud.google.com/neg"] = '{"ingress": false}' assert recovery.validate_chart_owned_resource_drift(annotation_drift, rendered, "Service") == [ @@ -374,6 +392,8 @@ def test_chart_only_workflow_skips_build_push_and_normal_paths_stay_available(): assert "--expected-evidence pusher-recovery-evidence.json" in workflow assert "verify_pusher_config_references.py" in workflow assert "--rendered .pusher-recovery-snapshot/rendered.yaml" in workflow + assert "pusher-recovery-rollback-identity.json" in workflow + assert "helm_history" in workflow def test_chart_only_preapply_gate_recaptures_all_chart_owned_resources(): From 06163208f92d61569eb99dbb3173eecb4b5b1fe3 Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 13:23:54 -0400 Subject: [PATCH 12/14] fix(pusher): preserve nondefault deployment drift Failure-Class: FC-runtime-image-boundary Co-authored-by: multica-agent --- .../verify_pusher_chart_only_recovery.py | 33 ++++++++++--------- .../test_verify_pusher_chart_only_recovery.py | 18 ++++++++++ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index dd0a434968e..2478b66a124 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -32,17 +32,15 @@ # from ``helm template`` output. ``normalize()`` strips them so the drift # comparison does not spuriously fail on a healthy live Deployment. VOLATILE_DEPLOYMENT_ANNOTATIONS = {"deployment.kubernetes.io/revision"} -VOLATILE_DEPLOYMENT_SPEC = { - "revisionHistoryLimit", +DEPLOYMENT_API_DEFAULTS = {"revisionHistoryLimit": 10} +POD_TEMPLATE_API_DEFAULTS = { + "restartPolicy": "Always", + "dnsPolicy": "ClusterFirst", + "schedulerName": "default-scheduler", } -VOLATILE_POD_TEMPLATE_SPEC = { - "restartPolicy", - "dnsPolicy", - "schedulerName", -} -VOLATILE_CONTAINER_FIELDS = { - "terminationMessagePath", - "terminationMessagePolicy", +CONTAINER_API_DEFAULTS = { + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", } # Cluster-added Service fields absent from ``helm template`` output. @@ -246,22 +244,25 @@ def _strip_expected_helm_annotations( def _strip_deployment_defaults(deployment: dict[str, Any]) -> None: """Remove Deployment-specific API-server/controller defaults absent from helm template.""" spec = deployment.get("spec", {}) - for key in VOLATILE_DEPLOYMENT_SPEC: - spec.pop(key, None) + for key, value in DEPLOYMENT_API_DEFAULTS.items(): + if spec.get(key) == value: + spec.pop(key, None) if spec.get("minReadySeconds") == 0: spec.pop("minReadySeconds", None) template = spec.get("template", {}) template_spec = template.get("spec", {}) - for key in VOLATILE_POD_TEMPLATE_SPEC: - template_spec.pop(key, None) + for key, value in POD_TEMPLATE_API_DEFAULTS.items(): + if template_spec.get(key) == value: + template_spec.pop(key, None) if template_spec.get("serviceAccount") == template_spec.get("serviceAccountName"): template_spec.pop("serviceAccount", None) _strip_default_security_context(template_spec, "securityContext", POD_SECURITY_DEFAULTS) for container in template_spec.get("containers", []): - for key in VOLATILE_CONTAINER_FIELDS: - container.pop(key, None) + for key, value in CONTAINER_API_DEFAULTS.items(): + if container.get(key) == value: + container.pop(key, None) _strip_default_security_context(container, "securityContext", CONTAINER_SECURITY_DEFAULTS) for probe_name in ("livenessProbe", "readinessProbe", "startupProbe"): probe = container.get(probe_name) diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index 3e6920aaf63..7117a2a2d62 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -176,6 +176,24 @@ def test_recovery_profile_ignores_only_deployment_api_defaults(recovery: SimpleN "recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition" ] + for mutate in ( + lambda value: value["spec"].__setitem__("revisionHistoryLimit", 1), + lambda value: value["spec"]["template"]["spec"].__setitem__("restartPolicy", "Never"), + lambda value: value["spec"]["template"]["spec"].__setitem__("dnsPolicy", "Default"), + lambda value: value["spec"]["template"]["spec"].__setitem__("schedulerName", "custom-scheduler"), + lambda value: value["spec"]["template"]["spec"]["containers"][0].__setitem__( + "terminationMessagePath", "/custom/termination-log" + ), + lambda value: value["spec"]["template"]["spec"]["containers"][0].__setitem__( + "terminationMessagePolicy", "FallbackToLogsOnError" + ), + ): + policy_drift = copy.deepcopy(live) + mutate(policy_drift) + assert recovery.allowed_recovery_drift(policy_drift, rendered) == [ + "recovery profile would change Deployment fields outside the exact image and REDIS_DB_HOST transition" + ] + def test_recovery_profile_ignores_deployment_and_service_defaults(recovery: SimpleNamespace): """Only API/controller defaults absent from helm template are ignored.""" From 45a01602910b20741cf9f14e05c92b9c36056f4e Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 13:43:55 -0400 Subject: [PATCH 13/14] chore: align Dart formatting with current main Co-authored-by: multica-agent --- .../discovery/rayban_meta_discoverer.dart | 3 +- .../wals/recording_transfer_coordinator.dart | 42 +++++++++---------- .../recording_transfer_coordinator_test.dart | 6 +-- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/app/lib/services/devices/discovery/rayban_meta_discoverer.dart b/app/lib/services/devices/discovery/rayban_meta_discoverer.dart index f26ab1658d1..bb9187a358b 100644 --- a/app/lib/services/devices/discovery/rayban_meta_discoverer.dart +++ b/app/lib/services/devices/discovery/rayban_meta_discoverer.dart @@ -72,8 +72,7 @@ class RayBanMetaDiscoverer extends DeviceDiscoverer { if (mode == 'audio_only') { final inputs = await host.getBluetoothHfpInputs(); final storedDevice = SharedPreferencesUtil().btDevice; - final hasStoredAudioSelection = - storedDevice.type == DeviceType.raybanMeta && + final hasStoredAudioSelection = storedDevice.type == DeviceType.raybanMeta && storedDevice.locator?.extras[audioOnlyExtraKey] == true && storedDevice.id.isNotEmpty && storedDevice.id != legacyAudioOnlyId; diff --git a/app/lib/services/wals/recording_transfer_coordinator.dart b/app/lib/services/wals/recording_transfer_coordinator.dart index ee8f7e2da48..4612da80e8d 100644 --- a/app/lib/services/wals/recording_transfer_coordinator.dart +++ b/app/lib/services/wals/recording_transfer_coordinator.dart @@ -23,17 +23,17 @@ class RecordingTransferDrainResult { /// Nothing eligible to drain (empty backlog). Not a retry signal. const RecordingTransferDrainResult.skipped() - : attempted = false, - failed = false, - needsReconciliation = false, - contended = false; + : attempted = false, + failed = false, + needsReconciliation = false, + contended = false; /// Drain could not run because another sync owned the seam. Retry later. const RecordingTransferDrainResult.contended() - : attempted = false, - failed = false, - needsReconciliation = false, - contended = true; + : attempted = false, + failed = false, + needsReconciliation = false, + contended = true; final bool attempted; final bool failed; @@ -61,24 +61,24 @@ class RecordingTransferCoordinator { bool initiallyConnected = true, DateTime Function()? clock, RecordingTransferCooldownScheduler? scheduleCooldown, - }) : _reconcile = reconcile, - _discover = discover, - _refreshPending = refreshPending, - _drain = drain, - _autoUploadEnabled = autoUploadEnabled, - _clock = clock ?? DateTime.now, - _scheduleCooldown = scheduleCooldown { + }) : _reconcile = reconcile, + _discover = discover, + _refreshPending = refreshPending, + _drain = drain, + _autoUploadEnabled = autoUploadEnabled, + _clock = clock ?? DateTime.now, + _scheduleCooldown = scheduleCooldown { _configured = true; _listenToConnectivity(connectivityChanges, initiallyConnected); } RecordingTransferCoordinator._singleton() - : _reconcile = _noop, - _discover = _noop, - _refreshPending = _noop, - _drain = _skippedDrain, - _autoUploadEnabled = _disabled, - _clock = DateTime.now; + : _reconcile = _noop, + _discover = _noop, + _refreshPending = _noop, + _drain = _skippedDrain, + _autoUploadEnabled = _disabled, + _clock = DateTime.now; static final RecordingTransferCoordinator instance = RecordingTransferCoordinator._singleton(); diff --git a/app/test/services/wals/recording_transfer_coordinator_test.dart b/app/test/services/wals/recording_transfer_coordinator_test.dart index 3fc357ecaa6..9191d9b9ec2 100644 --- a/app/test/services/wals/recording_transfer_coordinator_test.dart +++ b/app/test/services/wals/recording_transfer_coordinator_test.dart @@ -209,9 +209,9 @@ class _ScheduledCooldown { class _TransferHarness { _TransferHarness({bool autoUploadEnabled = true, List? backlog, List? drainedWalIds}) - : _autoUploadEnabled = autoUploadEnabled, - backlog = backlog ?? ['wal-1'], - drainedWalIds = drainedWalIds ?? [] { + : _autoUploadEnabled = autoUploadEnabled, + backlog = backlog ?? ['wal-1'], + drainedWalIds = drainedWalIds ?? [] { coordinator = RecordingTransferCoordinator( reconcile: _reconcile, discover: _discover, From 39e972d577fc82033359720e8b79a03e4752f97f Mon Sep 17 00:00:00 2001 From: David Zhang Date: Wed, 22 Jul 2026 13:49:34 -0400 Subject: [PATCH 14/14] fix(pusher): reject incomplete observed rollouts Co-authored-by: multica-agent --- .../verify_pusher_chart_only_recovery.py | 21 +++++++++++++++- .../test_verify_pusher_chart_only_recovery.py | 24 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/backend/scripts/verify_pusher_chart_only_recovery.py b/backend/scripts/verify_pusher_chart_only_recovery.py index 2478b66a124..e90e9d3e3e4 100644 --- a/backend/scripts/verify_pusher_chart_only_recovery.py +++ b/backend/scripts/verify_pusher_chart_only_recovery.py @@ -176,7 +176,26 @@ def is_concurrent_rollout(deployment: dict[str, Any]) -> bool: status = deployment.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") - return generation is None or observed is None or generation != observed + if generation is None or observed is None or generation != observed: + return True + + replicas = status.get("replicas") + if not isinstance(replicas, int) or replicas < 0: + return True + if any(status.get(field) != replicas for field in ("updatedReplicas", "readyReplicas", "availableReplicas")): + return True + if status.get("unavailableReplicas") not in (None, 0): + return True + + conditions = status.get("conditions") + if not isinstance(conditions, list): + return True + complete_conditions = { + condition.get("type") + for condition in conditions + if isinstance(condition, dict) and condition.get("status") == "True" + } + return not {"Available", "Progressing"}.issubset(complete_conditions) def ready_replicas(deployment: dict[str, Any]) -> int: diff --git a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py index 7117a2a2d62..b991b2a426b 100644 --- a/backend/tests/unit/test_verify_pusher_chart_only_recovery.py +++ b/backend/tests/unit/test_verify_pusher_chart_only_recovery.py @@ -50,7 +50,17 @@ def deployment(image: str, redis_source: dict | None = None) -> dict: } } }, - "status": {"observedGeneration": 2, "readyReplicas": 7}, + "status": { + "observedGeneration": 2, + "replicas": 7, + "updatedReplicas": 7, + "readyReplicas": 7, + "availableReplicas": 7, + "conditions": [ + {"type": "Available", "status": "True"}, + {"type": "Progressing", "status": "True", "reason": "NewReplicaSetAvailable"}, + ], + }, } @@ -288,6 +298,7 @@ def test_redis_validation_accepts_post_repair_env_without_secret_key(recovery: S def test_target_and_readiness_guards_fail_closed(recovery: SimpleNamespace): live = deployment("x") + assert not recovery.is_concurrent_rollout(live) live["status"]["observedGeneration"] = 1 assert recovery.is_concurrent_rollout(live) assert recovery.ready_replicas(live) == 7 @@ -299,6 +310,17 @@ def test_target_and_readiness_guards_fail_closed(recovery: SimpleNamespace): ) +def test_rollout_guard_rejects_an_incomplete_rollout_with_current_observed_generation(recovery: SimpleNamespace): + live = deployment("x") + live["status"]["updatedReplicas"] = 6 + live["status"]["unavailableReplicas"] = 1 + assert recovery.is_concurrent_rollout(live) + + missing_completion_evidence = deployment("x") + missing_completion_evidence["status"].pop("conditions") + assert recovery.is_concurrent_rollout(missing_completion_evidence) + + @pytest.mark.parametrize("kind", ["Service", "HorizontalPodAutoscaler", "PodDisruptionBudget"]) def test_chart_owned_resources_reject_unexpected_drift(recovery: SimpleNamespace, kind: str): live = {"kind": kind, "metadata": {"name": "prod-omi-pusher"}, "spec": {"guard": "unchanged"}}