feat(pusher): harden native RollingUpdate (SCA-39 long-term rollout, DRAFT)

.github/checks-manifest.yaml

@@ -249,6 +249,11 @@ checks:
     triggers: ["backend/charts/pusher/**", ".github/workflows/gcp_backend_pusher.yml", ".github/workflows/gcp_backend_pusher_auto_deploy.yml", "backend/scripts/verify_pusher_rollout_budget.py", "backend/tests/unit/test_verify_pusher_rollout_budget.py", ".github/checks-manifest.yaml"]
     lanes: ["local", "ci"]
     reason: "#9988: Pusher's Kubernetes deadline and workflow wait must cover chart-derived availability at the HPA ceiling"
+  - id: pusher-rollout-quality-gate
+    command: ["python3", "backend/scripts/verify_pusher_rollout_gate.py"]
+    triggers: ["backend/charts/pusher/**", "backend/utils/metrics.py", "backend/routers/pusher.py", "backend/utils/readiness.py", "backend/scripts/verify_pusher_rollout_gate.py", "backend/tests/unit/test_verify_pusher_rollout_gate.py", ".github/checks-manifest.yaml"]
+    lanes: ["local", "ci"]
+    reason: "SCA-39: static preflight that fails closed when a pusher rollout cannot be verified safe (capacity, image identity, readiness/health split, telemetry)"
   - id: conversation-lifecycle-write-guard
     command: ["python3", "backend/scripts/check_conversation_lifecycle_writes.py"]
     triggers: ["backend/**/*.py", "backend/scripts/check_conversation_lifecycle_writes.py"]

backend/charts/pusher/dev_omi_pusher_values.yaml

@@ -18,6 +18,10 @@ image:
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: ""
+  # Build-once promotion: set `digest: sha256:<64 lowercase hex>` (and clear
+  # `tag`, set `pullPolicy: IfNotPresent`) to deploy an exact content-addressed
+  # image qualified once in dev. Malformed/ambiguous identity is rejected at
+  # render time. See backend/scripts/verify_pusher_rollout_gate.py.
 
 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets: []
@@ -69,6 +73,12 @@ service:
     cloud.google.com/neg: '{"ingress": true}'
     cloud.google.com/backend-config: '{"default": "dev-pusher-backend-config"}'  # Link to BackendConfig
 
+# BackendConfig connectionDraining: bounds how long the LB waits before cutting
+# in-flight WS on endpoint removal (must be <= terminationGracePeriodSeconds 120).
+backendConfig:
+  connectionDraining:
+    drainingTimeoutSec: 60
+
 # This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/
 ingress:
   enabled: true
@@ -306,7 +316,7 @@ livenessProbe:
 
 readinessProbe:
   httpGet:
-    path: /health
+    path: /ready
     port: 8080
   failureThreshold: 3
   initialDelaySeconds: 0
@@ -366,7 +376,9 @@ terminationGracePeriodSeconds: 120
 lifecycle:
   preStop:
     exec:
-      command: ["sh", "-c", "sleep 15"]
+      # preStop flips readiness (via Lane A /__internal/drain) so the kubelet removes the endpoint
+      # early; sleep covers NEG->backend-service->ALB reprogram convergence before SIGTERM.
+      command: ["sh", "-c", "curl -sf -m 5 -X POST http://localhost:8080/__internal/drain || true; sleep 15"]
 
 # Additional volumes on the output Deployment definition.
 volumes: []

backend/charts/pusher/prod_omi_pusher_values.yaml

@@ -19,6 +19,10 @@ image:
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   tag: ""
+  # Build-once promotion: set `digest: sha256:<64 lowercase hex>` (and clear
+  # `tag`, set `pullPolicy: IfNotPresent`) to promote the exact dev-qualified
+  # image by content address. Malformed/ambiguous identity is rejected at
+  # render time. See backend/scripts/verify_pusher_rollout_gate.py.
 
 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets: []
@@ -70,6 +74,12 @@ service:
     cloud.google.com/neg: '{"ingress": true}'
     cloud.google.com/backend-config: '{"default": "prod-pusher-backend-config"}'  # Link to BackendConfig
 
+# BackendConfig connectionDraining: bounds how long the LB waits before cutting
+# in-flight WS on endpoint removal (must be <= terminationGracePeriodSeconds 120).
+backendConfig:
+  connectionDraining:
+    drainingTimeoutSec: 60
+
 # This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/
 ingress:
   enabled: true
@@ -325,7 +335,7 @@ livenessProbe:
 
 readinessProbe:
   httpGet:
-    path: /health
+    path: /ready
     port: 8080
   failureThreshold: 3
   initialDelaySeconds: 0
@@ -386,7 +396,9 @@ terminationGracePeriodSeconds: 120
 lifecycle:
   preStop:
     exec:
-      command: ["sh", "-c", "sleep 15"]
+      # preStop flips readiness (via Lane A /__internal/drain) so the kubelet removes the endpoint
+      # early; sleep covers NEG->backend-service->ALB reprogram convergence before SIGTERM.
+      command: ["sh", "-c", "curl -sf -m 5 -X POST http://localhost:8080/__internal/drain || true; sleep 15"]
 
 # Additional volumes on the output Deployment definition.
 volumes: []

backend/charts/pusher/templates/backendconfig.yaml

@@ -5,9 +5,14 @@ metadata:
 spec:
   # Long timeout required for persistent WebSocket connections between backend-listen and pusher
   timeoutSec: 3600
+  # Invariant: drainingTimeoutSec must be <= Deployment terminationGracePeriodSeconds (120) so the
+  # LB drains in-flight connections inside the pod grace window instead of racing SIGTERM.
+  connectionDraining:
+    drainingTimeoutSec: {{ .Values.backendConfig.connectionDraining.drainingTimeoutSec | default 60 }}
   healthCheck:
     checkIntervalSec: 10
     timeoutSec: 5
     port: {{ .Values.service.port }}
     type: HTTP
-    requestPath: /health  # Ensure this path returns 200 OK
+    # Liveness semantics: keep /health here so the LB health check never flips unhealthy during a readiness drain (that would defeat connectionDraining).
+    requestPath: /health

backend/charts/pusher/templates/deployment.yaml

@@ -54,7 +54,14 @@ spec:
           securityContext:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          image: "{{ required "image.repository is required" .Values.image.repository }}:{{ required "image.tag is required" .Values.image.tag }}"
+          {{- $repository := required "image.repository is required" .Values.image.repository }}
+          {{- $digest := .Values.image.digest }}
+          {{- if $digest }}
+          {{- if not (regexMatch "^sha256:[a-f0-9]{64}$" $digest) }}{{- fail (printf "image.digest must be sha256:<64 lowercase hex characters>, got %q" $digest) }}{{- end }}
+          {{- if .Values.image.tag }}{{- fail "image.tag must be empty when image.digest pins the release (digest is the content address)" }}{{- end }}
+          {{- if ne .Values.image.pullPolicy "IfNotPresent" }}{{- fail "image.pullPolicy must be IfNotPresent for a digest-pinned release (the digest is the content address; Always only adds a registry round-trip)" }}{{- end }}
+          {{- end }}
+          image: "{{ if $digest }}{{ $repository }}@{{ $digest }}{{ else }}{{ $repository }}:{{ required "image.tag is required when image.digest is absent" .Values.image.tag }}{{ end }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           {{- with .Values.envFrom }}
           envFrom: