diff --git a/.github/scripts/monitor-remote-workflow.mjs b/.github/scripts/monitor-remote-workflow.mjs index a9574d6b09..7976e55cdc 100644 --- a/.github/scripts/monitor-remote-workflow.mjs +++ b/.github/scripts/monitor-remote-workflow.mjs @@ -1,66 +1,301 @@ // @ts-nocheck -/** @param {{ github: any, core: any }} param0 */ -export default async ({ github, core }) => { +const RELEASE_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/; +const EVENT_TYPE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/; +const API_RETRY_ATTEMPTS = 5; +const API_RETRY_BASE_DELAY_MS = 1000; +const API_RETRY_MAX_DELAY_MS = 15000; +const RETRYABLE_NETWORK_CODES = new Set([ + "ECONNRESET", + "ETIMEDOUT", + "EAI_AGAIN", + "UND_ERR_CONNECT_TIMEOUT" +]); + +/** @param {string} name @param {string} value @param {number} fallback */ +function positiveNumber(name, value, fallback) { + const parsed = Number(value || String(fallback)); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive number`); + } + return parsed; +} + +/** @param {unknown} error */ +function errorText(error) { + return error instanceof Error ? error.message : String(error); +} + +/** @param {string} operationName @param {number} attempts @param {unknown} error */ +function describeFailure(operationName, attempts, error) { + const suffix = attempts > 1 ? ` after ${attempts} attempts` : ""; + return `${operationName} failed${suffix}: ${errorText(error)}`; +} + +/** @param {any} error */ +function isRetryableAPIError(error) { + const status = Number(error?.status ?? error?.response?.status ?? 0); + const headers = error?.response?.headers || {}; + return ( + status === 429 || + (status >= 500 && status <= 599) || + (status === 403 && + (headers["retry-after"] || headers["x-ratelimit-remaining"] === "0")) || + RETRYABLE_NETWORK_CODES.has(error?.code) || + RETRYABLE_NETWORK_CODES.has(error?.cause?.code) + ); +} + +/** @param {any} error */ +function retryAfterMilliseconds(error) { + const value = Number(error?.response?.headers?.["retry-after"] ?? 0); + return Number.isFinite(value) && value > 0 ? value * 1000 : 0; +} + +/** + * Dispatch or resume one correlated publisher run. + * + * @param {{ + * github: any, + * core: any, + * sleep?: (ms: number) => Promise, + * now?: () => number, + * random?: () => number, + * }} options + */ +export default async ({ + github, + core, + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + now = () => Date.now(), + random = () => Math.random() +}) => { try { const remoteOwner = core.getInput("OWNER", { required: true }); const remoteRepo = core.getInput("REPO", { required: true }); const remoteWorkflowFile = core.getInput("WORKFLOW_FILE", { required: true }); - const dispatchStartedAt = core.getInput("DISPATCH_STARTED_AT", { + const eventType = core.getInput("EVENT_TYPE", { required: true }); + const releaseIdentifier = core.getInput("RELEASE_IDENTIFIER", { + required: true + }); + const clientPayloadText = core.getInput("CLIENT_PAYLOAD", { required: true }); - const maxWaitSeconds = Number(core.getInput("MAX_WAIT_SECONDS") || "900"); // Default to 15 minutes - const pollIntervalSeconds = Number( - core.getInput("POLL_INTERVAL_SECONDS") || "10" + if (!EVENT_TYPE_PATTERN.test(eventType)) { + throw new Error(`Invalid event type: ${eventType}`); + } + if (!RELEASE_IDENTIFIER_PATTERN.test(releaseIdentifier)) { + throw new Error( + "Release identifier must contain 1-200 letters, numbers, dots, underscores, or hyphens" + ); + } + + const clientPayload = JSON.parse(clientPayloadText); + if ( + clientPayload === null || + Array.isArray(clientPayload) || + typeof clientPayload !== "object" + ) { + throw new Error("CLIENT_PAYLOAD must be a JSON object"); + } + if ( + clientPayload.release_identifier && + clientPayload.release_identifier !== releaseIdentifier + ) { + throw new Error("CLIENT_PAYLOAD release_identifier does not match input"); + } + clientPayload.release_identifier = releaseIdentifier; + + const maxWaitSeconds = positiveNumber( + "MAX_WAIT_SECONDS", + core.getInput("MAX_WAIT_SECONDS"), + 900 + ); + const pollIntervalSeconds = positiveNumber( + "POLL_INTERVAL_SECONDS", + core.getInput("POLL_INTERVAL_SECONDS"), + 10 ); + const deadline = now() + maxWaitSeconds * 1000; + const expectedRunName = `${eventType} / ${releaseIdentifier}`; - /** @param {number} ms */ - const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const sleepWithinBudget = async (milliseconds) => { + const remaining = deadline - now(); + if (remaining <= 0) { + return false; + } + await sleep(Math.min(milliseconds, remaining)); + return now() < deadline; + }; + + const retryDelay = (attempt, error) => { + const exponentialDelay = Math.min( + API_RETRY_MAX_DELAY_MS, + API_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1) + ); + const jitteredDelay = + exponentialDelay / 2 + random() * (exponentialDelay / 2); + return Math.max(Math.ceil(jitteredDelay), retryAfterMilliseconds(error)); + }; + + const retryAPI = async (operationName, operation) => { + for (let attempt = 1; attempt <= API_RETRY_ATTEMPTS; attempt += 1) { + try { + return await operation(); + } catch (error) { + if ( + !isRetryableAPIError(error) || + attempt === API_RETRY_ATTEMPTS || + now() >= deadline + ) { + throw new Error(describeFailure(operationName, attempt, error)); + } + + const delay = retryDelay(attempt, error); + core.info( + `Transient GitHub API failure during ${operationName}; retrying in ${delay}ms (attempt ${attempt + 1}/${API_RETRY_ATTEMPTS})` + ); + if (!(await sleepWithinBudget(delay))) { + throw new Error(describeFailure(operationName, attempt, error)); + } + } + } + throw new Error(`${operationName} exhausted its retry budget`); + }; + core.setOutput("release_identifier", releaseIdentifier); core.info( - `Waiting for remote workflow run in ${remoteOwner}/${remoteRepo}...` + `Reconciling publisher run '${expectedRunName}' in ${remoteOwner}/${remoteRepo}` ); - const findLatestRun = async () => { - const response = await github.rest.actions.listWorkflowRuns({ + /** @param {any[]} runs */ + const correlatedRuns = (runs) => + (runs || []) + .filter((run) => run.display_title === expectedRunName) + .sort((left, right) => right.id - left.id); + + /** @param {any[]} runs */ + const selectReusableRun = (runs) => + runs.find( + (run) => run.status === "completed" && run.conclusion === "success" + ) || runs.find((run) => run.status !== "completed"); + + const matchingRuns = async ({ allPages = true } = {}) => { + const parameters = { owner: remoteOwner, repo: remoteRepo, workflow_id: remoteWorkflowFile, event: "repository_dispatch", - per_page: 20 - }); - - const candidateRuns = (response.data.workflow_runs || []) - /** @param {any} run */ - .filter((run) => run.created_at >= dispatchStartedAt) - /** @param {any} left @param {any} right */ - .sort((left, right) => left.created_at.localeCompare(right.created_at)); - - return candidateRuns.length > 0 ? - candidateRuns[candidateRuns.length - 1] - : undefined; + per_page: 100 + }; + + if (!allPages) { + const response = await retryAPI("recent publisher run lookup", () => + github.rest.actions.listWorkflowRuns(parameters) + ); + return correlatedRuns(response.data.workflow_runs); + } + + const runs = await retryAPI("publisher run lookup", () => + github.paginate( + github.rest.actions.listWorkflowRuns, + parameters, + (response) => { + if (now() >= deadline) { + throw new Error( + "the monitor budget ran out before every history page was read; refusing to dispatch without completing discovery" + ); + } + return correlatedRuns(response.data); + } + ) + ); + return correlatedRuns(runs); }; - let run; - for ( - let elapsed = 0; - elapsed < maxWaitSeconds; - elapsed += pollIntervalSeconds - ) { - run = await findLatestRun(); - if (run) { - break; + const dispatchRun = async (previousRunID) => { + const operationName = `publisher dispatch '${eventType}'`; + for (let attempt = 1; attempt <= API_RETRY_ATTEMPTS; attempt += 1) { + try { + await github.rest.repos.createDispatchEvent({ + owner: remoteOwner, + repo: remoteRepo, + event_type: eventType, + client_payload: clientPayload + }); + return undefined; + } catch (error) { + if (!isRetryableAPIError(error)) { + throw new Error(describeFailure(operationName, attempt, error)); + } + + const delay = Math.max( + pollIntervalSeconds * 1000, + retryDelay(attempt, error) + ); + core.info( + `Publisher dispatch returned a transient error; checking for '${expectedRunName}' before retrying` + ); + if (now() < deadline) { + await sleepWithinBudget(delay); + } + + const newRuns = (await matchingRuns({ allPages: false })).filter( + (candidate) => candidate.id > previousRunID + ); + const discoveredRun = selectReusableRun(newRuns) || newRuns[0]; + if (discoveredRun) { + core.info( + `The uncertain dispatch created correlated run ${discoveredRun.id}; skipping redispatch` + ); + return discoveredRun; + } + if (attempt === API_RETRY_ATTEMPTS || now() >= deadline) { + throw new Error( + `${describeFailure(operationName, attempt, error)}; no run '${expectedRunName}' appeared, so a rerun of this job reconciles before dispatching again` + ); + } + } } + throw new Error("Publisher dispatch exhausted its retry budget"); + }; + + const existingRuns = await matchingRuns(); + const previousRunID = existingRuns.reduce( + (highest, candidate) => Math.max(highest, candidate.id), + 0 + ); + let run = selectReusableRun(existingRuns); + if (!run) { + core.info( + existingRuns.length > 0 ? + `Previous correlated runs failed; dispatching retry '${eventType}'` + : `No existing run found; dispatching '${eventType}'` + ); + run = await dispatchRun(previousRunID); - await sleep(pollIntervalSeconds * 1000); + while (!run && now() <= deadline) { + const newRuns = (await matchingRuns({ allPages: false })).filter( + (candidate) => candidate.id > previousRunID + ); + run = selectReusableRun(newRuns) || newRuns[0]; + if (run || now() >= deadline) { + break; + } + await sleep( + Math.min(pollIntervalSeconds * 1000, Math.max(0, deadline - now())) + ); + } + } else { + core.info(`Reusing correlated remote run ${run.id}`); } if (!run) { core.setFailed( - `Timed out waiting for remote workflow run to start: https://github.com/${remoteOwner}/${remoteRepo}/actions/workflows/${remoteWorkflowFile}` + `Timed out waiting for publisher run '${expectedRunName}': https://github.com/${remoteOwner}/${remoteRepo}/actions/workflows/${remoteWorkflowFile}` ); return; } @@ -68,74 +303,65 @@ export default async ({ github, core }) => { const runUrl = run.html_url || `https://github.com/${remoteOwner}/${remoteRepo}/actions/runs/${run.id}`; + core.setOutput("run_id", String(run.id)); + core.setOutput("run_url", runUrl); + core.info(`Monitoring correlated remote run: ${runUrl}`); - core.info(`Monitoring remote run id: ${run.id}`); - core.info(`Remote workflow run URL: ${runUrl}`); - - for ( - let elapsed = 0; - elapsed < maxWaitSeconds; - elapsed += pollIntervalSeconds - ) { - const runResponse = await github.rest.actions.getWorkflowRun({ - owner: remoteOwner, - repo: remoteRepo, - run_id: run.id - }); - - const status = runResponse.data.status; - const conclusion = runResponse.data.conclusion || ""; - - if (status === "completed") { - core.setOutput("run_id", String(run.id)); - core.setOutput("run_url", runUrl); + while (true) { + if (run.status === "completed") { + const conclusion = run.conclusion || ""; core.setOutput("conclusion", conclusion); - if (conclusion === "success") { - core.info("Remote workflow completed successfully"); + core.info("Correlated remote workflow completed successfully"); return; } - const jobsResponse = await github.rest.actions.listJobsForWorkflowRun({ - owner: remoteOwner, - repo: remoteRepo, - run_id: run.id, - per_page: 100 - }); - + const jobsResponse = await retryAPI("publisher job diagnostics", () => + github.rest.actions.listJobsForWorkflowRun({ + owner: remoteOwner, + repo: remoteRepo, + run_id: run.id, + per_page: 100 + }) + ); const failedJobs = (jobsResponse.data.jobs || []).filter( - /** @param {any} job */ (job) => job.conclusion !== "success" ); const failedJobText = failedJobs - /** @param {any} job */ .map((job) => { const failedSteps = (job.steps || []) - /** @param {any} step */ .filter((step) => step.conclusion === "failure") - /** @param {any} step */ .map((step) => ` - Step: ${step.name}`) .join("\n"); - return `- Job: ${job.name} [${job.conclusion}]${failedSteps ? `\n${failedSteps}` : ""}`; }) .join("\n"); core.setFailed( - `Remote workflow failed with conclusion: ${conclusion}\nRemote workflow run: ${runUrl}${failedJobText ? `\nFailed jobs/steps:\n${failedJobText}` : ""}` + `Correlated remote workflow failed with conclusion: ${conclusion}\nRemote workflow run: ${runUrl}${failedJobText ? `\nFailed jobs/steps:\n${failedJobText}` : ""}` ); return; } - await sleep(pollIntervalSeconds * 1000); + if (now() >= deadline) { + core.setFailed( + `Timed out waiting for correlated remote workflow completion: ${runUrl}` + ); + return; + } + await sleep( + Math.min(pollIntervalSeconds * 1000, Math.max(0, deadline - now())) + ); + const runResponse = await retryAPI("publisher run status", () => + github.rest.actions.getWorkflowRun({ + owner: remoteOwner, + repo: remoteRepo, + run_id: run.id + }) + ); + run = runResponse.data; } - - core.setOutput("run_id", String(run.id)); - core.setOutput("run_url", runUrl); - core.setFailed( - `Timed out waiting for remote workflow completion: ${runUrl}` - ); } catch (error) { - core.setFailed(error instanceof Error ? error.message : String(error)); + core.setFailed(errorText(error)); } }; diff --git a/.github/scripts/monitor-remote-workflow_test.mjs b/.github/scripts/monitor-remote-workflow_test.mjs new file mode 100644 index 0000000000..0165e01d26 --- /dev/null +++ b/.github/scripts/monitor-remote-workflow_test.mjs @@ -0,0 +1,631 @@ +// Copyright 2026 The Radius Authors. +// Licensed under the Apache License, Version 2.0. + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import monitorRemoteWorkflow from "./monitor-remote-workflow.mjs"; + +function createCore(overrides = {}) { + const inputs = { + OWNER: "azure-octo", + REPO: "radius-publisher", + WORKFLOW_FILE: "publish-deployment-engine.yml", + EVENT_TYPE: "deployment-engine", + RELEASE_IDENTIFIER: "0.61.0-aaaaaaaa", + CLIENT_PAYLOAD: JSON.stringify({ tag: "0.61" }), + MAX_WAIT_SECONDS: "30", + POLL_INTERVAL_SECONDS: "1", + ...overrides + }; + const outputs = new Map(); + const failures = []; + const infos = []; + + return { + inputs, + outputs, + failures, + infos, + getInput(name, options = {}) { + const value = inputs[name] || ""; + if (options.required && !value) { + throw new Error(`Input required and not supplied: ${name}`); + } + return value; + }, + setOutput(name, value) { + outputs.set(name, value); + }, + setFailed(message) { + failures.push(message); + }, + info(message) { + infos.push(message); + } + }; +} + +function createClock() { + let time = 0; + return { + now: () => time, + sleep: async (milliseconds) => { + time += milliseconds; + } + }; +} + +function apiError(status, message = `GitHub API returned ${status}`) { + return Object.assign(new Error(message), { status }); +} + +function createGithub({ runs = [], getRun, jobs = [], pages } = {}) { + const dispatches = []; + const calls = { list: 0, get: 0, jobs: 0, pages: 0 }; + let nextRunID = 1000; + let paginating = false; + + const github = { + dispatches, + calls, + runs, + // Mirrors octokit: the map callback gets a `done` stopper, and pages are + // fetched until it is called or the fixture runs out of pages. + async paginate(method, parameters, map) { + const collected = []; + let stopped = false; + const done = () => { + stopped = true; + }; + paginating = true; + try { + const pageCount = pages ? pages.length : 1; + for (let page = 0; page < pageCount && !stopped; page += 1) { + const response = await method({ ...parameters, page: page + 1 }); + collected.push( + ...map({ ...response, data: response.data.workflow_runs }, done) + ); + } + } finally { + paginating = false; + } + return collected; + }, + rest: { + actions: { + async listWorkflowRuns({ page = 1 } = {}) { + calls.list += 1; + if (pages && paginating) { + calls.pages += 1; + return { + data: { workflow_runs: [...(pages[page - 1] || [])] } + }; + } + return { data: { workflow_runs: [...runs] } }; + }, + async getWorkflowRun({ run_id: runID }) { + calls.get += 1; + const current = + getRun?.(runID, calls.get) || runs.find((run) => run.id === runID); + return { data: current }; + }, + async listJobsForWorkflowRun() { + calls.jobs += 1; + return { data: { jobs } }; + } + }, + repos: { + async createDispatchEvent({ + event_type: eventType, + client_payload: payload + }) { + dispatches.push({ eventType, payload }); + runs.push({ + id: nextRunID++, + display_title: `${eventType} / ${payload.release_identifier}`, + status: "completed", + conclusion: "success", + html_url: `https://example.test/runs/${nextRunID - 1}` + }); + } + } + } + }; + return github; +} + +function successfulRun(id, identifier) { + return { + id, + display_title: `deployment-engine / ${identifier}`, + status: "completed", + conclusion: "success", + html_url: `https://example.test/runs/${id}` + }; +} + +test("finds correlated runs beyond unrelated history pages", async () => { + const identifier = "0.61.0-abababab"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub({ + pages: [ + [{ ...successfulRun(44, identifier), conclusion: "failure" }], + [successfulRun(43, "another-release")], + [successfulRun(42, identifier)] + ] + }); + + await monitorRemoteWorkflow({ github, core, ...createClock() }); + + assert.deepEqual(core.failures, []); + assert.equal(github.calls.pages, 3); + assert.equal(github.dispatches.length, 0); + assert.equal(core.outputs.get("run_id"), "42"); +}); + +test("finds an existing run beyond five pages without redispatching", async () => { + const identifier = "0.61.0-acacacac"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const unrelated = [{ id: 7, display_title: "deployment-engine / other" }]; + const github = createGithub({ + pages: [ + ...Array.from({ length: 7 }, () => unrelated), + [successfulRun(6, identifier)] + ] + }); + + await monitorRemoteWorkflow({ github, core, ...createClock() }); + + assert.deepEqual(core.failures, []); + assert.equal(github.calls.pages, 8); + assert.equal(github.dispatches.length, 0); + assert.equal(core.outputs.get("run_id"), "6"); + assert.equal(core.outputs.get("conclusion"), "success"); +}); + +test("never dispatches when history discovery times out", async () => { + const core = createCore(); + const clock = createClock(); + const github = createGithub({ pages: [[], [], []] }); + const listWorkflowRuns = github.rest.actions.listWorkflowRuns; + github.rest.actions.listWorkflowRuns = async (parameters) => { + await clock.sleep(15000); + return listWorkflowRuns(parameters); + }; + + await monitorRemoteWorkflow({ github, core, ...clock }); + + assert.match(core.failures[0], /refusing to dispatch/); + assert.equal(github.dispatches.length, 0); +}); + +test("restarts paginated discovery after a transient failure", async () => { + const identifier = "0.61.0-adadadad"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub({ + pages: [[], [], [], [successfulRun(51, identifier)], []] + }); + const listWorkflowRuns = github.rest.actions.listWorkflowRuns; + let failed = false; + github.rest.actions.listWorkflowRuns = async (parameters) => { + if (!failed && parameters.page === 2) { + failed = true; + throw apiError(500); + } + return listWorkflowRuns(parameters); + }; + + await monitorRemoteWorkflow({ + github, + core, + ...createClock(), + random: () => 0 + }); + + assert.deepEqual(core.failures, []); + assert.equal(github.dispatches.length, 0); + assert.equal(core.outputs.get("run_id"), "51"); +}); + +test("reuses an existing successful run without dispatching", async () => { + const identifier = "0.61.0-aaaaaaaa"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub({ runs: [successfulRun(42, identifier)] }); + const clock = createClock(); + + await monitorRemoteWorkflow({ github, core, ...clock }); + + assert.deepEqual(core.failures, []); + assert.equal(github.dispatches.length, 0); + assert.equal(core.outputs.get("run_id"), "42"); + assert.equal(core.outputs.get("conclusion"), "success"); +}); + +test("dispatches once and injects the release identifier into the payload", async () => { + const identifier = "0.61.0-bbbbbbbb"; + const core = createCore({ + RELEASE_IDENTIFIER: identifier, + CLIENT_PAYLOAD: JSON.stringify({ tag: "0.61", source_sha: "b".repeat(40) }) + }); + const github = createGithub(); + const clock = createClock(); + + await monitorRemoteWorkflow({ github, core, ...clock }); + + assert.deepEqual(core.failures, []); + assert.equal(github.dispatches.length, 1); + assert.deepEqual(github.dispatches[0], { + eventType: "deployment-engine", + payload: { + tag: "0.61", + source_sha: "b".repeat(40), + release_identifier: identifier + } + }); +}); + +test("retries transient run lookups with bounded backoff", async () => { + const identifier = "0.61.0-bcbcbcbc"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub({ runs: [successfulRun(45, identifier)] }); + const listWorkflowRuns = github.rest.actions.listWorkflowRuns; + let attempts = 0; + github.rest.actions.listWorkflowRuns = async (parameters) => { + attempts += 1; + if (attempts < 3) { + throw apiError(503); + } + return listWorkflowRuns(parameters); + }; + + await monitorRemoteWorkflow({ + github, + core, + ...createClock(), + random: () => 0 + }); + + assert.deepEqual(core.failures, []); + assert.equal(attempts, 3); + assert.equal(github.dispatches.length, 0); + assert.equal(core.outputs.get("run_id"), "45"); +}); + +test("does not retry non-transient API failures", async () => { + const core = createCore(); + const github = createGithub(); + let attempts = 0; + github.rest.actions.listWorkflowRuns = async () => { + attempts += 1; + throw apiError(422, "Invalid workflow query"); + }; + + await monitorRemoteWorkflow({ + github, + core, + ...createClock(), + random: () => 0 + }); + + assert.equal(attempts, 1); + assert.equal( + core.failures[0], + "publisher run lookup failed: Invalid workflow query" + ); +}); + +test("reconciles an accepted final dispatch after a transient response error", async () => { + const identifier = "0.61.0-bdbdbdbd"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub(); + const createDispatchEvent = github.rest.repos.createDispatchEvent; + let attempts = 0; + github.rest.repos.createDispatchEvent = async (parameters) => { + attempts += 1; + if (attempts < 5) { + throw apiError(502); + } + await createDispatchEvent(parameters); + throw apiError(502); + }; + + await monitorRemoteWorkflow({ + github, + core, + ...createClock(), + random: () => 0 + }); + + assert.deepEqual(core.failures, []); + assert.equal(attempts, 5); + assert.equal(github.dispatches.length, 1); + assert.equal(core.outputs.get("conclusion"), "success"); +}); + +test("retries a rejected transient dispatch after correlated lookup", async () => { + const identifier = "0.61.0-bebebebe"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub(); + const createDispatchEvent = github.rest.repos.createDispatchEvent; + let attempts = 0; + github.rest.repos.createDispatchEvent = async (parameters) => { + attempts += 1; + if (attempts === 1) { + throw apiError(503); + } + return createDispatchEvent(parameters); + }; + + await monitorRemoteWorkflow({ + github, + core, + ...createClock(), + random: () => 0 + }); + + assert.deepEqual(core.failures, []); + assert.equal(attempts, 2); + assert.equal(github.dispatches.length, 1); + assert.equal(core.outputs.get("conclusion"), "success"); +}); + +test("names the operation when lookup retries are exhausted", async () => { + const core = createCore(); + const github = createGithub(); + let attempts = 0; + github.rest.actions.listWorkflowRuns = async () => { + attempts += 1; + throw apiError(503); + }; + + await monitorRemoteWorkflow({ + github, + core, + ...createClock(), + random: () => 0 + }); + + assert.equal(attempts, 5); + assert.equal(github.dispatches.length, 0); + assert.equal( + core.failures[0], + "publisher run lookup failed after 5 attempts: GitHub API returned 503" + ); +}); + +test("names the expected run when dispatch retries are exhausted", async () => { + const identifier = "0.61.0-bfbfbfbf"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub(); + let attempts = 0; + github.rest.repos.createDispatchEvent = async () => { + attempts += 1; + throw apiError(502); + }; + + await monitorRemoteWorkflow({ + github, + core, + ...createClock(), + random: () => 0 + }); + + assert.equal(attempts, 5); + assert.equal(github.dispatches.length, 0); + assert.equal( + core.failures[0], + `publisher dispatch 'deployment-engine' failed after 5 attempts: GitHub API returned 502; no run 'deployment-engine / ${identifier}' appeared, so a rerun of this job reconciles before dispatching again` + ); +}); + +test("monitors an existing active run to completion", async () => { + const identifier = "0.61.0-cccccccc"; + const active = { + id: 43, + display_title: `deployment-engine / ${identifier}`, + status: "in_progress", + conclusion: null + }; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub({ + runs: [active], + getRun: (runID) => ({ + ...active, + id: runID, + status: "completed", + conclusion: "success" + }) + }); + const clock = createClock(); + + await monitorRemoteWorkflow({ github, core, ...clock }); + + assert.deepEqual(core.failures, []); + assert.equal(github.dispatches.length, 0); + assert.equal(github.calls.get, 1); + assert.equal(core.outputs.get("conclusion"), "success"); +}); + +test("dispatches a retry after an existing failed run", async () => { + const identifier = "0.61.0-dddddddd"; + const failed = { + id: 44, + display_title: `deployment-engine / ${identifier}`, + status: "completed", + conclusion: "failure", + html_url: "https://example.test/runs/44" + }; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub({ runs: [failed] }); + + await monitorRemoteWorkflow({ github, core, ...createClock() }); + + assert.deepEqual(core.failures, []); + assert.equal(github.dispatches.length, 1); + assert.notEqual(core.outputs.get("run_id"), "44"); + assert.equal(core.outputs.get("conclusion"), "success"); +}); + +test("reports a newly dispatched failed retry with job diagnostics", async () => { + const identifier = "0.61.0-ffffffff"; + const core = createCore({ RELEASE_IDENTIFIER: identifier }); + const github = createGithub({ + runs: [], + jobs: [ + { + name: "Publish", + conclusion: "failure", + steps: [{ name: "Copy", conclusion: "failure" }] + } + ] + }); + github.rest.repos.createDispatchEvent = async ({ + event_type: eventType, + client_payload: payload + }) => { + github.dispatches.push({ eventType, payload }); + github.runs.push({ + id: 1001, + display_title: `${eventType} / ${payload.release_identifier}`, + status: "completed", + conclusion: "failure", + html_url: "https://example.test/runs/1001" + }); + }; + + await monitorRemoteWorkflow({ github, core, ...createClock() }); + + assert.equal(github.dispatches.length, 1); + assert.equal(github.calls.jobs, 1); + assert.match(core.failures[0], /Correlated remote workflow failed/); + assert.match(core.failures[0], /Step: Copy/); +}); + +test("ignores concurrent runs with other identifiers", async () => { + const firstID = "0.61.0-11111111"; + const secondID = "0.61.0-22222222"; + const runs = []; + const github = createGithub({ runs }); + const firstCore = createCore({ RELEASE_IDENTIFIER: firstID }); + const secondCore = createCore({ RELEASE_IDENTIFIER: secondID }); + + await Promise.all([ + monitorRemoteWorkflow({ github, core: firstCore, ...createClock() }), + monitorRemoteWorkflow({ github, core: secondCore, ...createClock() }) + ]); + + assert.deepEqual(firstCore.failures, []); + assert.deepEqual(secondCore.failures, []); + assert.equal(github.dispatches.length, 2); + assert.notEqual( + firstCore.outputs.get("run_id"), + secondCore.outputs.get("run_id") + ); + assert.equal( + github.dispatches.find( + (item) => item.payload.release_identifier === firstID + ).payload.release_identifier, + firstID + ); + assert.equal( + github.dispatches.find( + (item) => item.payload.release_identifier === secondID + ).payload.release_identifier, + secondID + ); +}); + +test("rejects malformed identifiers and mismatched payloads before API calls", async () => { + const malformedCore = createCore({ RELEASE_IDENTIFIER: "bad identifier" }); + const malformedGithub = createGithub(); + await monitorRemoteWorkflow({ + github: malformedGithub, + core: malformedCore, + ...createClock() + }); + assert.match(malformedCore.failures[0], /Release identifier/); + assert.equal(malformedGithub.calls.list, 0); + + const mismatchedCore = createCore({ + CLIENT_PAYLOAD: JSON.stringify({ release_identifier: "other" }) + }); + const mismatchedGithub = createGithub(); + await monitorRemoteWorkflow({ + github: mismatchedGithub, + core: mismatchedCore, + ...createClock() + }); + assert.match(mismatchedCore.failures[0], /does not match/); + assert.equal(mismatchedGithub.calls.list, 0); +}); + +test("uses one total timeout budget for discovery and completion", async () => { + const identifier = "0.61.0-eeeeeeee"; + const core = createCore({ + RELEASE_IDENTIFIER: identifier, + MAX_WAIT_SECONDS: "2", + POLL_INTERVAL_SECONDS: "1" + }); + const github = createGithub(); + github.rest.repos.createDispatchEvent = async ({ + event_type: eventType, + client_payload: payload + }) => { + github.dispatches.push({ eventType, payload }); + }; + const clock = createClock(); + + await monitorRemoteWorkflow({ github, core, ...clock }); + + assert.equal(github.dispatches.length, 1); + assert.match(core.failures[0], /Timed out waiting for publisher run/); +}); + +test("all publisher callers use stable identifiers without time-window discovery", async () => { + const cases = [ + { + file: "__build-bicep-types.yaml", + identifier: + "INPUT_RELEASE_IDENTIFIER: ${{ steps.release-metadata.outputs.release_version }}-${{ github.sha }}", + eventType: "INPUT_EVENT_TYPE: bicep-types", + timeout: "timeout-minutes: 18" + }, + { + file: "publish-de-image.yaml", + identifier: + "INPUT_RELEASE_IDENTIFIER: ${{ steps.payload.outputs.tag }}-${{ github.run_id }}", + eventType: "INPUT_EVENT_TYPE: deployment-engine", + timeout: "timeout-minutes: 18" + }, + { + file: "release.yaml", + identifier: + "INPUT_RELEASE_IDENTIFIER: ${{ steps.get-version.outputs.release-version }}-${{ github.sha }}", + eventType: "INPUT_EVENT_TYPE: deployment-engine", + timeout: "timeout-minutes: 25" + } + ]; + + for (const fixture of cases) { + const contents = await readFile( + new URL(`../workflows/${fixture.file}`, import.meta.url), + "utf8" + ); + assert.match(contents, /monitor-remote-workflow\.mjs/); + assert.ok(contents.includes(fixture.identifier), fixture.file); + assert.ok(contents.includes(fixture.eventType), fixture.file); + assert.ok(contents.includes(fixture.timeout), fixture.file); + assert.ok(contents.includes('INPUT_MAX_WAIT_SECONDS: "720"'), fixture.file); + assert.doesNotMatch(contents, /INPUT_DISPATCH_STARTED_AT/); + assert.doesNotMatch(contents, /peter-evans\/repository-dispatch/); + + const payload = contents.match( + /INPUT_CLIENT_PAYLOAD: \|-\n([\s\S]*?)\n\s*INPUT_MAX_WAIT_SECONDS/ + ); + assert.ok(payload, fixture.file); + assert.match(payload[1], /\$\{\{ toJSON\(/, fixture.file); + assert.doesNotMatch(payload[1], /"\$\{\{/, fixture.file); + } +}); diff --git a/.github/workflows/__build-bicep-types.yaml b/.github/workflows/__build-bicep-types.yaml index cacd224017..06b3ad11da 100644 --- a/.github/workflows/__build-bicep-types.yaml +++ b/.github/workflows/__build-bicep-types.yaml @@ -27,7 +27,7 @@ jobs: build-and-push-bicep-types: name: Dispatch Bicep Types publish runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 18 environment: publish-bicep permissions: contents: read # Required for actions/checkout @@ -45,6 +45,15 @@ jobs: - name: Parse release version and set environment variables run: python ./.github/scripts/get_release_version.py + - name: Capture release metadata + id: release-metadata + shell: bash + run: | + { + echo "release_version=${REL_VERSION}" + echo "release_channel=${REL_CHANNEL}" + } >> "$GITHUB_OUTPUT" + - name: Get App Token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 id: get-token @@ -58,29 +67,7 @@ jobs: repositories: | radius-publisher - - name: Capture dispatch start time - id: dispatch-start - shell: bash - run: | - echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" - - - name: Repository Dispatch - id: repository-dispatch - uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 - with: - token: ${{ steps.get-token.outputs.token }} - repository: azure-octo/radius-publisher - event-type: bicep-types - client-payload: |- - { - "source_repository": "${{ github.repository }}", - "source_ref": "${{ github.ref }}", - "source_sha": "${{ github.sha }}", - "rel_channel": "${{ env.REL_CHANNEL }}", - "registry_target": "radius" - } - - - name: Monitor remote workflow + - name: Dispatch or monitor correlated publisher run id: monitor-remote-workflow uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: @@ -92,8 +79,19 @@ jobs: INPUT_OWNER: azure-octo INPUT_REPO: radius-publisher INPUT_WORKFLOW_FILE: publish-bicep-types.yml - INPUT_DISPATCH_STARTED_AT: ${{ steps.dispatch-start.outputs.started_at }} - INPUT_MAX_WAIT_SECONDS: "600" + INPUT_EVENT_TYPE: bicep-types + INPUT_RELEASE_IDENTIFIER: ${{ steps.release-metadata.outputs.release_version }}-${{ github.sha }} + # toJSON emits each value already quoted and escaped, so no ref or + # channel string can break out of the payload it is placed in. + INPUT_CLIENT_PAYLOAD: |- + { + "source_repository": ${{ toJSON(github.repository) }}, + "source_ref": ${{ toJSON(github.ref) }}, + "source_sha": ${{ toJSON(github.sha) }}, + "rel_channel": ${{ toJSON(steps.release-metadata.outputs.release_channel) }}, + "registry_target": "radius" + } + INPUT_MAX_WAIT_SECONDS: "720" INPUT_POLL_INTERVAL_SECONDS: "15" - name: Show failed logs diff --git a/.github/workflows/publish-de-image.yaml b/.github/workflows/publish-de-image.yaml index f426525660..d906049628 100644 --- a/.github/workflows/publish-de-image.yaml +++ b/.github/workflows/publish-de-image.yaml @@ -27,7 +27,7 @@ jobs: dispatch-publish: name: Dispatch DE image publish runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 18 environment: name: publish-de-image permissions: @@ -64,27 +64,7 @@ jobs: repositories: | radius-publisher - - name: Capture dispatch start time - id: dispatch-start - shell: bash - run: | - echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" - - - name: Repository Dispatch - uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 - with: - token: ${{ steps.get-token.outputs.token }} - repository: azure-octo/radius-publisher - event-type: deployment-engine - client-payload: |- - { - "source_repository": "${{ github.repository }}", - "src_image": "${{ steps.payload.outputs.src_image }}", - "dest_image": "${{ steps.payload.outputs.dest_image }}", - "tag": "${{ steps.payload.outputs.tag }}" - } - - - name: Monitor remote workflow + - name: Dispatch or monitor correlated publisher run id: monitor-remote-workflow uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: @@ -96,8 +76,18 @@ jobs: INPUT_OWNER: azure-octo INPUT_REPO: radius-publisher INPUT_WORKFLOW_FILE: publish-deployment-engine.yml - INPUT_DISPATCH_STARTED_AT: ${{ steps.dispatch-start.outputs.started_at }} - INPUT_MAX_WAIT_SECONDS: "600" + INPUT_EVENT_TYPE: deployment-engine + INPUT_RELEASE_IDENTIFIER: ${{ steps.payload.outputs.tag }}-${{ github.run_id }} + # These values arrive from an external repository_dispatch, so emit + # them with toJSON rather than interpolating them into JSON text. + INPUT_CLIENT_PAYLOAD: |- + { + "source_repository": ${{ toJSON(github.repository) }}, + "src_image": ${{ toJSON(steps.payload.outputs.src_image) }}, + "dest_image": ${{ toJSON(steps.payload.outputs.dest_image) }}, + "tag": ${{ toJSON(steps.payload.outputs.tag) }} + } + INPUT_MAX_WAIT_SECONDS: "720" INPUT_POLL_INTERVAL_SECONDS: "15" - name: Show failed logs diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 54643470d1..06e91c827d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -151,7 +151,7 @@ jobs: # Must stay above the Deployment Engine monitor budget below # (INPUT_MAX_WAIT_SECONDS) plus checkout and reconciliation time, otherwise # the job is killed while the monitor is still waiting. - timeout-minutes: 20 + timeout-minutes: 25 environment: release permissions: contents: read # Required for actions/checkout @@ -332,29 +332,7 @@ jobs: repositories: | radius-publisher - - name: Capture DE dispatch start time - if: success() && steps.release-should-skip.outputs.result == 'false' - id: de-dispatch-start - shell: bash - run: | - echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" - - - name: Dispatch Deployment Engine image publish - if: success() && steps.release-should-skip.outputs.result == 'false' - uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1 - with: - token: ${{ steps.get-de-token.outputs.token }} - repository: azure-octo/radius-publisher - event-type: deployment-engine - client-payload: |- - { - "source_repository": "${{ github.repository }}", - "src_image": "radiusdeploymentengine.azurecr.io/deployment-engine", - "dest_image": "ghcr.io/radius-project/deployment-engine", - "tag": "${{ steps.get-version.outputs.release-channel }}" - } - - - name: Monitor DE image publish workflow + - name: Dispatch or monitor correlated DE publisher run if: success() && steps.release-should-skip.outputs.result == 'false' id: monitor-de-workflow uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -367,8 +345,19 @@ jobs: INPUT_OWNER: azure-octo INPUT_REPO: radius-publisher INPUT_WORKFLOW_FILE: publish-deployment-engine.yml - INPUT_DISPATCH_STARTED_AT: ${{ steps.de-dispatch-start.outputs.started_at }} - INPUT_MAX_WAIT_SECONDS: "600" + INPUT_EVENT_TYPE: deployment-engine + INPUT_RELEASE_IDENTIFIER: ${{ steps.get-version.outputs.release-version }}-${{ github.sha }} + # toJSON emits each value already quoted and escaped, so no computed + # version string can break out of the payload it is placed in. + INPUT_CLIENT_PAYLOAD: |- + { + "source_repository": ${{ toJSON(github.repository) }}, + "source_sha": ${{ toJSON(github.sha) }}, + "src_image": "radiusdeploymentengine.azurecr.io/deployment-engine", + "dest_image": "ghcr.io/radius-project/deployment-engine", + "tag": ${{ toJSON(steps.get-version.outputs.release-channel) }} + } + INPUT_MAX_WAIT_SECONDS: "720" INPUT_POLL_INTERVAL_SECONDS: "15" - name: Show failed DE publish logs diff --git a/build/test.mk b/build/test.mk index 2b297d5d16..7983cda6ca 100644 --- a/build/test.mk +++ b/build/test.mk @@ -53,7 +53,7 @@ GOTEST_OPTS ?= GOTEST_TOOL ?= go tool gotestsum $(GOTESTSUM_OPTS) -- .PHONY: test -test: test-get-envtools test-helm test-manage-radius-installation test-release-parity-manifest test-verify-goreleaser-snapshot test-changelog-range test-changelog-config test-build-summary test-goreleaser-shadow test-capture-release-image-digests test-release-get-version test-release-tag-and-branch ## Runs unit tests, excluding kubernetes controller tests +test: test-get-envtools test-helm test-manage-radius-installation test-release-parity-manifest test-verify-goreleaser-snapshot test-changelog-range test-changelog-config test-build-summary test-goreleaser-shadow test-capture-release-image-digests test-release-get-version test-release-tag-and-branch test-monitor-remote-workflow ## Runs unit tests, excluding kubernetes controller tests KUBEBUILDER_ASSETS="$(shell $(ENV_SETUP) use -p path ${K8S_VERSION} --arch amd64)" CGO_ENABLED=1 $(GOTEST_TOOL) ./pkg/... ./test/validation/... $(GOTEST_OPTS) .PHONY: test-manage-radius-installation @@ -96,6 +96,10 @@ test-release-tag-and-branch: ## Tests release tag and branch reconciliation test-release-get-version: ## Tests release version selection across repositories @bash ./.github/scripts/release-get-version_test.sh +.PHONY: test-monitor-remote-workflow +test-monitor-remote-workflow: ## Tests exact remote workflow dispatch correlation + @node --test ./.github/scripts/monitor-remote-workflow_test.mjs + .PHONY: test-compile test-compile: test-get-envtools ## Compiles all tests without running them @echo "$(ARROW) Compiling unit tests..." diff --git a/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md b/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md index 3650bde32b..52c392dba8 100644 --- a/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md +++ b/eng/design-notes/tools/2026-09-goreleaser-stack-review/README.md @@ -4,17 +4,18 @@ This directory records the review of the 18-pull-request stack that implements t ## Review notes by pull request -| PR | Branch | Note | -|----|-----------------------------------------|----------------------------------------------------------------------------| -| 1 | `dp/goreleaser-parity-manifest` | [pr-01-parity-manifest.md](./pr-01-parity-manifest.md) | -| 2 | `dp/goreleaser-snapshot-ci` | [pr-02-goreleaser-snapshot.md](./pr-02-goreleaser-snapshot.md) | -| 3 | `dp/conventional-commit-title-check` | [pr-03-conventional-commit-title.md](./pr-03-conventional-commit-title.md) | -| 4 | `dp/git-cliff-changelog-bootstrap` | [pr-04-git-cliff-changelog.md](./pr-04-git-cliff-changelog.md) | -| 5 | `dp/split-build-workflows` | [pr-05-split-build-workflows.md](./pr-05-split-build-workflows.md) | -| 6 | `dp/edge-tags-deprecation` | [pr-06-edge-tags.md](./pr-06-edge-tags.md) | -| 7 | `dp/goreleaser-shadow-release` | [pr-07-goreleaser-shadow.md](./pr-07-goreleaser-shadow.md) | -| 8 | `dp/conventional-commit-title-required` | [pr-08-title-check-required.md](./pr-08-title-check-required.md) | -| 9 | `dp/idempotent-tag-reconciliation` | [pr-09-tag-reconciliation.md](./pr-09-tag-reconciliation.md) | +| PR | Branch | Note | +|----|-----------------------------------------|--------------------------------------------------------------------------------------| +| 1 | `dp/goreleaser-parity-manifest` | [pr-01-parity-manifest.md](./pr-01-parity-manifest.md) | +| 2 | `dp/goreleaser-snapshot-ci` | [pr-02-goreleaser-snapshot.md](./pr-02-goreleaser-snapshot.md) | +| 3 | `dp/conventional-commit-title-check` | [pr-03-conventional-commit-title.md](./pr-03-conventional-commit-title.md) | +| 4 | `dp/git-cliff-changelog-bootstrap` | [pr-04-git-cliff-changelog.md](./pr-04-git-cliff-changelog.md) | +| 5 | `dp/split-build-workflows` | [pr-05-split-build-workflows.md](./pr-05-split-build-workflows.md) | +| 6 | `dp/edge-tags-deprecation` | [pr-06-edge-tags.md](./pr-06-edge-tags.md) | +| 7 | `dp/goreleaser-shadow-release` | [pr-07-goreleaser-shadow.md](./pr-07-goreleaser-shadow.md) | +| 8 | `dp/conventional-commit-title-required` | [pr-08-title-check-required.md](./pr-08-title-check-required.md) | +| 9 | `dp/idempotent-tag-reconciliation` | [pr-09-tag-reconciliation.md](./pr-09-tag-reconciliation.md) | +| 10 | `dp/release-identifier-correlation` | [pr-10-release-identifier-correlation.md](./pr-10-release-identifier-correlation.md) | Later notes are added as the review progresses up the stack. diff --git a/eng/design-notes/tools/2026-09-goreleaser-stack-review/pr-10-release-identifier-correlation.md b/eng/design-notes/tools/2026-09-goreleaser-stack-review/pr-10-release-identifier-correlation.md new file mode 100644 index 0000000000..9efa580d22 --- /dev/null +++ b/eng/design-notes/tools/2026-09-goreleaser-stack-review/pr-10-release-identifier-correlation.md @@ -0,0 +1,33 @@ +# Review note: PR 10 - Release-identifier correlation for remote dispatch + +- **Pull request**: [#12783](https://github.com/radius-project/radius/pull/12783) +- **Plan phase**: [PR 10](../2026-03-goreleaser-release-lifecycle-implementation-plan.md#pr-10-release-identifier-correlation-for-remote-dispatch) +- **Stack index**: [README](./README.md) + +## Verdict + +The layer does what the plan asks and implements the design's idempotency rule for remote publishers. Each caller hands the publisher a stable identifier, and the monitor looks for the exact run name before it dispatches anything: an existing successful run is reused, an active run is monitored to completion, a failed run is retried by a new dispatch under the same identifier, and only an absent run leads to a dispatch. The companion publisher change ([azure-octo/radius-publisher#25](https://github.com/azure-octo/radius-publisher/pull/25)) renders `run-name` as ` / ` and keys its concurrency groups by the same identifier without cancellation, which is exactly the string and the behavior the monitor expects; dispatches without an identifier fall back to the publisher run id, so they can never collide with a release. The identifier choices fit each caller: version plus source commit for the release and Bicep dispatches, so a rerun at the same commit reuses the publication, and tag plus run id for the Deployment Engine bridge, whose dispatches must publish every time. + +Discovery walks the publisher's whole `repository_dispatch` history under the monitor's deadline instead of a page cap, which is the right trade: the earlier five-page bound assumed that one identifier's runs are contiguous, and a retry days later is not. Today that costs seven requests for the Bicep publisher (607 retained runs) and two for the Deployment Engine publisher (186), well inside the budget of an App token. Payload values are emitted with `toJSON`, so an externally supplied image name, tag, or ref cannot break out of the JSON it is placed in. Transient API failures are retried with randomized exponential backoff inside one total budget, an uncertain dispatch is reconciled before any retry, and the job timeouts (18 and 25 minutes) sit above the 12-minute monitor budget. The hermetic scenarios cover reuse, retry, races, pagination, timeouts, and the caller wiring; Prettier and actionlint are clean. + +## Changes made in this review + +### 1. Failures name the call that failed + +- **What changed**: an error that ends the monitor now says which operation failed and after how many attempts. A dispatch that exhausts its retries also says that no run with the expected name appeared and that a rerun reconciles before dispatching again, and the lookup-timeout message says that the monitor budget ran out before every history page was read. Two scenarios pin the exhausted-lookup and exhausted-dispatch messages, and the non-retryable case asserts the full text. +- **Why**: the raw GitHub error ("Not Found", "Bad Gateway") was the whole job annotation. It did not say whether the run lookup, the dispatch, or a status poll had failed, nor whether the dispatch may have been accepted, while the design asks every error summary to name the stage, the observed state, and the recovery action. +- **Value**: a release engineer reads the annotation and knows what to check and that rerunning the job is safe. +- **Impact**: messages only, no control-flow change; 19 scenarios pass. + +## Findings left as-is + +- **Retry by dispatch**: the design's idempotency contract says the controller "reruns a failed retryable run"; this layer dispatches a new run under the same identifier instead of calling the rerun API. The outcome is the same, it needs no `actions: write` on the publisher, which the App token does not hold, and a rerun would pin the publisher's old workflow definition. The design text needs no change. +- **Full-history discovery**: bounded by the monitor deadline, and measured above at seven and two requests per dispatch. A `created` window would cut this but would make a late resume miss its run; not worth it at this size. +- **`Capture release metadata` step in `__build-bicep-types.yaml`**: the values it copies are also available through the `env` context, so the step could go, but PR 18 edits the surrounding lines and the step keeps the identifier expression explicit in one place. +- **Node version in the unit-test job**: `make test` runs `node --test` with the runner image's Node 22 rather than the Node 24 pinned in `.node-version`; `node:test` behaves the same on both, and the lint job pins the version for Prettier. +- **Companion ordering**: the publisher pull request must be deployed before this layer merges, as the pull request body says; the publisher's current run titles are still the plain event names. + +## Verification + +- 19 scenarios pass; Prettier with the repository configuration passes for the script, the test, and the three workflows; actionlint passes for the three workflows. +- The companion pull request's head was read for `run-name`, the dispatch `types`, and `concurrency`; the history sizes come from the workflow-runs API of the publisher repository.