diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 83d70e9ed7..8c7a056d5a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,7 +21,8 @@ jobs: name: CodeQL (${{ matrix.language }}) runs-on: ubuntu-latest permissions: - actions: read + # Merge-group uploads can cancel this run after verifying that its candidate expired. + actions: write contents: read security-events: write strategy: @@ -32,6 +33,7 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ github.sha }} persist-credentials: false - name: Initialize CodeQL @@ -41,6 +43,38 @@ jobs: build-mode: none - name: Perform CodeQL Analysis + id: analysis uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: category: "/language:${{ matrix.language }}" + # A literal value is required by the action's failed-analysis diagnostic recovery. + upload: failure-only + post-processed-sarif-path: ${{ runner.temp }}/codeql-upload + + - name: Publish CodeQL results + if: github.event_name != 'merge_group' + uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + sarif_file: ${{ runner.temp }}/codeql-upload/upload.sarif + category: "/language:${{ matrix.language }}" + + - name: Publish merge-group CodeQL results + if: github.event_name == 'merge_group' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + timeout-minutes: 8 + env: + ANALYSIS_OUTCOME: ${{ steps.analysis.outcome }} + SARIF_FILE: ${{ runner.temp }}/codeql-upload/upload.sarif + SARIF_CATEGORY: "/language:${{ matrix.language }}" + with: + retries: 0 + script: | + const publish = require(require('node:path').resolve('build_scripts', 'codeql_merge_group.cjs')); + await publish({ + github, context, core, + sarifFile: process.env.SARIF_FILE, + category: process.env.SARIF_CATEGORY, + analysisOutcome: process.env.ANALYSIS_OUTCOME, + runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT), + checkout: process.env.GITHUB_WORKSPACE, + }); diff --git a/build_scripts/codeql_merge_group.cjs b/build_scripts/codeql_merge_group.cjs new file mode 100644 index 0000000000..30765f6acb --- /dev/null +++ b/build_scripts/codeql_merge_group.cjs @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// @ts-check + +const { readFile } = require("node:fs/promises"); +const { setTimeout: delay } = require("node:timers/promises"); +const { gzipSync } = require("node:zlib"); +const { pathToFileURL } = require("node:url"); + +/** @param {unknown} value @returns {value is Record} */ +function isObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** @param {unknown} value @returns {Record} */ +function object(value) { + if (!isObject(value)) throw new Error("Invalid GitHub API or SARIF object"); + return value; +} + +/** + * @typedef {{repo: {owner: string, repo: string}, eventName: string, ref: string, sha: string, + * runId: number, payload: {action?: string, merge_group?: {base_ref?: string, head_ref?: string, head_sha?: string}}}} Context + * @typedef {{request: (route: string, options: Record) => + * Promise<{status: number, data: unknown}>}} GitHub + * @param {{github: GitHub, context: Context, core: {notice: (message: string) => void, info: (message: string) => void}, + * sarifFile: string, category: string, analysisOutcome: string, runAttempt: number, + * checkout: string, sleep?: (milliseconds: number) => Promise}} options + */ +module.exports = async function publish({ + github, context, core, sarifFile, category, analysisOutcome, runAttempt, checkout, sleep = delay +}) { + const group = context.payload.merge_group; + const queuePrefix = `refs/heads/gh-readonly-queue/${group?.base_ref?.slice(11)}/`; + if (analysisOutcome !== "success") throw new Error("Publishing requires successful CodeQL analysis"); + if ( + context.eventName !== "merge_group" || context.payload.action !== "checks_requested" || + !group?.base_ref?.startsWith("refs/heads/") || !context.ref.startsWith(queuePrefix) || context.ref === queuePrefix || + group.head_ref !== context.ref || group.head_sha !== context.sha || !/^[0-9a-f]{40}$/.test(context.sha) || + !Number.isSafeInteger(context.runId) || context.runId <= 0 || !Number.isSafeInteger(runAttempt) || runAttempt <= 0 + ) { + throw new Error("The merge-group context does not match the tested queue ref, SHA and run"); + } + const sarif = await readFile(sarifFile); + const document = object(JSON.parse(sarif.toString("utf8"))); + if ( + document.version !== "2.1.0" || !Array.isArray(document.runs) || !document.runs.length || + document.runs.some((value) => { + const run = object(value); + return object(object(run.tool).driver).name !== "CodeQL" || + object(run.automationDetails).id !== `${category}/`; + }) + ) { + throw new Error("Expected prepared CodeQL SARIF matching the matrix language category"); + } + const missingRef = `ref '${context.ref}' not found in this repository`; + const runOptions = { run_id: context.runId }; + + /** @param {string} route @param {Record} [options] @param {number} [status] */ + async function request(route, options = {}, status = 200) { + const response = await github.request(route, { + ...context.repo, ...options, + headers: { "X-GitHub-Api-Version": "2022-11-28" }, + request: { timeout: 30000 } + }); + if (response.status !== status) throw new Error(`Unexpected HTTP ${response.status} for ${route}`); + return response.data; + } + + async function currentSha() { + const refs = await request("GET /repos/{owner}/{repo}/git/matching-refs/{ref}", { ref: context.ref.slice(5) }); + if (!Array.isArray(refs) || refs.some((value) => typeof object(value).ref !== "string")) { + throw new Error("Invalid matching-refs response"); + } + const matches = refs.map(object).filter((value) => value.ref === context.ref); + if (!matches.length) return null; + if (matches.length !== 1) throw new Error("Matching-refs returned multiple exact queue refs"); + const target = object(matches[0].object); + if (target.type !== "commit" || typeof target.sha !== "string" || !/^[0-9a-f]{40}$/.test(target.sha)) { + throw new Error("The queue ref does not point to a valid commit"); + } + return target.sha; + } + + async function verifyRun() { + const run = object(await request("GET /repos/{owner}/{repo}/actions/runs/{run_id}", runOptions)); + const expected = { + id: context.runId, run_attempt: runAttempt, event: "merge_group", head_sha: context.sha, + head_branch: context.ref.slice(11), path: ".github/workflows/codeql.yml" + }; + if (Object.entries(expected).some(([key, value]) => run[key] !== value)) { + throw new Error("Refusing to cancel a different workflow run or attempt"); + } + if (run.status === "completed" && run.conclusion === "cancelled") throw new Error("CodeQL run is already cancelled"); + if (run.status !== "in_progress" || run.conclusion !== null) { + throw new Error("Refusing to cancel a workflow run that is not in progress"); + } + } + + async function ensureLive() { + const sha = await currentSha(); + if (sha === context.sha) return; + await verifyRun(); + const result = object(await request( + "GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}/jobs", + { ...runOptions, attempt_number: runAttempt, per_page: 100 } + )); + if (!Array.isArray(result.jobs) || !result.jobs.length || result.total_count !== result.jobs.length) { + throw new Error("Cannot verify all CodeQL matrix jobs before cancellation"); + } + for (const value of result.jobs) { + const job = object(value); + if (!Array.isArray(job.steps)) throw new Error("Invalid workflow job steps"); + const conclusions = [job.conclusion, ...job.steps.map((step) => object(step).conclusion)]; + if (conclusions.some((value) => + value != null && value !== "success" && value !== "skipped" && value !== "cancelled" + )) { + throw new Error("A CodeQL matrix job or step failed; refusing to replace failure with cancellation"); + } + } + core.notice(`Verified obsolete merge candidate ${context.ref}: ${sha === null ? "absent" : `now at ${sha}`}`); + try { + await request("POST /repos/{owner}/{repo}/actions/runs/{run_id}/cancel", runOptions, 202); + } catch (error) { + if (isObject(error) && error.status === 409) await verifyRun(); + throw error; + } + // Never finish green while waiting for the runner's cancellation signal. + await sleep(60000); + throw new Error("Cancellation accepted, but the runner did not stop within 60 seconds"); + } + + await ensureLive(); + let accepted; + try { + accepted = object(await request("POST /repos/{owner}/{repo}/code-scanning/sarifs", { + commit_sha: context.sha, + ref: context.ref, + sarif: gzipSync(sarif).toString("base64"), + checkout_uri: pathToFileURL(checkout).href, + tool_name: "CodeQL" + }, 202)); + } catch (error) { + if ( + isObject(error) && (error.status === 400 || error.status === 404) && isObject(error.response) && + isObject(error.response.data) && error.response.data.message === missingRef && + (error.response.data.errors == null || + (Array.isArray(error.response.data.errors) && !error.response.data.errors.length)) + ) { + await ensureLive(); + } + throw error; + } + const sarifId = accepted.id; + if (typeof sarifId !== "string" || !sarifId) throw new Error("SARIF upload accepted without a valid ID"); + for (const milliseconds of [5000, 10000, 20000, 40000, 80000]) { + await sleep(milliseconds); + const result = object(await request( + "GET /repos/{owner}/{repo}/code-scanning/sarifs/{sarif_id}", { sarif_id: sarifId } + )); + if (result.processing_status === "complete") { + await ensureLive(); + core.info(`CodeQL SARIF ${sarifId} processed for ${context.ref} at ${context.sha}`); + return sarifId; + } + if (result.processing_status === "failed") { + if (Array.isArray(result.errors) && result.errors.length === 1 && result.errors[0] === missingRef) { + await ensureLive(); + } + throw new Error(`SARIF processing failed: ${JSON.stringify(result.errors)}`); + } + if (result.processing_status !== "pending") { + throw new Error(`Invalid SARIF processing status: ${result.processing_status}`); + } + await ensureLive(); + } + throw new Error("SARIF processing did not complete within five status checks"); +}; diff --git a/doc/contributing/4_running_tests.md b/doc/contributing/4_running_tests.md index 20f862aa9c..a4d3bd2685 100644 --- a/doc/contributing/4_running_tests.md +++ b/doc/contributing/4_running_tests.md @@ -38,6 +38,35 @@ server, including when the `litellm` or `all` extra is installed. Real adapter and outgoing-header checks run separately as [loopback integration tests](./6_integration_tests.md#litellm-loopback-integration-tests). +### CodeQL merge-group uploads + +CodeQL still analyzes all three languages on PRs, merge groups, main/release pushes, +manual runs and its weekly schedule. Merge groups publish the analyzer's prepared +SARIF through the documented code-scanning API so an expired queue ref can be +distinguished from a genuine upload failure. Categories and fingerprints are +preserved; failed analysis retains CodeQL's native diagnostic upload. +The merge-group helper uses the authenticated client and context supplied by the +pinned `actions/github-script` action, with no separate HTTP client or Python setup. + +Only a successful matching-refs response proving that the exact queue ref is absent +or points to another SHA permits retirement. A missing-ref upload rejection is +rechecked to cover deletion between verification and upload. Authentication, +service, processing and analysis failures stay failures. Retirement cancels the +verified current CodeQL run, never reports a skipped scan as success, and refuses +cancellation when a matrix job has already failed. The CodeQL job needs +`actions: write` for self-cancellation; workflow-wide permissions remain empty. +Fork PRs use native CodeQL publishing and GitHub's read-only fork token policy. + +Run the deterministic, offline regression coverage with Node.js installed. +The pytest entry point runs the Node scenario tests and checks workflow wiring: + +```bash +uv run --frozen pytest tests/unit/build_scripts/test_codeql_merge_group.py -q +``` + +The helper's scenarios can also run directly with +`node --test tests/unit/build_scripts/test_codeql_merge_group.cjs`. + ## Running a subset while iterating For a narrower run, invoke `pytest` directly. You can invoke pytest if it's in your path or via python; either `pytest` or `python -m pytest`. For the following examples, we will use `pytest`. diff --git a/tests/unit/build_scripts/test_codeql_merge_group.cjs b/tests/unit/build_scripts/test_codeql_merge_group.cjs new file mode 100644 index 0000000000..72c2315e09 --- /dev/null +++ b/tests/unit/build_scripts/test_codeql_merge_group.cjs @@ -0,0 +1,255 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +const assert = require("node:assert/strict"); +const { test } = require("node:test"); +const { mkdtemp, writeFile, rm } = require("node:fs/promises"); +const { tmpdir } = require("node:os"); +const path = require("node:path"); +const { gunzipSync } = require("node:zlib"); +const { pathToFileURL } = require("node:url"); +const publish = require(path.resolve(__dirname, "..", "..", "..", "build_scripts", "codeql_merge_group.cjs")); +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; +const script = process.env.CODEQL_TEST_SCRIPT; + +const SHA = "a".repeat(40); +const REF = `refs/heads/gh-readonly-queue/main/pr-42-${SHA}`; +const ID = "sarif-upload-id"; +const CATEGORY = "/language:python"; +const context = { + repo: { owner: "owner", repo: "repository" }, eventName: "merge_group", ref: REF, sha: SHA, runId: 123, + payload: { action: "checks_requested", merge_group: { base_ref: "refs/heads/main", head_ref: REF, head_sha: SHA } } +}; +const routes = { + ref: "GET /repos/{owner}/{repo}/git/matching-refs/{ref}", + run: "GET /repos/{owner}/{repo}/actions/runs/{run_id}", + jobs: "GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}/jobs", + cancel: "POST /repos/{owner}/{repo}/actions/runs/{run_id}/cancel", + upload: "POST /repos/{owner}/{repo}/code-scanning/sarifs", + processing: "GET /repos/{owner}/{repo}/code-scanning/sarifs/{sarif_id}" +}; +const sarif = { + version: "2.1.0", runs: [{ tool: { driver: { name: "CodeQL" } }, automationDetails: { id: `${CATEGORY}/` }, + results: [{ partialFingerprints: { primaryLocationLineHash: "existing-fingerprint" } }] }] +}; +const refs = (sha = SHA) => [{ ref: REF, object: { type: "commit", sha } }]; +const run = (changes = {}) => ({ + id: 123, run_attempt: 1, event: "merge_group", head_sha: SHA, head_branch: REF.slice(11), + path: ".github/workflows/codeql.yml", status: "in_progress", conclusion: null, ...changes +}); +const jobs = () => ({ + total_count: 3, jobs: Array.from({ length: 3 }, () => ({ conclusion: null, steps: [{ conclusion: "success" }] })) +}); +const apiError = (status, message = "Request failed", errors) => + Object.assign(new Error(`${message} - https://docs.github.com/rest`), { status, response: { data: { message, errors } } }); +const missing = () => apiError(404, `ref '${REF}' not found in this repository`); +const cancelled = new Error("runner cancelled"); +const live = [[routes.ref, refs()], [routes.upload, { id: ID }, 202], [routes.processing, { processing_status: "complete" }], + [routes.ref, refs()]]; +const obsolete = () => [[routes.ref, []], [routes.run, run()], [routes.jobs, jobs()]]; +const scenarios = []; +function scenario(name, replies, error, changes = {}) { + scenarios.push({ name, replies, error, ...changes }); +} + +scenario("live matching candidate preserves prepared SARIF", live); +for (const category of ["/language:actions", "/language:javascript-typescript"]) { + scenario(`live ${category} preserves its category`, live, undefined, { + document: { ...sarif, runs: [{ ...sarif.runs[0], automationDetails: { id: `${category}/` } }] }, + options: { category } + }); +} +const otherSha = "b".repeat(40); +scenario("queue name suffix is not the tested head SHA", [ + [routes.ref, refs(otherSha)], ...live.slice(1, 3), [routes.ref, refs(otherSha)] +], undefined, { + context: { sha: otherSha, payload: { ...context.payload, + merge_group: { ...context.payload.merge_group, head_sha: otherSha } } } +}); +for (const current of [[], refs("b".repeat(40)), [{ ref: REF + "-suffix" }]]) { + scenario(`obsolete ref ${JSON.stringify(current)}`, [[routes.ref, current], ...obsolete().slice(1), + [routes.cancel, {}, 202]], cancelled); +} +for (const status of [400, 404]) { + for (const current of [[], refs("b".repeat(40))]) { + scenario(`upload race ${status} ${JSON.stringify(current)}`, [[routes.ref, refs()], + [routes.upload, apiError(status, `ref '${REF}' not found in this repository`)], [routes.ref, current], + ...obsolete().slice(1), [routes.cancel, {}, 202]], cancelled); + } +} +scenario("missing-ref rejection while candidate remains live fails", [[routes.ref, refs()], [routes.upload, missing()], + [routes.ref, refs()]], /not found in this repository/); + +const stages = { + precheck: [[], routes.ref], + upload: [[[routes.ref, refs()]], routes.upload], + processing: [live.slice(0, 2), routes.processing], + recheck: [[[routes.ref, refs()], [routes.upload, missing()]], routes.ref], + run: [obsolete().slice(0, 1), routes.run], + jobs: [obsolete().slice(0, 2), routes.jobs], + cancel: [obsolete(), routes.cancel] +}; +for (const [stage, [prefix, route]] of Object.entries(stages)) { + for (const status of [401, 403, 404, 429, 500, 502, 503]) { + scenario(`${stage} HTTP ${status} remains a failure`, [...prefix, [route, apiError(status)]], /Request failed/); + } + scenario(`${stage} network failure remains a failure`, [...prefix, [route, new Error("network failed")]], /network failed/); +} +for (const error of [apiError(400, "Invalid SARIF"), apiError(404, "Not Found"), + apiError(422, `ref '${REF}' not found in this repository`), + apiError("404", `ref '${REF}' not found in this repository`), + apiError(400, `ref '${REF}' not found in this repository`, ["Invalid SARIF"])]) { + scenario(`genuine upload rejection ${error.message}`, [[routes.ref, refs()], [routes.upload, error]], error); +} +for (const status of ["pending", "complete", "failed"]) { + scenario(`expiry after accepted upload ${status}`, [...live.slice(0, 2), + [routes.processing, { processing_status: status, errors: [`ref '${REF}' not found in this repository`] }], + ...obsolete(), [routes.cancel, {}, 202]], cancelled); +} +for (const errors of [["Invalid SARIF"], [`ref '${REF}' not found in this repository`, "Invalid SARIF"]]) { + scenario(`processing rejection ${errors}`, [...live.slice(0, 2), + [routes.processing, { processing_status: "failed", errors }]], /SARIF processing failed/); +} +scenario("missing-ref processing rejection with live ref fails", [...live.slice(0, 2), + [routes.processing, { processing_status: "failed", errors: [`ref '${REF}' not found in this repository`] }], + [routes.ref, refs()]], /SARIF processing failed/); +scenario("bounded processing timeout", [...live.slice(0, 2), + ...Array.from({ length: 5 }, () => [[routes.processing, { processing_status: "pending" }], [routes.ref, refs()]]).flat()], + /five status checks/, { delays: [5000, 10000, 20000, 40000, 80000] }); +scenario("accepted cancellation cannot return success", [...obsolete(), [routes.cancel, {}, 202]], + /runner did not stop within 60 seconds/, { signal: false, delays: [60000] }); +scenario("already cancelled workflow", [[routes.ref, []], [routes.run, run({ status: "completed", conclusion: "cancelled" })]], + /already cancelled/); +scenario("cancellation conflict verifies existing cancellation", [...obsolete(), [routes.cancel, apiError(409)], + [routes.run, run({ status: "completed", conclusion: "cancelled" })]], /already cancelled/); +scenario("unexplained cancellation conflict stays failure", [...obsolete(), [routes.cancel, apiError(409)], + [routes.run, run()]], /Request failed/); +for (const change of [{ id: 999 }, { run_attempt: 2 }, { event: "push" }, { head_sha: "b".repeat(40) }, + { head_branch: "main" }, { path: ".github/workflows/other.yml" }, + { status: "completed", conclusion: "failure" }, { status: "completed", conclusion: "success" }]) { + scenario(`cannot cancel unrelated or completed run ${JSON.stringify(change)}`, [[routes.ref, []], [routes.run, run(change)]], + /Refusing to cancel/); +} +for (const conclusion of ["failure", "timed_out", "startup_failure", "action_required"]) { + for (const location of ["job", "step"]) { + const result = jobs(); + if (location === "job") result.jobs[0].conclusion = conclusion; + else result.jobs[0].steps[0].conclusion = conclusion; + scenario(`${location} ${conclusion} cannot be hidden by retirement`, + [[routes.ref, []], [routes.run, run()], [routes.jobs, result]], /refusing to replace failure/); + } +} +for (const result of [{ total_count: 101, jobs: jobs().jobs }, { total_count: 0, jobs: [] }, + { total_count: 1, jobs: [{ steps: {} }] }, { total_count: 1, jobs: [{ steps: [null] }] }]) { + scenario(`invalid job listing ${JSON.stringify(result)}`, [[routes.ref, []], [routes.run, run()], [routes.jobs, result]], + /Cannot verify|Invalid/); +} +for (const outcome of ["failure", "cancelled", "skipped", ""]) { + scenario(`analysis outcome ${outcome} cannot publish or cancel`, [], /successful CodeQL analysis/, + { options: { analysisOutcome: outcome } }); +} +for (const change of [{ eventName: "push" }, { ref: "refs/heads/main" }, { sha: "b".repeat(40) }, { runId: 0 }, + { eventName: "pull_request" }, { eventName: "schedule" }, { eventName: "workflow_dispatch" }, + { payload: { action: "destroyed" } }, + { ref: "refs/heads/gh-readonly-queue/main/", payload: { ...context.payload, + merge_group: { ...context.payload.merge_group, head_ref: "refs/heads/gh-readonly-queue/main/" } } }]) { + scenario(`invalid context ${JSON.stringify(change)}`, [], /merge-group context/, { context: change }); +} +scenario("invalid attempt cannot cancel another run", [], /merge-group context/, { options: { runAttempt: 0 } }); +for (const result of [null, {}, [{ ref: REF, object: null }], [...refs(), ...refs()], + [{ ref: REF, object: { type: "tag", sha: SHA } }], [{ ref: REF, object: { type: "commit", sha: "short" } }]]) { + scenario(`invalid ref response ${JSON.stringify(result)}`, [[routes.ref, result]], /Invalid|multiple|valid commit/); +} +for (const document of [[], { version: "2.1.0", runs: [] }, { version: "2.1.0", runs: [{ tool: null }] }, + { ...sarif, runs: [{ ...sarif.runs[0], automationDetails: { id: "/language:actions/" } }] }]) { + scenario(`invalid prepared SARIF ${JSON.stringify(document)}`, [], /Invalid|Expected prepared/, { document }); +} +scenario("malformed JSON fails before API calls", [], /JSON/, { raw: "not JSON" }); +scenario("unknown processing state fails", [...live.slice(0, 2), [routes.processing, { processing_status: "other" }]], + /Invalid SARIF processing status/); +for (const result of [{}, { id: "" }, { id: 123 }]) { + scenario(`upload ID must be valid ${JSON.stringify(result)}`, [[routes.ref, refs()], [routes.upload, result, 202]], + /without a valid ID/); +} +scenario("unexpected successful HTTP status fails", [[routes.ref, refs(), 201]], /Unexpected HTTP/); +scenario("runner cancellation does not continue upload", [[routes.ref, refs()], [routes.upload, cancelled]], cancelled); + +for (const item of scenarios) { + test(item.name, async (t) => { + const directory = await mkdtemp(path.join(tmpdir(), "pyrit-codeql-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const file = path.join(directory, "upload.sarif"); + const prepared = item.raw ?? JSON.stringify(item.document ?? sarif); + await writeFile(file, prepared); + const replies = [...item.replies]; + const calls = []; + const delays = []; + const notices = []; + const github = { request: async (route, options) => { + calls.push({ route, options }); + assert.ok(replies.length, `Unexpected API call: ${route}`); + const [expected, data, status = 200] = replies.shift(); + assert.equal(route, expected); + assert.equal(options.owner, context.repo.owner); + assert.equal(options.repo, context.repo.repo); + assert.equal(options.request.timeout, 30000); + if (route === routes.ref) assert.equal(options.ref, REF.slice(5)); + if ([routes.run, routes.jobs, routes.cancel].includes(route)) assert.equal(options.run_id, 123); + if (route === routes.jobs) assert.deepEqual([options.attempt_number, options.per_page], [1, 100]); + if (data instanceof Error) throw data; + return { status, data }; + } }; + const actualContext = { ...context, ...item.context }; + const options = { + github, context: actualContext, core: { notice: (message) => notices.push(message), info: () => {} }, + sarifFile: file, category: CATEGORY, analysisOutcome: "success", runAttempt: 1, checkout: directory, + sleep: async (milliseconds) => { + delays.push(milliseconds); + if (milliseconds === 60000 && item.signal !== false) throw cancelled; + }, ...item.options + }; + let resultId; + const execute = async (args) => { + resultId = await publish({ ...args, sleep: options.sleep }); + return resultId; + }; + const invoke = async () => { + if (!script) return execute(options); + Object.assign(process.env, { + SARIF_FILE: file, SARIF_CATEGORY: options.category, ANALYSIS_OUTCOME: options.analysisOutcome, + GITHUB_RUN_ATTEMPT: String(options.runAttempt), GITHUB_WORKSPACE: directory + }); + const requireForScript = (name) => { + if (name === "node:path") return path; + assert.equal(name, path.resolve("build_scripts", "codeql_merge_group.cjs")); + return execute; + }; + await new AsyncFunction("require", "github", "context", "core", script)( + requireForScript, github, actualContext, options.core + ); + }; + if (item.error) { + await assert.rejects(invoke, item.error instanceof Error ? (error) => error === item.error : item.error); + assert.equal(resultId, undefined); + } else { + await invoke(); + assert.equal(resultId, ID); + assert.deepEqual(delays, [5000]); + assert.equal(notices.length, 0); + } + assert.equal(replies.length, 0, "All expected API calls must be consumed"); + if (item.delays) assert.deepEqual(delays, item.delays); + for (const { route, options } of calls.filter((call) => call.route === routes.upload)) { + assert.equal(route, routes.upload); + assert.equal(options.commit_sha, actualContext.sha); + assert.equal(options.ref, REF); + assert.equal(options.tool_name, "CodeQL"); + assert.equal(options.checkout_uri, pathToFileURL(directory).href); + assert.equal(gunzipSync(Buffer.from(options.sarif, "base64")).toString("utf8"), prepared); + } + if (item.error === cancelled && calls.at(-1)?.route === routes.cancel) { + assert.equal(notices.length, 1); + assert.deepEqual(delays.at(-1), 60000); + } + }); +} diff --git a/tests/unit/build_scripts/test_codeql_merge_group.py b/tests/unit/build_scripts/test_codeql_merge_group.py new file mode 100644 index 0000000000..08be155345 --- /dev/null +++ b/tests/unit/build_scripts/test_codeql_merge_group.py @@ -0,0 +1,103 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Run the real merge-group helper offline and preserve CodeQL workflow contracts.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] +CODEQL_PIN = "2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2" + + +@pytest.fixture(scope="module") +def steps() -> dict[str, Any]: + workflow = yaml.load( + (REPO_ROOT / ".github" / "workflows" / "codeql.yml").read_text(encoding="utf-8"), Loader=yaml.BaseLoader + ) + assert set(workflow["on"]) == {"push", "pull_request", "merge_group", "workflow_dispatch", "schedule"} + assert workflow["on"]["push"]["branches"] == ["main", "releases/v*"] + assert workflow["on"]["pull_request"]["branches"] == ["main", "releases/**"] + assert workflow["on"]["schedule"] == [{"cron": "23 2 * * 1"}] + assert workflow["permissions"] == {} + assert set(workflow["jobs"]) == {"analyze"} + job = workflow["jobs"]["analyze"] + assert job["name"] == "CodeQL (${{ matrix.language }})" + assert job["strategy"] == { + "fail-fast": "false", + "matrix": {"language": ["actions", "javascript-typescript", "python"]}, + } + assert job["permissions"] == {"actions": "write", "contents": "read", "security-events": "write"} + assert "if" not in job and "continue-on-error" not in job + assert all("continue-on-error" not in step for step in job["steps"]) + return {step["name"]: step for step in job["steps"]} + + +def test_codeql_merge_group_scenarios(steps: dict[str, Any]) -> None: + script = steps["Publish merge-group CodeQL results"]["with"]["script"] + with patch.dict("os.environ", {"CODEQL_TEST_SCRIPT": script}): + result = subprocess.run( + ["node", "--test", "--test-reporter=tap", str(Path(__file__).with_suffix(".cjs"))], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + timeout=60, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "# fail 0" in result.stdout and "# skipped 0" in result.stdout + + +def test_codeql_workflow_analysis_and_upload_wiring(steps: dict[str, Any]) -> None: + assert steps["Checkout"]["with"] == {"ref": "${{ github.sha }}", "persist-credentials": "false"} + analyze = steps["Perform CodeQL Analysis"] + assert "if" not in analyze + assert analyze["id"] == "analysis" + assert analyze["uses"] == f"github/codeql-action/analyze@{CODEQL_PIN}" + assert steps["Initialize CodeQL"]["uses"] == f"github/codeql-action/init@{CODEQL_PIN}" + assert steps["Initialize CodeQL"]["with"] == {"languages": "${{ matrix.language }}", "build-mode": "none"} + assert analyze["with"] == { + "category": "/language:${{ matrix.language }}", + "upload": "failure-only", + "post-processed-sarif-path": "${{ runner.temp }}/codeql-upload", + } + native = steps["Publish CodeQL results"] + assert native["uses"] == f"github/codeql-action/upload-sarif@{CODEQL_PIN}" + assert native["with"] == { + "sarif_file": "${{ runner.temp }}/codeql-upload/upload.sarif", + "category": analyze["with"]["category"], + } + publish = steps["Publish merge-group CodeQL results"] + assert publish["uses"] == "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3" + assert publish["with"]["retries"] == "0" + assert publish["timeout-minutes"] == "8" + assert publish["env"] == { + "ANALYSIS_OUTCOME": "${{ steps.analysis.outcome }}", + "SARIF_FILE": "${{ runner.temp }}/codeql-upload/upload.sarif", + "SARIF_CATEGORY": analyze["with"]["category"], + } + script = publish["with"]["script"] + for value in ("codeql_merge_group.cjs", "github, context, core", "await publish(", "process.env.ANALYSIS_OUTCOME"): + assert value in script + assert "process.env.GITHUB_RUN_ATTEMPT" in script + assert "process.env.GITHUB_WORKSPACE" in script + assert "github-token" not in publish["with"] + assert not any("setup-uv@" in step.get("uses", "") for step in steps.values()) + + +@pytest.mark.parametrize("event", ["push", "pull_request", "schedule", "workflow_dispatch", "merge_group"]) +def test_native_and_merge_group_publishers_are_mutually_exclusive(*, steps: dict[str, Any], event: str) -> None: + selected = [] + for name, operator in (("Publish CodeQL results", "!="), ("Publish merge-group CodeQL results", "==")): + assert steps[name]["if"] == f"github.event_name {operator} 'merge_group'" + if (event == "merge_group") == (operator == "=="): + selected.append(name) + assert selected == ["Publish merge-group CodeQL results" if event == "merge_group" else "Publish CodeQL results"] + # Fork PRs retain the action's native token/ref handling, without a privileged PR trigger. + assert not {"ref", "sha", "token"} & steps["Publish CodeQL results"]["with"].keys()