Skip to content

Commit b73e648

Browse files
aravelo7aravelo7
committed
fix(ci): align release checks with AgentPatchCheck
1 parent 746fba7 commit b73e648

8 files changed

Lines changed: 249 additions & 146 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,6 @@ jobs:
6262
fail-fast: false
6363
matrix:
6464
include:
65-
- os: ubuntu-latest
66-
node-version: 20
6765
- os: ubuntu-latest
6866
node-version: 22
6967
# - os: windows-latest

‎scripts/windows-release-artifact-smoke.mjs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { spawn } from "node:child_process";
2-
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
2+
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
33
import { tmpdir } from "node:os";
44
import { basename, dirname, join, resolve } from "node:path";
55

@@ -9,6 +9,8 @@ if (process.platform !== "win32") {
99
}
1010

1111
const root = resolve(import.meta.dirname, "..");
12+
const packageName = JSON.parse(await readFile(join(root, "package.json"), "utf8")).name;
13+
if (typeof packageName !== "string" || packageName.length === 0) throw new Error("package.json must declare a package name.");
1214

1315
function quoteForCmd(value) {
1416
return `"${value.replaceAll('"', '""')}"`;
@@ -81,7 +83,7 @@ try {
8183
hiddenOracle: { script: "oracle.mjs", timeoutMs: 5_000, isolation: "process", memoryLimitBytes: 134_217_728, cpuRatePercent: 25 },
8284
}), "utf8");
8385
await npm(["install", "--ignore-scripts", "--no-audit", "--no-fund", "--package-lock=false", "--prefix", consumer, tarballPath], { cwd: smokeRoot });
84-
const cli = join(consumer, "node_modules", "kanban", "dist", "agentpatchcheck.js");
86+
const cli = join(consumer, "node_modules", packageName, "dist", "agentpatchcheck.js");
8587
const executed = await requireSuccess(process.execPath, [cli, "run", "--task-spec", join(specDirectory, "task.json")], { cwd: smokeRoot });
8688
const response = JSON.parse(executed.stdout);
8789
const isolation = response.data?.hiddenOracle?.isolation;

‎src/agentpatchcheck/swebench-cli.ts‎

Lines changed: 62 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { randomUUID } from "node:crypto";
21
import { spawn } from "node:child_process";
2+
import { randomUUID } from "node:crypto";
33
import { constants } from "node:fs";
44
import { access, mkdir, open, readFile, stat, unlink, writeFile } from "node:fs/promises";
55
import { dirname, join, resolve } from "node:path";
@@ -89,7 +89,12 @@ interface SWEbenchBootstrapManifest {
8989
name: string;
9090
version: string;
9191
fullSubset: { path: string; sha256?: string };
92-
evaluator: { dataset: string; revision: string; timeoutSeconds: number; metadataAuthority?: SWEbenchEvaluatorMetadataAuthority };
92+
evaluator: {
93+
dataset: string;
94+
revision: string;
95+
timeoutSeconds: number;
96+
metadataAuthority?: SWEbenchEvaluatorMetadataAuthority;
97+
};
9398
execution: { classification: "formal-frozen" | "engineering-validation"; deepseekModel: string };
9499
instanceIds: string[];
95100
sourceDataset?: SWEbenchSourceDatasetIdentity;
@@ -141,7 +146,8 @@ const defaultSWEbenchEvaluatorRevisionProbe: SWEbenchEvaluatorRevisionProbe = as
141146
await getGitStdout(["-c", `safe.directory=${resolve(sourceRoot)}`, "rev-parse", "HEAD"], resolve(sourceRoot));
142147

143148
function requireManifestString(value: unknown, field: string): string {
144-
if (typeof value !== "string" || !value.trim()) throw new Error(`SWE-bench bootstrap manifest ${field} is required.`);
149+
if (typeof value !== "string" || !value.trim())
150+
throw new Error(`SWE-bench bootstrap manifest ${field} is required.`);
145151
return value.trim();
146152
}
147153

@@ -158,11 +164,13 @@ function resolveManifestPath(manifestDirectory: string, value: unknown, field: s
158164

159165
function parseSourceDatasetIdentity(value: unknown): SWEbenchSourceDatasetIdentity | undefined {
160166
if (value === undefined) return undefined;
161-
if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error("SWE-bench bootstrap manifest sourceDataset must be an object.");
167+
if (value === null || typeof value !== "object" || Array.isArray(value))
168+
throw new Error("SWE-bench bootstrap manifest sourceDataset must be an object.");
162169
const source = value as Record<string, unknown>;
163170
const revision = requireManifestString(source.revision, "sourceDataset.revision");
164171
const sha256 = requireManifestString(source.sha256, "sourceDataset.sha256");
165-
if (!/^[0-9a-f]{40}$/u.test(revision)) throw new Error("SWE-bench bootstrap sourceDataset revision must be a full commit SHA.");
172+
if (!/^[0-9a-f]{40}$/u.test(revision))
173+
throw new Error("SWE-bench bootstrap sourceDataset revision must be a full commit SHA.");
166174
if (!/^[0-9a-f]{64}$/u.test(sha256)) throw new Error("SWE-bench bootstrap sourceDataset sha256 must be a SHA-256.");
167175
return {
168176
name: requireManifestString(source.name, "sourceDataset.name"),
@@ -174,10 +182,12 @@ function parseSourceDatasetIdentity(value: unknown): SWEbenchSourceDatasetIdenti
174182

175183
function parseMetadataAuthority(value: unknown): SWEbenchEvaluatorMetadataAuthority | undefined {
176184
if (value === undefined) return undefined;
177-
if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error("SWE-bench bootstrap metadataAuthority must be an object.");
185+
if (value === null || typeof value !== "object" || Array.isArray(value))
186+
throw new Error("SWE-bench bootstrap metadataAuthority must be an object.");
178187
const authority = value as Record<string, unknown>;
179188
const revision = requireManifestString(authority.revision, "evaluator.metadataAuthority.revision");
180-
if (!/^[0-9a-f]{40}$/u.test(revision)) throw new Error("SWE-bench bootstrap metadataAuthority revision must be a full commit SHA.");
189+
if (!/^[0-9a-f]{40}$/u.test(revision))
190+
throw new Error("SWE-bench bootstrap metadataAuthority revision must be a full commit SHA.");
181191
return {
182192
dataset: requireManifestString(authority.dataset, "evaluator.metadataAuthority.dataset"),
183193
split: requireManifestString(authority.split, "evaluator.metadataAuthority.split"),
@@ -207,18 +217,26 @@ export async function loadSWEbenchBootstrapConfiguration(
207217
const manifestVersion = requireManifestString(manifest.version, "version");
208218
if (manifest.fullSubset === undefined || manifest.evaluator === undefined || manifest.execution === undefined)
209219
throw new Error("SWE-bench bootstrap manifest is missing its dataset, evaluator, or execution contract.");
210-
if (!Array.isArray(manifest.instanceIds) || manifest.instanceIds.some((instanceId) => typeof instanceId !== "string"))
220+
if (
221+
!Array.isArray(manifest.instanceIds) ||
222+
manifest.instanceIds.some((instanceId) => typeof instanceId !== "string")
223+
)
211224
throw new Error("SWE-bench bootstrap manifest instanceIds must be an array of strings.");
212225
const revision = requireManifestString(manifest.evaluator.revision, "evaluator.revision");
213-
if (!/^[0-9a-f]{40}$/u.test(revision)) throw new Error("SWE-bench bootstrap evaluator revision must be a full commit SHA.");
226+
if (!/^[0-9a-f]{40}$/u.test(revision))
227+
throw new Error("SWE-bench bootstrap evaluator revision must be a full commit SHA.");
214228
if (!Number.isSafeInteger(manifest.evaluator.timeoutSeconds) || manifest.evaluator.timeoutSeconds <= 0)
215229
throw new Error("SWE-bench bootstrap evaluator timeoutSeconds must be a positive integer.");
216-
if (manifest.execution.classification !== "formal-frozen" && manifest.execution.classification !== "engineering-validation")
230+
if (
231+
manifest.execution.classification !== "formal-frozen" &&
232+
manifest.execution.classification !== "engineering-validation"
233+
)
217234
throw new Error("SWE-bench bootstrap execution classification is invalid.");
218235
const sourceDataset = parseSourceDatasetIdentity(manifest.sourceDataset);
219236
const metadataAuthority = parseMetadataAuthority(manifest.evaluator.metadataAuthority);
220237
const sourceDatasetSha256 = requireManifestString(manifest.fullSubset.sha256, "fullSubset.sha256");
221-
if (!/^[0-9a-f]{64}$/u.test(sourceDatasetSha256)) throw new Error("SWE-bench bootstrap fullSubset sha256 must be a SHA-256.");
238+
if (!/^[0-9a-f]{64}$/u.test(sourceDatasetSha256))
239+
throw new Error("SWE-bench bootstrap fullSubset sha256 must be a SHA-256.");
222240
const manifestDirectory = dirname(manifestPath);
223241
return {
224242
manifestPath,
@@ -230,7 +248,9 @@ export async function loadSWEbenchBootstrapConfiguration(
230248
evaluatorTimeoutSeconds: manifest.evaluator.timeoutSeconds,
231249
evaluatorSourceRoot: requireProjectEnvironmentValue(environment, SWE_BENCH_EVALUATOR_ROOT_ENV),
232250
evaluatorPythonPath: requireProjectEnvironmentValue(environment, SWE_BENCH_EVALUATOR_PYTHON_ENV),
233-
deepseekModel: parseDeepSeekV4Model(requireManifestString(manifest.execution.deepseekModel, "execution.deepseekModel")),
251+
deepseekModel: parseDeepSeekV4Model(
252+
requireManifestString(manifest.execution.deepseekModel, "execution.deepseekModel"),
253+
),
234254
classification: manifest.execution.classification,
235255
engineeringValidation: manifest.execution.classification === "engineering-validation",
236256
instanceIds: [...manifest.instanceIds],
@@ -297,7 +317,10 @@ export async function preflightSWEbenchEvaluator(
297317
}
298318
if (failedChecks.length === 0 && !(await dependencyProbe(evaluatorPythonPath)))
299319
failedChecks.push("evaluator-python-docker-module-unavailable");
300-
if (!failedChecks.includes("evaluator-source-root-missing") && !failedChecks.includes("evaluator-source-root-not-directory")) {
320+
if (
321+
!failedChecks.includes("evaluator-source-root-missing") &&
322+
!failedChecks.includes("evaluator-source-root-not-directory")
323+
) {
301324
try {
302325
actualRevision = (await revisionProbe(evaluatorSourceRoot)).trim();
303326
if (actualRevision !== input.expectedRevision) failedChecks.push("evaluator-revision-mismatch");
@@ -372,9 +395,12 @@ const CONFIGURED_OPTIONS = new Set([
372395
function assertCanonicalOperatorArguments(argv: string[]): void {
373396
for (let index = 0; index < argv.length; index += 2) {
374397
const name = argv[index];
375-
if (name === undefined || !name.startsWith("--")) throw new Error(`Unexpected SWE-bench CLI argument: ${name ?? "<missing>"}`);
398+
if (name === undefined || !name.startsWith("--"))
399+
throw new Error(`Unexpected SWE-bench CLI argument: ${name ?? "<missing>"}`);
376400
if (CONFIGURED_OPTIONS.has(name))
377-
throw new Error(`${name} is owned by the canonical SWE-bench bootstrap configuration and is no longer accepted.`);
401+
throw new Error(
402+
`${name} is owned by the canonical SWE-bench bootstrap configuration and is no longer accepted.`,
403+
);
378404
if (!OPERATOR_OPTIONS.has(name)) throw new Error(`Unknown SWE-bench CLI option: ${name}`);
379405
const value = argv[index + 1];
380406
if (value === undefined || value.startsWith("--")) throw new Error(`Missing value for option: ${name}`);
@@ -472,7 +498,10 @@ function createNotRunGradingResult(result: SWEbenchAdapterResult): SWEbenchGradi
472498
};
473499
}
474500

475-
function deriveCandidateValidity(result: SWEbenchAdapterResult, grading: SWEbenchGradingResult): {
501+
function deriveCandidateValidity(
502+
result: SWEbenchAdapterResult,
503+
grading: SWEbenchGradingResult,
504+
): {
476505
executionValidity: "valid" | "invalid";
477506
pass1Eligible: boolean;
478507
predictionStatus: "generated" | "not_generated";
@@ -485,7 +514,8 @@ function deriveCandidateValidity(result: SWEbenchAdapterResult, grading: SWEbenc
485514
pass1Eligible: executionValidity === "valid",
486515
predictionStatus: result.prediction === null ? "not_generated" : "generated",
487516
gradingValidity:
488-
grading.normalizedStatus === "infrastructure_error" || grading.normalizedStatus === "grading_error_or_ambiguous"
517+
grading.normalizedStatus === "infrastructure_error" ||
518+
grading.normalizedStatus === "grading_error_or_ambiguous"
489519
? "grading_invalid"
490520
: grading.normalizedStatus === "not_run"
491521
? "not_run"
@@ -578,7 +608,9 @@ export async function runSWEbenchCli(
578608
const bootstrap = await resolvedDependencies.loadBootstrapConfiguration(projectRoot);
579609
const options = parseOptions(argv);
580610
if (!bootstrap.instanceIds.includes(options.instance))
581-
throw new Error(`SWE-bench instance is not part of ${bootstrap.manifestName} ${bootstrap.manifestVersion}: ${options.instance}`);
611+
throw new Error(
612+
`SWE-bench instance is not part of ${bootstrap.manifestName} ${bootstrap.manifestVersion}: ${options.instance}`,
613+
);
582614
const outputVariant = options.variant ?? `${bootstrap.manifestName}-${bootstrap.manifestVersion}`;
583615
const outputDirectory = resolve(
584616
projectRoot,
@@ -611,7 +643,11 @@ export async function runSWEbenchCli(
611643
bootstrap.engineeringValidation,
612644
);
613645
const instance = await resolvedDependencies.loadInstance(bootstrap.datasetPath, options.instance);
614-
const repositoryRoot = await resolvedDependencies.resolveRepositoryRoot(projectRoot, instance, resolvedDependencies.getGitStdout);
646+
const repositoryRoot = await resolvedDependencies.resolveRepositoryRoot(
647+
projectRoot,
648+
instance,
649+
resolvedDependencies.getGitStdout,
650+
);
615651
const runtime = createSWEbenchRuntimeConfiguration(bootstrap.deepseekModel);
616652
if (bootstrap.engineeringValidation) {
617653
runtime.dockerTaskEnvironment = {
@@ -622,7 +658,7 @@ export async function runSWEbenchCli(
622658
instanceImageTag: "latest",
623659
},
624660
};
625-
};
661+
}
626662
const result = await resolvedDependencies.runInstance({
627663
instance,
628664
repositoryRoot,
@@ -641,7 +677,8 @@ export async function runSWEbenchCli(
641677
grading = createNotRunGradingResult(result);
642678
} else {
643679
try {
644-
if (bootstrap.sourceDataset === undefined) throw new Error("SWE-bench bootstrap sourceDataset provenance is required for grading.");
680+
if (bootstrap.sourceDataset === undefined)
681+
throw new Error("SWE-bench bootstrap sourceDataset provenance is required for grading.");
645682
const preparedEvaluatorDataset = await resolvedDependencies.prepareEvaluatorDataset({
646683
sourceDatasetPath: bootstrap.evaluatorDatasetPath,
647684
sourceDatasetSha256: bootstrap.sourceDatasetSha256,
@@ -666,7 +703,10 @@ export async function runSWEbenchCli(
666703
version: 2,
667704
instanceId: result.instance.instance_id,
668705
normalizedStatus: "infrastructure_error",
669-
reason: preparedEvaluatorDatasetPath === null ? "evaluator_input_preparation_failed" : "post_run_evaluator_invocation_failed",
706+
reason:
707+
preparedEvaluatorDatasetPath === null
708+
? "evaluator_input_preparation_failed"
709+
: "post_run_evaluator_invocation_failed",
670710
officialReportPath: null,
671711
officialRunId: null,
672712
evaluatorVersion: evaluator.evaluatorVersion,

‎test/agentpatchcheck/benchmark-runner.test.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,11 @@ function createResult(options: {
8888

8989
describe("Benchmark Runner", () => {
9090
it("writes reports beside the runner-managed evidence and runtime roots, not below the BenchmarkSpec", () => {
91-
expect(getBenchmarkReportPath("D:\\runtime\\b2-c01\\worktrees", "benchmark-output-boundary")).toBe(
92-
"D:\\runtime\\b2-c01\\benchmarks\\benchmark-output-boundary.json",
91+
const worktreeRoot = "D:\\runtime\\b2-c01\\worktrees";
92+
expect(getBenchmarkReportPath(worktreeRoot, "benchmark-output-boundary")).toBe(
93+
process.platform === "win32"
94+
? join("D:\\runtime\\b2-c01", "benchmarks", "benchmark-output-boundary.json")
95+
: join("benchmarks", "benchmark-output-boundary.json"),
9396
);
9497
});
9598

‎test/agentpatchcheck/cline-runtime-adapter.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ describe("Cline control runtime adapter", () => {
4242

4343
const captured = capturedConfigurations[0];
4444
if (captured === undefined) throw new Error("Cline runtime factory was not called.");
45-
expect(captured).toMatchObject({ providerId: "openai-native", modelId: "test-model", maxIterations: 12 });
45+
expect(captured).toMatchObject({ providerId: "openai-native", modelId: "test-model", maxIterations: 24 });
4646
expect(captured.tools?.map((tool) => tool.name)).toContain("finish");
4747
expect(captured.tools?.map((tool) => tool.name)).toContain("apply-patch");
4848
expect(result).toMatchObject({

‎test/agentpatchcheck/harness-native-repetition-report.test.mjs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,14 @@ describe("Harness-native repetition report", () => {
6767
passedRuns: 1,
6868
publicVerificationFalsePositives: 0,
6969
statusCounts: { "hidden-oracle-error": 1, passed: 1 },
70+
failureClassification: { byExecution: {}, byCompletion: {}, bySemantic: {} },
7071
},
7172
{
7273
id: "exact",
7374
passedRuns: 1,
7475
publicVerificationFalsePositives: 1,
7576
statusCounts: { "hidden-oracle-failed": 1, passed: 1 },
77+
failureClassification: { byExecution: {}, byCompletion: {}, bySemantic: {} },
7678
},
7779
]);
7880
expect(sumNativeQuality(outputs)).toMatchObject({ nativeTasks: 4, hiddenOraclePassed: 2 });

‎test/agentpatchcheck/runtime-record.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,9 @@ describe("Harness-native durable Runtime record", () => {
128128
};
129129

130130
const record = await HarnessNativeRuntimeRecord.open({ path: recordPath, identity, worktree });
131-
expect((await loadHarnessNativeRuntimeRecord(recordPath)).header.initialWorktreeSha256).toBe("initial-fingerprint");
131+
expect((await loadHarnessNativeRuntimeRecord(recordPath)).header.initialWorktreeSha256).toBe(
132+
"initial-fingerprint",
133+
);
132134
new HarnessNativeRuntimeEventSpine([], record).append({
133135
version: 1,
134136
attempt: 1,

0 commit comments

Comments
 (0)