Skip to content

Commit 473fdd9

Browse files
committed
feat(harness): add controlled edit benchmark
1 parent bb83eeb commit 473fdd9

13 files changed

Lines changed: 347 additions & 8 deletions

File tree

‎scripts/run-harness-native-benchmark-suite.mjs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ const providerProfiles = {
2929
requiredEnvironment: "DEEPSEEK_API_KEY",
3030
},
3131
};
32-
const suiteVersions = new Set(["v1", "v2", "v3", "v4"]);
32+
const suiteVersions = new Set(["v1", "v2", "v3", "v4", "v5"]);
3333

3434
function getProviderProfile(profileId) {
3535
const profile = providerProfiles[profileId];
@@ -64,11 +64,11 @@ function parseArguments(argv) {
6464
}
6565
if (outputRoot === undefined || model === undefined)
6666
throw new Error(
67-
"Usage: npm run benchmark:harness-native-suite -- --output-root <new-directory> --model <model> [--provider-profile <profile>] [--suite-version <v1|v2|v3|v4>] [--dry-run]",
67+
"Usage: npm run benchmark:harness-native-suite -- --output-root <new-directory> --model <model> [--provider-profile <profile>] [--suite-version <v1|v2|v3|v4|v5>] [--dry-run]",
6868
);
6969
if (!modelPattern.test(model)) throw new Error("model must be a valid 1-128 character model identifier.");
7070
getProviderProfile(providerProfile);
71-
if (!suiteVersions.has(suiteVersion)) throw new Error("suite-version must be one of: v1, v2, v3, v4.");
71+
if (!suiteVersions.has(suiteVersion)) throw new Error("suite-version must be one of: v1, v2, v3, v4, v5.");
7272
return { outputRoot, model, providerProfile, suiteVersion, dryRun };
7373
}
7474

@@ -140,10 +140,11 @@ async function main() {
140140
const options = parseArguments(process.argv.slice(2));
141141
const providerProfile = getProviderProfile(options.providerProfile ?? "openai-responses");
142142
const fixtureSources =
143-
options.suiteVersion === "v4"
143+
options.suiteVersion === "v4" || options.suiteVersion === "v5"
144144
? [
145145
join(fixtureRoot, "harness-native-benchmark-suite-v3"),
146146
join(fixtureRoot, "harness-native-benchmark-suite-v4-extension"),
147+
...(options.suiteVersion === "v5" ? [join(fixtureRoot, "harness-native-benchmark-suite-v5-extension")] : []),
147148
]
148149
: [join(fixtureRoot, `harness-native-benchmark-suite-${options.suiteVersion}`)];
149150
if (!options.dryRun && !process.env[providerProfile.requiredEnvironment]?.trim())

‎src/agentpatchcheck/harness-native-runtime.ts‎

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ const registeredTools: HarnessNativeToolName[] = [
3333
"git-diff",
3434
"apply-patch",
3535
"apply-patch-batch",
36+
"apply-edit-batch",
3637
"create-file",
3738
];
3839

@@ -172,6 +173,11 @@ interface ConstrainedPatch {
172173
replacementText: string;
173174
}
174175

176+
interface ConstrainedNewFile {
177+
path: string;
178+
content: string;
179+
}
180+
175181
function exactOccurrenceCount(content: string, expectedText: string): number {
176182
return content.split(expectedText).length - 1;
177183
}
@@ -199,12 +205,13 @@ function replaceExactText(
199205
return content.replace(normalizedExpected, normalizeLineEndings(replacementText, lineEnding));
200206
}
201207

202-
async function preparePatchBatch(
208+
async function preparePatches(
203209
root: string,
204210
value: unknown,
211+
options: { minimum: number; maximum: number; failureMessage: string },
205212
): Promise<Array<{ path: string; content: string; replacement: string }>> {
206-
if (!Array.isArray(value) || value.length < 2 || value.length > 8)
207-
throw new Error("Patch batch must contain 2-8 patches.");
213+
if (!Array.isArray(value) || value.length < options.minimum || value.length > options.maximum)
214+
throw new Error(options.failureMessage);
208215
const patches: ConstrainedPatch[] = [];
209216
for (const item of value) {
210217
if (item === null || typeof item !== "object") throw new Error("Patch batch entry is invalid.");
@@ -244,6 +251,45 @@ async function preparePatchBatch(
244251
return prepared;
245252
}
246253

254+
async function preparePatchBatch(root: string, value: unknown) {
255+
return await preparePatches(root, value, {
256+
minimum: 2,
257+
maximum: 8,
258+
failureMessage: "Patch batch must contain 2-8 patches.",
259+
});
260+
}
261+
262+
async function prepareNewFiles(root: string, value: unknown): Promise<ConstrainedNewFile[]> {
263+
if (!Array.isArray(value) || value.length > 8) throw new Error("New-file batch is invalid.");
264+
const files: ConstrainedNewFile[] = [];
265+
for (const item of value) {
266+
if (item === null || typeof item !== "object") throw new Error("New-file batch entry is invalid.");
267+
const file = item as Partial<ConstrainedNewFile>;
268+
if (typeof file.content !== "string" || file.content.length > 32_768 || file.content.includes("\0"))
269+
throw new Error("New-file batch entry content is invalid.");
270+
files.push({ path: await safeNewFile(root, file.path), content: file.content });
271+
}
272+
if (new Set(files.map((file) => file.path)).size !== files.length)
273+
throw new Error("New-file batch must not target the same file twice.");
274+
return files;
275+
}
276+
277+
async function prepareEditBatch(root: string, value: unknown) {
278+
if (value === null || typeof value !== "object") throw new Error("Edit batch is invalid.");
279+
const batch = value as { patches?: unknown; creates?: unknown };
280+
const patches = await preparePatches(root, batch.patches, {
281+
minimum: 0,
282+
maximum: 8,
283+
failureMessage: "Edit batch patches are invalid.",
284+
});
285+
const creates = await prepareNewFiles(root, batch.creates);
286+
const editCount = patches.length + creates.length;
287+
if (editCount < 2 || editCount > 8) throw new Error("Edit batch must contain 2-8 edits.");
288+
if (new Set([...patches.map((patch) => patch.path), ...creates.map((file) => file.path)]).size !== editCount)
289+
throw new Error("Edit batch must not mix a patch and creation for the same file.");
290+
return { patches, creates };
291+
}
292+
247293
function summary(value: string, limit: number): string {
248294
return value.length <= limit ? value : `${value.slice(0, limit)}\n[truncated]`;
249295
}
@@ -398,6 +444,16 @@ async function executeTool(
398444
evidence: `Applied ${patches.length} constrained text replacements after batch preflight.`,
399445
};
400446
}
447+
if (name === "apply-edit-batch") {
448+
const batch = await prepareEditBatch(root, request.arguments);
449+
for (const patch of batch.patches) await writeFile(patch.path, patch.replacement, "utf8");
450+
for (const file of batch.creates) await writeFile(file.path, file.content, { encoding: "utf8", flag: "wx" });
451+
return {
452+
status: "ok" as const,
453+
observation: `Edit batch applied to ${batch.patches.length} existing and ${batch.creates.length} new files.`,
454+
evidence: `Applied ${batch.patches.length} constrained replacements and created ${batch.creates.length} files after batch preflight.`,
455+
};
456+
}
401457
if (name === "create-file") {
402458
const content = request.arguments.content;
403459
if (typeof content !== "string" || content.length > 32_768 || content.includes("\0"))

‎src/agentpatchcheck/model-provider.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,39 @@ const toolParameters: Record<HarnessNativeToolName, Record<string, unknown>> = {
133133
required: ["patches"],
134134
additionalProperties: false,
135135
},
136+
"apply-edit-batch": {
137+
type: "object",
138+
properties: {
139+
patches: {
140+
type: "array",
141+
minItems: 0,
142+
maxItems: 8,
143+
items: {
144+
type: "object",
145+
properties: {
146+
path: { type: "string" },
147+
expectedText: { type: "string" },
148+
replacementText: { type: "string" },
149+
},
150+
required: ["path", "expectedText", "replacementText"],
151+
additionalProperties: false,
152+
},
153+
},
154+
creates: {
155+
type: "array",
156+
minItems: 0,
157+
maxItems: 8,
158+
items: {
159+
type: "object",
160+
properties: { path: { type: "string" }, content: { type: "string" } },
161+
required: ["path", "content"],
162+
additionalProperties: false,
163+
},
164+
},
165+
},
166+
required: ["patches", "creates"],
167+
additionalProperties: false,
168+
},
136169
"create-file": {
137170
type: "object",
138171
properties: { path: { type: "string" }, content: { type: "string" } },
@@ -195,7 +228,9 @@ function selectedTools(tools: readonly HarnessNativeToolName[]) {
195228
description:
196229
name === "run-public-verification"
197230
? "Run one TaskSpec-declared public verification command by its zero-based index. It cannot run arbitrary commands."
198-
: `Request the Harness-owned ${name} tool.`,
231+
: name === "apply-edit-batch"
232+
? "Apply a preflighted batch of existing-file text replacements and new-file creations. Every target must stay within the managed workspace."
233+
: `Request the Harness-owned ${name} tool.`,
199234
parameters: toolParameters[name],
200235
})),
201236
{

‎src/agentpatchcheck/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ export type HarnessNativeToolName =
231231
| "git-diff"
232232
| "apply-patch"
233233
| "apply-patch-batch"
234+
| "apply-edit-batch"
234235
| "create-file"
235236
| "run-public-verification";
236237
export type HarnessNativeTerminationReason =

‎test/agentpatchcheck/harness-native-benchmark-suite.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,36 @@ describe("Harness-native Benchmark Suite v1", () => {
207207
}
208208
});
209209

210+
it("materializes the v5 controlled edit and self-verification corpus without changing v4", async () => {
211+
const root = await mkdtemp(join(tmpdir(), "agentpatchcheck-native-benchmark-suite-"));
212+
const outputRoot = join(root, "suite");
213+
try {
214+
const { stdout } = await execFile(
215+
process.execPath,
216+
[suiteScript, "--output-root", outputRoot, "--model", "test-model", "--suite-version", "v5", "--dry-run"],
217+
{ cwd: projectRoot, windowsHide: true },
218+
);
219+
const output = JSON.parse(stdout) as NativeSuiteDryRunOutput;
220+
expect(output).toMatchObject({
221+
suite: { id: "harness-native-public-repair", fixtureVersion: "v5" },
222+
baseCommit: "1f487c7346407098397230807a9f4c971a826781",
223+
});
224+
expect(output.taskSpecPaths).toHaveLength(13);
225+
const taskPath = output.taskSpecPaths.find((path) => path.endsWith("edit-batch-public-verification.json"));
226+
const policy = await validateTaskPolicy(await loadTaskSpec(taskPath ?? ""));
227+
expect(policy).toMatchObject({
228+
verificationProfile: { name: "edit-batch-public" },
229+
hiddenOracle: { scriptPath: expect.stringContaining("oracle-edit-batch-public-verification.mjs") },
230+
nativeAgent: { maxIterations: 8, maxToolCalls: 10 },
231+
});
232+
expect(
233+
await readFile(join(outputRoot, "tasks", "prompts", "edit-batch-public-verification.txt"), "utf8"),
234+
).toContain("apply-edit-batch");
235+
} finally {
236+
await rm(root, { recursive: true, force: true });
237+
}
238+
});
239+
210240
it("fails before materializing a suite when model selection is omitted", async () => {
211241
const root = await mkdtemp(join(tmpdir(), "agentpatchcheck-native-benchmark-suite-"));
212242
const outputRoot = join(root, "suite");

‎test/agentpatchcheck/harness-native-runtime.test.ts‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,138 @@ describe("Harness-native Agent Runtime", () => {
373373
}
374374
});
375375

376+
it("preflights and applies a combined existing-file and new-file edit batch", async () => {
377+
const worktree = await mkdtemp(join(tmpdir(), "agentpatchcheck-native-runtime-"));
378+
try {
379+
await mkdir(join(worktree, "docs"), { recursive: true });
380+
await writeFile(join(worktree, "README.md"), "before\n", "utf8");
381+
const provider: HarnessNativeModelProvider = {
382+
id: "test-provider",
383+
decide: async ({ observations }) =>
384+
observations.length === 0
385+
? {
386+
decision: {
387+
kind: "tool",
388+
tool: "apply-edit-batch",
389+
arguments: {
390+
patches: [
391+
{
392+
path: "README.md",
393+
expectedText: "before",
394+
replacementText: "after",
395+
},
396+
],
397+
creates: [{ path: "docs/usage.md", content: "usage\n" }],
398+
},
399+
},
400+
}
401+
: { decision: { kind: "finish" } },
402+
};
403+
404+
const result = await runHarnessNativeRuntime({
405+
policy: testNativePolicy({ maxIterations: 2, maxToolCalls: 1 }),
406+
prompt: "Update README and add usage documentation.",
407+
model: "test-model",
408+
worktreePath: worktree,
409+
provider,
410+
timeoutMs: 1_000,
411+
});
412+
413+
expect(result).toMatchObject({ status: "succeeded", toolCalls: 1 });
414+
expect(result.trajectory[0]).toMatchObject({ tool: "apply-edit-batch", toolStatus: "ok" });
415+
expect(await readFile(join(worktree, "README.md"), "utf8")).toBe("after\n");
416+
expect(await readFile(join(worktree, "docs", "usage.md"), "utf8")).toBe("usage\n");
417+
} finally {
418+
await rm(worktree, { recursive: true, force: true });
419+
}
420+
});
421+
422+
it("rejects a combined edit batch before changing any target when creation preflight fails", async () => {
423+
const worktree = await mkdtemp(join(tmpdir(), "agentpatchcheck-native-runtime-"));
424+
try {
425+
await writeFile(join(worktree, "README.md"), "before\n", "utf8");
426+
await writeFile(join(worktree, "existing.md"), "original\n", "utf8");
427+
const provider: HarnessNativeModelProvider = {
428+
id: "test-provider",
429+
decide: async ({ observations }) =>
430+
observations.length === 0
431+
? {
432+
decision: {
433+
kind: "tool",
434+
tool: "apply-edit-batch",
435+
arguments: {
436+
patches: [
437+
{
438+
path: "README.md",
439+
expectedText: "before",
440+
replacementText: "after",
441+
},
442+
],
443+
creates: [{ path: "existing.md", content: "overwrite\n" }],
444+
},
445+
},
446+
}
447+
: { decision: { kind: "finish" } },
448+
};
449+
450+
const result = await runHarnessNativeRuntime({
451+
policy: testNativePolicy({ maxIterations: 2, maxToolCalls: 1 }),
452+
prompt: "Try an invalid batch.",
453+
model: "test-model",
454+
worktreePath: worktree,
455+
provider,
456+
timeoutMs: 1_000,
457+
});
458+
459+
expect(result.trajectory[0]).toMatchObject({ tool: "apply-edit-batch", toolStatus: "rejected" });
460+
expect(await readFile(join(worktree, "README.md"), "utf8")).toBe("before\n");
461+
expect(await readFile(join(worktree, "existing.md"), "utf8")).toBe("original\n");
462+
} finally {
463+
await rm(worktree, { recursive: true, force: true });
464+
}
465+
});
466+
467+
it("rejects more than eight combined edits before changing any target", async () => {
468+
const worktree = await mkdtemp(join(tmpdir(), "agentpatchcheck-native-runtime-"));
469+
try {
470+
await writeFile(join(worktree, "README.md"), "before\n", "utf8");
471+
const provider: HarnessNativeModelProvider = {
472+
id: "test-provider",
473+
decide: async ({ observations }) =>
474+
observations.length === 0
475+
? {
476+
decision: {
477+
kind: "tool",
478+
tool: "apply-edit-batch",
479+
arguments: {
480+
patches: [{ path: "README.md", expectedText: "before", replacementText: "after" }],
481+
creates: Array.from({ length: 8 }, (_, index) => ({
482+
path: `new-${index}.txt`,
483+
content: "new\n",
484+
})),
485+
},
486+
},
487+
}
488+
: { decision: { kind: "finish" } },
489+
};
490+
491+
const result = await runHarnessNativeRuntime({
492+
policy: testNativePolicy({ maxIterations: 2, maxToolCalls: 1 }),
493+
prompt: "Try an oversized batch.",
494+
model: "test-model",
495+
worktreePath: worktree,
496+
provider,
497+
timeoutMs: 1_000,
498+
});
499+
500+
expect(result.trajectory[0]).toMatchObject({ tool: "apply-edit-batch", toolStatus: "rejected" });
501+
expect(await readFile(join(worktree, "README.md"), "utf8")).toBe("before\n");
502+
await expect(readFile(join(worktree, "new-0.txt"), "utf8")).rejects.toMatchObject({ code: "ENOENT" });
503+
} finally {
504+
await rm(worktree, { recursive: true, force: true });
505+
}
506+
});
507+
376508
it("applies a multi-line LF request to a CRLF file while preserving CRLF output", async () => {
377509
const worktree = await mkdtemp(join(tmpdir(), "agentpatchcheck-native-runtime-"));
378510
try {
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"version": 1,
3+
"name": "harness-native-public-repair-v5",
4+
"suite": { "id": "harness-native-public-repair", "fixtureVersion": "v5" },
5+
"tasks": [
6+
{ "id": "public-repair", "taskSpec": "tasks/public-repair.json" },
7+
{ "id": "create-new-file", "taskSpec": "tasks/create-new-file.json" },
8+
{ "id": "multi-file-local-repair", "taskSpec": "tasks/multi-file-local-repair.json" },
9+
{ "id": "recursive-cross-directory-repair", "taskSpec": "tasks/recursive-cross-directory-repair.json" },
10+
{ "id": "recursive-feedback-repair", "taskSpec": "tasks/recursive-feedback-repair.json" },
11+
{ "id": "configuration-semantic-repair", "taskSpec": "tasks/configuration-semantic-repair.json" },
12+
{ "id": "recursive-feedback-invariant-repair", "taskSpec": "tasks/recursive-feedback-invariant-repair.json" },
13+
{ "id": "nested-configuration-invariant-repair", "taskSpec": "tasks/nested-configuration-invariant-repair.json" },
14+
{ "id": "source-normalization-repair", "taskSpec": "tasks/source-normalization-repair.json" },
15+
{ "id": "cross-file-contract-repair", "taskSpec": "tasks/cross-file-contract-repair.json" },
16+
{ "id": "feature-flag-invariant-repair", "taskSpec": "tasks/feature-flag-invariant-repair.json" },
17+
{ "id": "validation-boundary-repair", "taskSpec": "tasks/validation-boundary-repair.json" },
18+
{ "id": "edit-batch-public-verification", "taskSpec": "tasks/edit-batch-public-verification.json" }
19+
]
20+
}
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"version": 1,
3+
"baseCommit": "1f487c7346407098397230807a9f4c971a826781",
4+
"suite": { "id": "harness-native-public-repair", "fixtureVersion": "v5" },
5+
"taskCount": 13,
6+
"budgets": { "timeoutMs": 120000, "maxIterations": 8, "maxToolCalls": 10, "maxObservationBytes": 4096 }
7+
}

0 commit comments

Comments
 (0)