feat(benchmark): add v3 invariant repair suite

Author: aravelo7

docs/harness-native-benchmark-suite.md

@@ -11,7 +11,10 @@ failure first, then measure the bounded single repair attempt before the Hidden
 
 The separately versioned v2 corpus preserves all five v1 tasks and adds `configuration-semantic-repair`: public
 verification checks the requested settings values, while the Hidden Oracle checks the exact intended JSON document.
-The corpus is selected explicitly; no existing suite identity is mutated.
+The separately versioned v3 corpus preserves v2 and adds `recursive-feedback-invariant-repair` (a bounded feedback
+repair whose Oracle also protects an unrelated source file) and `nested-configuration-invariant-repair` (publicly
+checks two nested settings while the Oracle protects the complete configuration document). The corpus is selected
+explicitly; no existing suite identity is mutated.
 
 ## Run
 
@@ -113,13 +116,13 @@ npm.cmd run benchmark:harness-native-suite -- `
   --output-root D:\Benchmarks\agentpatchcheck-native-v1-dry-run `
   --model <model-id> `
   --provider-profile <openai-responses|deepseek-chat> `
-  --suite-version <v1|v2> `
+  --suite-version <v1|v2|v3> `
   --dry-run
 ```
 
 ## Limits and boundaries
 
-- The v1 corpus uses five independent managed workspaces and requires a patch from each task; v2 uses six.
+- The v1/v2/v3 corpora use five/six/eight independent managed workspaces respectively and require a patch from each task.
 - The multi-file task allows only exact replacements in two existing files; it verifies both public prefixes and exact
   final content through a Hidden Oracle.
 - The recursive task requires bounded cross-directory discovery before two nested source-file replacements; its Hidden
@@ -149,8 +152,8 @@ precomputed percentages so a consumer cannot accidentally mix model transport fa
 - `providerFailureTasks` is separate from `agentExecutionFailureTasks`; neither should be silently counted as a semantic
   patch failure.
 
-The v1/v2 suites contain five/six tasks respectively and are integration fixtures, not statistically meaningful quality
-scores. Future versioned corpora must grow the task denominator before publishing rates.
+The v1/v2/v3 suites contain five/six/eight tasks respectively and are integration fixtures, not statistically meaningful
+quality scores. Future versioned corpora must grow the task denominator before publishing rates.
 
 The first retained v2 quality sample is documented in
 [`harness-native-quality-baseline-v2.md`](./harness-native-quality-baseline-v2.md). It records only comparable-run

scripts/run-harness-native-benchmark-repetitions.mjs

@@ -13,7 +13,7 @@ const scriptDirectory = dirname(fileURLToPath(import.meta.url));
 const suiteScript = join(scriptDirectory, "run-harness-native-benchmark-suite.mjs");
 const modelPattern = /^[a-zA-Z0-9][a-zA-Z0-9._/-]{0,127}$/u;
 const providerProfiles = new Set(["openai-responses", "deepseek-chat"]);
-const suiteVersions = new Set(["v1", "v2"]);
+const suiteVersions = new Set(["v1", "v2", "v3"]);
 const maxRuns = 20;
 
 function parseArguments(argv) {
@@ -43,11 +43,11 @@ function parseArguments(argv) {
 	}
 	if (outputRoot === undefined || model === undefined || runs === undefined)
 		throw new Error(
-			"Usage: npm run benchmark:harness-native-repetitions -- --output-root <new-directory> --runs <2-20> --model <model> [--provider-profile <profile>] [--suite-version <v1|v2>]",
+			"Usage: npm run benchmark:harness-native-repetitions -- --output-root <new-directory> --runs <2-20> --model <model> [--provider-profile <profile>] [--suite-version <v1|v2|v3>]",
 		);
 	if (!modelPattern.test(model)) throw new Error("model must be a valid 1-128 character model identifier.");
 	if (!providerProfiles.has(providerProfile)) throw new Error("provider-profile must be one of: openai-responses, deepseek-chat.");
-	if (!suiteVersions.has(suiteVersion)) throw new Error("suite-version must be one of: v1, v2.");
+	if (!suiteVersions.has(suiteVersion)) throw new Error("suite-version must be one of: v1, v2, v3.");
 	if (!Number.isSafeInteger(runs) || runs < 2 || runs > maxRuns) throw new Error(`runs must be an integer from 2 to ${maxRuns}.`);
 	return { outputRoot, model, providerProfile, suiteVersion, runs };
 }

scripts/run-harness-native-benchmark-suite.mjs

@@ -29,7 +29,7 @@ const providerProfiles = {
 		requiredEnvironment: "DEEPSEEK_API_KEY",
 	},
 };
-const suiteVersions = new Set(["v1", "v2"]);
+const suiteVersions = new Set(["v1", "v2", "v3"]);
 
 function getProviderProfile(profileId) {
 	const profile = providerProfiles[profileId];
@@ -64,11 +64,11 @@ function parseArguments(argv) {
 	}
 	if (outputRoot === undefined || model === undefined)
 		throw new Error(
-			"Usage: npm run benchmark:harness-native-suite -- --output-root <new-directory> --model <model> [--provider-profile <profile>] [--suite-version <v1|v2>] [--dry-run]",
+			"Usage: npm run benchmark:harness-native-suite -- --output-root <new-directory> --model <model> [--provider-profile <profile>] [--suite-version <v1|v2|v3>] [--dry-run]",
 		);
 	if (!modelPattern.test(model)) throw new Error("model must be a valid 1-128 character model identifier.");
 	getProviderProfile(providerProfile);
-	if (!suiteVersions.has(suiteVersion)) throw new Error("suite-version must be one of: v1, v2.");
+	if (!suiteVersions.has(suiteVersion)) throw new Error("suite-version must be one of: v1, v2, v3.");
 	return { outputRoot, model, providerProfile, suiteVersion, dryRun };
 }
 

src/agentpatchcheck/evidence-bundle.ts

@@ -98,6 +98,15 @@ export function createEvidenceBundle(options: {
 }): EvidenceBundle {
 	const createdAt = (options.createdAt ?? new Date()).toISOString();
 	const promptSha256 = createHash("sha256").update(options.policy.prompt, "utf8").digest("hex");
+	const publicVerificationRepairInstruction =
+		options.policy.publicVerificationRepairInstruction === null
+			? null
+			: {
+					length: options.policy.publicVerificationRepairInstruction.length,
+					sha256: createHash("sha256")
+						.update(options.policy.publicVerificationRepairInstruction, "utf8")
+						.digest("hex"),
+				};
 	const agent = redactAgentExecution(options.execution.agent, options.policy.prompt);
 	const patch = redactPatchSnapshot(options.execution.patch, options.policy.prompt);
 	const commandVerification = redactCommandVerification(options.execution.commandVerification, options.policy.prompt);
@@ -113,6 +122,7 @@ export function createEvidenceBundle(options: {
 			worktreeRoot: options.policy.worktreeRoot,
 			promptLength: options.policy.prompt.length,
 			promptSha256,
+			publicVerificationRepairInstruction,
 			codexExecutable: options.policy.codexExecutable ?? null,
 			agentAdapter: options.policy.agentAdapter,
 			model: options.policy.model ?? null,

src/agentpatchcheck/execute.ts

@@ -115,6 +115,7 @@ export async function executeAgentPatchCheck(
 	const initialAgent = await runAgentSafely(resolvedDependencies.runAgent, policy, workspace.path, {
 		phase: "initial",
 		publicVerificationFeedback: null,
+		repairInstruction: null,
 	});
 	let agent = initialAgent;
 	let commandVerification = await runVerificationSafely(resolvedDependencies.runVerification, policy, workspace.path);
@@ -132,7 +133,11 @@ export async function executeAgentPatchCheck(
 						resolvedDependencies.runAgent,
 						{ ...policy, timeoutMs: remainingAgentBudgetMs },
 						workspace.path,
-						{ phase: "public-verification-repair", publicVerificationFeedback: feedback },
+						{
+							phase: "public-verification-repair",
+							publicVerificationFeedback: feedback,
+							repairInstruction: policy.publicVerificationRepairInstruction,
+						},
 					)
 				: failedAgentExecution(policy, "Harness-native public verification repair budget was exhausted.");
 		const attempts: AgentExecutionAttempt[] = [

src/agentpatchcheck/harness-native-runtime.ts

@@ -391,7 +391,11 @@ export async function runHarnessNativeRuntime(options: {
 	repairContext?: RepairContext;
 }): Promise<HarnessNativeRuntimeResult> {
 	const startedAt = Date.now();
-	const repairContext = options.repairContext ?? { phase: "initial", publicVerificationFeedback: null };
+	const repairContext = options.repairContext ?? {
+		phase: "initial",
+		publicVerificationFeedback: null,
+		repairInstruction: null,
+	};
 	const trajectory: HarnessNativeTrajectoryStep[] = [];
 	const observations: string[] = [];
 	const session = options.provider.createSession?.() ?? {

src/agentpatchcheck/model-provider.ts

@@ -173,7 +173,12 @@ function requestInput(context: ModelProviderContext): string {
 		context.repairContext.phase === "initial"
 			? "None."
 			: JSON.stringify(context.repairContext.publicVerificationFeedback);
-	return `${phaseInstruction}\n\nTask instructions:\n${context.prompt}\n\nPublic verification feedback:\n${feedback}\n\nObservations:\n${context.observations.join("\n---\n")}`;
+	const repairInstruction =
+		context.repairContext.phase === "public-verification-repair" &&
+		typeof context.repairContext.repairInstruction === "string"
+			? `\n\nHarness-owned targeted repair instruction:\n${context.repairContext.repairInstruction}`
+			: "";
+	return `${phaseInstruction}${repairInstruction}\n\nTask instructions:\n${context.prompt}\n\nPublic verification feedback:\n${feedback}\n\nObservations:\n${context.observations.join("\n---\n")}`;
 }
 
 function selectedTools(tools: readonly HarnessNativeToolName[]) {

src/agentpatchcheck/task-policy.ts

@@ -27,6 +27,7 @@ import { validateVerificationPolicy } from "./verification-policy";
 export const DEFAULT_TASK_TIMEOUT_MS = 15 * 60 * 1_000;
 export const MAX_TASK_TIMEOUT_MS = 60 * 60 * 1_000;
 export const MAX_TASK_PROMPT_LENGTH = 16_000;
+export const MAX_PUBLIC_VERIFICATION_REPAIR_INSTRUCTION_LENGTH = 2_048;
 
 const SANDBOXES = new Set<AgentPatchCheckSandbox>(["read-only", "workspace-write"]);
 const MODEL_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._/-]{0,127}$/;
@@ -147,6 +148,17 @@ function normalizePrompt(prompt: string): string {
 	return prompt;
 }
 
+function normalizePublicVerificationRepairInstruction(instruction: string | undefined): string | null {
+	if (instruction === undefined) return null;
+	assertNoNullBytes(instruction, "Public verification repair instruction");
+	if (!instruction.trim()) throw new Error("Public verification repair instruction must not be empty.");
+	if (instruction.length > MAX_PUBLIC_VERIFICATION_REPAIR_INSTRUCTION_LENGTH)
+		throw new Error(
+			`Public verification repair instruction must not exceed ${MAX_PUBLIC_VERIFICATION_REPAIR_INSTRUCTION_LENGTH} characters.`,
+		);
+	return instruction;
+}
+
 function normalizePatchExpectation(expectation: PatchExpectation | undefined): PatchExpectation {
 	const normalized = expectation ?? "changes-required";
 	if (!PATCH_EXPECTATIONS.has(normalized)) {
@@ -353,6 +365,9 @@ export async function validateTaskPolicy(input: TaskPolicyInput): Promise<TaskPo
 		baseCommit,
 		worktreeRoot,
 		prompt: normalizePrompt(input.prompt),
+		publicVerificationRepairInstruction: normalizePublicVerificationRepairInstruction(
+			input.publicVerificationRepairInstruction,
+		),
 		runId: normalizeOptionalRunId(input.runId),
 		codexExecutable: normalizeOptionalExecutable(input.codexExecutable),
 		agentAdapter,

src/agentpatchcheck/task-spec.ts

@@ -12,6 +12,7 @@ const taskSpecSchema = z
 		repositoryRoot: z.string(),
 		prompt: z.string().optional(),
 		promptFile: z.string().optional(),
+		publicVerificationRepairInstruction: z.string().optional(),
 		baseRef: z.string().optional(),
 		worktreeRoot: z.string().optional(),
 		runId: z.string().optional(),
@@ -193,6 +194,7 @@ export async function loadTaskSpec(specPath: string): Promise<TaskPolicyInput> {
 	return {
 		repositoryRoot,
 		prompt,
+		publicVerificationRepairInstruction: parsed.data.publicVerificationRepairInstruction,
 		baseRef: parsed.data.baseRef,
 		worktreeRoot:
 			parsed.data.worktreeRoot === undefined

src/agentpatchcheck/types.ts

@@ -5,6 +5,7 @@ export const TASK_POLICY_BRAND: unique symbol = Symbol("TaskPolicy");
 export interface TaskPolicyInput {
 	repositoryRoot: string;
 	prompt: string;
+	publicVerificationRepairInstruction?: string;
 	baseRef?: string;
 	worktreeRoot?: string;
 	runId?: string;
@@ -114,6 +115,7 @@ export interface TaskPolicy {
 	baseCommit: string;
 	worktreeRoot: string;
 	prompt: string;
+	publicVerificationRepairInstruction: string | null;
 	runId?: string;
 	codexExecutable?: string;
 	agentAdapter: AgentAdapterId;
@@ -166,8 +168,12 @@ export interface PublicVerificationFeedback {
 
 /** Harness-owned phase contract for one Agent execution. Hidden Oracle data is never included. */
 export type RepairContext =
-	| { phase: "initial"; publicVerificationFeedback: null }
-	| { phase: "public-verification-repair"; publicVerificationFeedback: PublicVerificationFeedback };
+	| { phase: "initial"; publicVerificationFeedback: null; repairInstruction?: null }
+	| {
+			phase: "public-verification-repair";
+			publicVerificationFeedback: PublicVerificationFeedback;
+			repairInstruction?: string | null;
+	  };
 
 export interface AgentExecutionAttempt {
 	phase: "initial" | "public-verification-repair";
@@ -340,6 +346,7 @@ export interface TaskPolicyEvidenceSnapshot {
 	worktreeRoot: string;
 	promptLength: number;
 	promptSha256: string;
+	publicVerificationRepairInstruction?: { length: number; sha256: string } | null;
 	codexExecutable: string | null;
 	agentAdapter?: AgentAdapterId;
 	model: string | null;