diff --git a/docs/agent-benchmark.md b/docs/agent-benchmark.md index 8e2d943a..185cfdc8 100644 --- a/docs/agent-benchmark.md +++ b/docs/agent-benchmark.md @@ -261,6 +261,10 @@ a repaired Settings screenshot. The Stim arm must preserve `stim ios` launch output and `stim logs --errors`; control collects the equivalent Metro and simulator logs manually. The injected error text is unique per run so the collector can prove that the reported stack and repair refer to this failure. +The unique token and source location must appear in captured runtime errors; +the earlier successful launch command does not have to print the token inline. +Before capture, normal guide, doctor, worktree warming, and narrowly scoped +installed-dependency resolution are setup, not application-source inspection. The coordinator creates a per-run fixture branch, injects and commits the exception, and checks out that fixture before dispatch, outside the timed diff --git a/scripts/agent-benchmark/run-record.mjs b/scripts/agent-benchmark/run-record.mjs index f436ece6..bfc910fa 100644 --- a/scripts/agent-benchmark/run-record.mjs +++ b/scripts/agent-benchmark/run-record.mjs @@ -22,6 +22,28 @@ export function completedCleanupRecord(record) { } export function durableRunRecord(previous, next, cleanupCompleted = false) { + if ( + cleanupCompleted && + previous?.variant === 'launch-crash' && + next.variant === 'launch-crash' && + typeof previous.runId === 'string' && + previous.runId === next.runId && + previous.proof?.valid === true && + previous.proof.kind === 'launch-crash-source-repair' && + previous.proof.sourceSha256 && + previous.proof.sourceSha256 === previous.evidenceSha256?.proof && + next.proof?.reason === 'launch-crash-source-missing' && + sameEvidence(previous, next) + ) { + const invalidReasons = next.invalidReasons.filter((reason) => reason !== 'launch-crash-source-missing'); + return { + ...next, + proof: previous.proof, + evidenceSha256: { ...next.evidenceSha256, proof: previous.evidenceSha256.proof }, + invalidReasons, + valid: invalidReasons.length === 0, + }; + } if ( cleanupCompleted && previous?.valid === true && diff --git a/scripts/agent-benchmark/run-record.test.mjs b/scripts/agent-benchmark/run-record.test.mjs index 466fff78..819e06ed 100644 --- a/scripts/agent-benchmark/run-record.test.mjs +++ b/scripts/agent-benchmark/run-record.test.mjs @@ -5,6 +5,39 @@ const hashes = { events: 'events', settingsPng: 'settings', transcript: 'transcr const valid = { valid: true, invalidReasons: [], evidenceSha256: hashes, collectedAt: 'first' }; describe('durable benchmark run records', () => { + it('retains verified crash repair evidence after cleanup without retaining a stale audit verdict', () => { + const previous = { + runId: 'crash-run', + variant: 'launch-crash', + valid: false, + invalidReasons: ['launch-crash-initial-launch-missing'], + proof: { valid: true, kind: 'launch-crash-source-repair', sourceSha256: 'source' }, + evidenceSha256: { ...hashes, proof: 'source' }, + }; + const next = { + runId: 'crash-run', + variant: 'launch-crash', + valid: false, + invalidReasons: ['launch-crash-source-missing'], + proof: { valid: false, reason: 'launch-crash-source-missing' }, + evidenceSha256: hashes, + }; + expect(durableRunRecord(previous, next, true)).toMatchObject({ + valid: true, + proof: previous.proof, + invalidReasons: [], + }); + expect( + durableRunRecord(previous, { ...next, invalidReasons: [...next.invalidReasons, 'timeout'] }, true), + ).toMatchObject({ valid: false, invalidReasons: ['timeout'] }); + expect(durableRunRecord(previous, next)).toBe(next); + for (const changed of [ + { ...next, runId: 'different-run' }, + { ...next, evidenceSha256: { ...hashes, events: 'changed' } }, + ]) + expect(durableRunRecord(previous, changed, true)).toBe(changed); + expect(durableRunRecord({ ...previous, evidenceSha256: { ...hashes, proof: 'different' } }, next, true)).toBe(next); + }); it('requires a successful recorded worktree cleanup', () => { expect(completedCleanupRecord({ cleanedAt: 'now', actions: ['stim worktree remove --force'] })).toBe(true); expect(completedCleanupRecord({ cleanedAt: 'now', actions: ['verified agent-device sessions empty'] })).toBe(false); diff --git a/scripts/export-benchmark-viewer.mjs b/scripts/export-benchmark-viewer.mjs index 857f2ec8..76c63721 100644 --- a/scripts/export-benchmark-viewer.mjs +++ b/scripts/export-benchmark-viewer.mjs @@ -14,6 +14,7 @@ import { userInfo } from 'node:os'; import { fileURLToPath } from 'node:url'; import { stripVTControlCharacters } from 'node:util'; import { launchCrashDiagnosis, launchCrashRecovery } from './launch-crash-benchmark.mjs'; +import { topLevelShellCommand } from './agent-benchmark/run-guards.mjs'; const modelPricing = { 'gpt-5.6-luna': { @@ -156,9 +157,10 @@ function replacementLabel(path) { } function unwrapShellCommand(command) { - return String(command) - .replace(/^\/bin\/(?:zsh|bash|sh) -lc /, '') - .replace(/^(['"])([\s\S]*)\1$/, '$2'); + const value = String(command); + const normalized = topLevelShellCommand(value); + if (normalized !== value) return normalized; + return value.replace(/^\/bin\/(?:zsh|bash|sh) -lc /, '').replace(/^(['"])([\s\S]*)\1$/, '$2'); } export function sanitizeBenchmarkText(value, replacements = []) { diff --git a/scripts/export-benchmark-viewer.test.mjs b/scripts/export-benchmark-viewer.test.mjs index 2bb5d0fc..99f063ec 100644 --- a/scripts/export-benchmark-viewer.test.mjs +++ b/scripts/export-benchmark-viewer.test.mjs @@ -998,7 +998,14 @@ describe('benchmark viewer export', () => { 'skill', ], ['worktree', '2026-09-04T12:00:03.000Z', '2026-09-04T12:00:05.000Z', 'stim worktree create bench/run', 'ready'], - ['launch', '2026-09-04T12:00:10.000Z', '2026-09-04T12:00:20.000Z', 'stim ios', token], + [ + 'metadata', + '2026-09-04T12:00:06.000Z', + '2026-09-04T12:00:07.000Z', + String.raw`/bin/zsh -lc "node -p \"require.resolve('expo/package.json')\" && node_modules/.bin/expo --version"`, + '58.0.0-canary', + ], + ['launch', '2026-09-04T12:00:10.000Z', '2026-09-04T12:00:20.000Z', 'stim ios', 'app launched'], ['logs', '2026-09-04T12:00:25.000Z', '2026-09-04T12:00:30.000Z', 'stim logs --errors', token], [ 'diagnosis', @@ -1067,7 +1074,7 @@ describe('benchmark viewer export', () => { valid: true, invalidReasons: [], dispatchToDiagnosisSeconds: 90, - diagnosisCommandCount: 5, + diagnosisCommandCount: 6, diagnosisUsage: { input_tokens: 100_000, cached_input_tokens: 80_000, @@ -1118,7 +1125,7 @@ describe('benchmark viewer export', () => { id: 'launch-crash-stim', platform: 'ios', diagnosisSeconds: 90, - diagnosisCommandCount: 5, + diagnosisCommandCount: 6, launchCrashAudit: { initialLaunchCommandId: 'launch', errorCaptureCommandId: 'logs', diff --git a/scripts/launch-crash-benchmark.mjs b/scripts/launch-crash-benchmark.mjs index 1c070fed..253a56b4 100644 --- a/scripts/launch-crash-benchmark.mjs +++ b/scripts/launch-crash-benchmark.mjs @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { shellCommandSegments, topLevelShellCommand } from './agent-benchmark/run-guards.mjs'; export function launchCrashToken(runId) { const digest = createHash('sha256').update(runId).digest('hex').slice(0, 12).toUpperCase(); @@ -35,6 +36,8 @@ function successful(command) { function shellCommand(command) { const value = String(command ?? '').trim(); + const normalized = topLevelShellCommand(value); + if (normalized !== value) return normalized; const body = value.replace(/^\/bin\/(?:zsh|bash|sh) -lc\s+/, ''); return body.replace(/^["']/, '').replace(/["']$/, '').trim(); } @@ -58,7 +61,9 @@ function errorCaptureCommand(command, arm, platform) { command = shellCommand(command); if (arm === 'stim') return /(?:^|\s)stim\s+logs\s+--errors(?:\s|$)/.test(command); const explicitLogFile = - /\b(?:tail|rg|grep)\b[\s\S]*(?:\.log\b|(?:^|[\s'"])(?:\.?\/)?(?:tmp|logs?|\.expo\/dev\/logs)\/)/.test(command); + /\b(?:tail|rg|grep|sed|cat)\b[\s\S]*(?:\.log\b|(?:^|[\s'"])(?:\.?\/)?(?:tmp|logs?|\.expo\/dev\/logs)\/)/.test( + command, + ); if (platform === 'android') return /\badb\s+logcat\b/.test(command) || explicitLogFile; return /\bxcrun\s+simctl\s+spawn\b|\blog\s+(?:show|stream)\b/.test(command) || explicitLogFile; } @@ -92,6 +97,10 @@ function sourceInspectionBeforeCapture(command, arm, platform) { function allowedBeforeErrorCapture(command, arm, platform) { const value = shellCommand(command); + if (/^(?:stim\s+(?:guide|doctor|worktree\s+warm)\b|rsync\b|pgrep\b|sed\b|cat\b)/.test(value)) { + const segments = shellCommandSegments(value); + if (segments.length > 1) return segments.every((segment) => allowedBeforeErrorCapture(segment, arm, platform)); + } if (/^tool:todo_list\b/.test(value)) return true; if (/^(?:env\s+)?(?:[^\s=]+=[^\s]+\s+)*agent-device\s+/.test(value)) return true; if ( @@ -103,6 +112,16 @@ function allowedBeforeErrorCapture(command, arm, platform) { return true; } if (sourceInspectionBeforeCapture(value, arm, platform)) return false; + if (/^(?:\.\/)?node_modules\/\.bin\/expo\s+--version$/.test(value)) return true; + if (/^node\s+-p\s+(?:process\.execPath|(["'])process\.execPath\1)$/.test(value)) return true; + if (/^print\s+-r\s+--\s+\d+\s*\|\s*tee\s+\/(?:private\/)?tmp\/[A-Za-z0-9_./-]+\.pid$/.test(value)) return true; + if ( + /^node\s+-p\s+(["'])require\.resolve\((["'])(?:expo|react-native)\/package\.json\2\)\1(?:\s*&&\s*(?:\.\/)?node_modules\/\.bin\/expo\s+--version)?$/.test( + value, + ) + ) { + return true; + } if ( /(?:\/(?:skills|skill)\/[^\s]+\/|(?:^|\s)workspace\/)SKILL\.md\b/.test(value) && /(?:^|\s)(?:cat|sed|head)(?:\s|$)/.test(value) @@ -131,14 +150,19 @@ function allowedBeforeErrorCapture(command, arm, platform) { ) { return true; } - if (/^cp\b/.test(value) && /(?:node_modules|ios\/Pods|ios\/build|android\/(?:\.gradle|app\/build))/.test(value)) { + if ( + /^(?:cp|rsync)\b/.test(value) && + /(?:node_modules|ios\/Pods|ios\/build|android\/(?:\.gradle|app\/build))/.test(value) + ) { return true; } if (arm === 'stim') { - return new RegExp(`^stim\\s+(?:worktree\\s+create|start|${platform}|logs\\s+--errors)(?:\\s|$)`).test(value); + return new RegExp( + `^stim\\s+(?:guide|doctor|worktree\\s+(?:warm|create)|start|${platform}|logs\\s+--errors)(?:\\s|$)`, + ).test(value); } return ( - /^(?:(?:[A-Za-z_][A-Za-z0-9_]*=(?:\S+|\$\([^)]*\))[;\s]+)*)(?:open\s+-a\s+Simulator|xcrun\s+simctl\s+|npx\s+expo\s+|xcodebuild\b|\.\/gradlew\b|adb\b|nohup\b|launchctl\b|ps\b|sleep\b|tail\b|cat\s+\/?tmp\/|wc\b|lsof\b|command\s+-v\b|test\b|kill\b)/.test( + /^(?:(?:[A-Za-z_][A-Za-z0-9_]*=(?:\S+|\$\([^)]*\))[;\s]+)*)(?:open\s+-a\s+Simulator|xcrun\s+simctl\s+|npx\s+expo\s+|xcodebuild\b|\.\/gradlew\b|adb\b|nohup\b|launchctl\b|ps\b|pgrep\b|sleep\b|tail\b|cat\s+\/?tmp\/|wc\b|lsof\b|command\s+-v\b|test\b|kill\b)/.test( value, ) || launchCommand(value, arm, platform) || @@ -150,10 +174,7 @@ export function launchCrashDiagnosis(commands, { dispatchAt, token, arm = 'stim' const ordered = orderedCommands(commands); const sourceMarkers = ['app/_layout.tsx', 'RootLayout']; const initialLaunchIndex = ordered.findIndex( - (command) => - successful(command) && - launchCommand(command.command, arm, platform) && - (arm !== 'stim' || (typeof command.output === 'string' && command.output.includes(token))), + (command) => successful(command) && launchCommand(command.command, arm, platform), ); if (initialLaunchIndex === -1) { return { valid: false, reason: 'launch-crash-initial-launch-evidence-missing' }; diff --git a/scripts/launch-crash-benchmark.test.mjs b/scripts/launch-crash-benchmark.test.mjs index b10fb658..f1c1f06e 100644 --- a/scripts/launch-crash-benchmark.test.mjs +++ b/scripts/launch-crash-benchmark.test.mjs @@ -46,7 +46,7 @@ describe('launch crash benchmark', () => { { id: 'launch', command: 'stim ios', - output: token, + output: 'launch com.example.app\n1 error-level record during launch (logs --errors --source device)', exitCode: 0, startedAt: '2026-09-04T12:00:01.000Z', endedAt: '2026-09-04T12:00:10.000Z', @@ -108,6 +108,64 @@ describe('launch crash benchmark', () => { }); }); + it('allows current Stim setup and narrow dependency resolution before separately captured errors', () => { + const token = launchCrashToken('setup'); + const setup = [ + 'stim guide agent', + 'git worktree add -b bench/run /tmp/bench-run HEAD', + 'stim worktree warm', + 'stim doctor --platform ios', + `node -p "require.resolve('expo/package.json')" && node_modules/.bin/expo --version`, + ].map((command, index) => ({ + id: `setup-${index}`, + command: `/bin/zsh -lc ${JSON.stringify(command)}`, + exitCode: 0, + endedAt: `2026-09-04T12:00:0${index + 1}Z`, + })); + const evidence = [ + { + id: 'launch', + command: 'stim ios', + output: 'launched com.example.app', + exitCode: 0, + endedAt: '2026-09-04T12:00:10Z', + }, + { + id: 'logs', + command: 'stim logs --errors', + output: `${token}\napp/_layout.tsx in RootLayout`, + exitCode: 0, + startedAt: '2026-09-04T12:00:11Z', + endedAt: '2026-09-04T12:00:12Z', + }, + ]; + const options = { dispatchAt: '2026-09-04T12:00:00Z', token }; + expect(launchCrashDiagnosis([...setup, ...evidence], options)).toMatchObject({ + valid: true, + commandId: 'logs', + dispatchToDiagnosisSeconds: 12, + }); + for (const command of [ + `node -p "require('fs').readFileSync('app/_layout.tsx', 'utf8')"`, + `node -p "require.resolve('expo/package.json'); require('./app/_layout.tsx')"`, + 'stim guide agent && cat app/_layout.tsx', + 'stim doctor --platform ios; git diff', + 'stim guide agent && rg "throw new Error" .', + 'stim doctor --platform ios; rg "throw new Error" .', + 'stim worktree warm | rg "throw new Error" .', + 'rsync -a node_modules /tmp/wt/ && rg "throw new Error" .', + ]) { + expect(launchCrashDiagnosis([{ ...setup[0], command }, ...evidence], options)).toMatchObject({ + valid: false, + reason: 'launch-crash-pre-capture-command-not-allowed', + commandId: 'setup-0', + }); + } + expect( + launchCrashDiagnosis([...setup, evidence[0], { ...evidence[1], output: 'unrelated error' }], options), + ).toMatchObject({ valid: false, reason: 'launch-crash-error-capture-missing' }); + }); + it('rejects source inspection hidden after an allowed compound-command prefix', () => { const token = launchCrashToken('run'); const tail = [ @@ -191,6 +249,56 @@ describe('launch crash benchmark', () => { ).toMatchObject({ valid: true, commandId: 'logs' }); }); + it('allows dependency copying and PID/log diagnostics without treating source reads as logs', () => { + const token = launchCrashToken('control-setup'); + const setup = [ + 'rsync -aR node_modules ios/Pods ios/build /tmp/worktree/', + './node_modules/.bin/expo --version', + 'node -p process.execPath', + 'pgrep -P 35182 -fl .', + 'print -r -- 35182 | tee /tmp/run-metro.pid', + "sed -n '1,160p' /tmp/run-metro.log", + ].map((command, index) => ({ + id: `setup-${index}`, + command, + exitCode: 0, + endedAt: `2026-09-04T12:00:0${index + 1}Z`, + })); + const launch = { + id: 'launch', + command: 'npx expo run:ios --device SIMULATOR', + output: 'launched', + exitCode: 0, + endedAt: '2026-09-04T12:00:10Z', + }; + const logs = { + id: 'logs', + command: "sed -n '1,160p' /tmp/run-runtime.log", + output: `${token}\napp/_layout.tsx in RootLayout`, + exitCode: 0, + startedAt: '2026-09-04T12:00:11Z', + endedAt: '2026-09-04T12:00:12Z', + }; + const options = { dispatchAt: '2026-09-04T12:00:00Z', token, arm: 'control' }; + expect(launchCrashDiagnosis([...setup, launch, logs], options)).toMatchObject({ valid: true, commandId: 'logs' }); + for (const command of [ + 'pgrep -fl Metro && rg "throw new Error" .', + 'sed -n "1,160p" /tmp/run-metro.log && rg "throw new Error" .', + 'cat /tmp/run-metro.log; rg "throw new Error" .', + ]) { + expect(launchCrashDiagnosis([{ ...setup[0], command }, launch, logs], options)).toMatchObject({ + valid: false, + reason: 'launch-crash-pre-capture-command-not-allowed', + }); + } + expect( + launchCrashDiagnosis([launch, { ...logs, command: "sed -n '1,160p' app/_layout.tsx" }], options), + ).toMatchObject({ valid: false, reason: 'launch-crash-error-capture-missing' }); + expect( + launchCrashDiagnosis([launch, { ...logs, command: 'cat app/_layout.tsx /tmp/run-runtime.log' }], options), + ).toMatchObject({ valid: false, reason: 'launch-crash-pre-capture-command-not-allowed' }); + }); + it('unwraps a shell command whose nested quoting changes the closing quote', () => { const token = launchCrashToken('run'); expect(