From 69dc4fff95f74dbdce98db3656e080871b3b77ab Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 6 Sep 2026 11:04:22 -0400 Subject: [PATCH 1/2] fix: audit separate crash launch and error capture --- docs/agent-benchmark.md | 4 + scripts/agent-benchmark/run-record.mjs | 22 +++++ scripts/agent-benchmark/run-record.test.mjs | 33 +++++++ scripts/launch-crash-benchmark.mjs | 33 +++++-- scripts/launch-crash-benchmark.test.mjs | 96 ++++++++++++++++++++- 5 files changed, 179 insertions(+), 9 deletions(-) diff --git a/docs/agent-benchmark.md b/docs/agent-benchmark.md index 8e2d943a..185cfdc8 100644 --- a/docs/agent-benchmark.md +++ b/docs/agent-benchmark.md @@ -261,6 +261,10 @@ a repaired Settings screenshot. The Stim arm must preserve `stim ios` launch output and `stim logs --errors`; control collects the equivalent Metro and simulator logs manually. The injected error text is unique per run so the collector can prove that the reported stack and repair refer to this failure. +The unique token and source location must appear in captured runtime errors; +the earlier successful launch command does not have to print the token inline. +Before capture, normal guide, doctor, worktree warming, and narrowly scoped +installed-dependency resolution are setup, not application-source inspection. The coordinator creates a per-run fixture branch, injects and commits the exception, and checks out that fixture before dispatch, outside the timed diff --git a/scripts/agent-benchmark/run-record.mjs b/scripts/agent-benchmark/run-record.mjs index f436ece6..bfc910fa 100644 --- a/scripts/agent-benchmark/run-record.mjs +++ b/scripts/agent-benchmark/run-record.mjs @@ -22,6 +22,28 @@ export function completedCleanupRecord(record) { } export function durableRunRecord(previous, next, cleanupCompleted = false) { + if ( + cleanupCompleted && + previous?.variant === 'launch-crash' && + next.variant === 'launch-crash' && + typeof previous.runId === 'string' && + previous.runId === next.runId && + previous.proof?.valid === true && + previous.proof.kind === 'launch-crash-source-repair' && + previous.proof.sourceSha256 && + previous.proof.sourceSha256 === previous.evidenceSha256?.proof && + next.proof?.reason === 'launch-crash-source-missing' && + sameEvidence(previous, next) + ) { + const invalidReasons = next.invalidReasons.filter((reason) => reason !== 'launch-crash-source-missing'); + return { + ...next, + proof: previous.proof, + evidenceSha256: { ...next.evidenceSha256, proof: previous.evidenceSha256.proof }, + invalidReasons, + valid: invalidReasons.length === 0, + }; + } if ( cleanupCompleted && previous?.valid === true && diff --git a/scripts/agent-benchmark/run-record.test.mjs b/scripts/agent-benchmark/run-record.test.mjs index 466fff78..819e06ed 100644 --- a/scripts/agent-benchmark/run-record.test.mjs +++ b/scripts/agent-benchmark/run-record.test.mjs @@ -5,6 +5,39 @@ const hashes = { events: 'events', settingsPng: 'settings', transcript: 'transcr const valid = { valid: true, invalidReasons: [], evidenceSha256: hashes, collectedAt: 'first' }; describe('durable benchmark run records', () => { + it('retains verified crash repair evidence after cleanup without retaining a stale audit verdict', () => { + const previous = { + runId: 'crash-run', + variant: 'launch-crash', + valid: false, + invalidReasons: ['launch-crash-initial-launch-missing'], + proof: { valid: true, kind: 'launch-crash-source-repair', sourceSha256: 'source' }, + evidenceSha256: { ...hashes, proof: 'source' }, + }; + const next = { + runId: 'crash-run', + variant: 'launch-crash', + valid: false, + invalidReasons: ['launch-crash-source-missing'], + proof: { valid: false, reason: 'launch-crash-source-missing' }, + evidenceSha256: hashes, + }; + expect(durableRunRecord(previous, next, true)).toMatchObject({ + valid: true, + proof: previous.proof, + invalidReasons: [], + }); + expect( + durableRunRecord(previous, { ...next, invalidReasons: [...next.invalidReasons, 'timeout'] }, true), + ).toMatchObject({ valid: false, invalidReasons: ['timeout'] }); + expect(durableRunRecord(previous, next)).toBe(next); + for (const changed of [ + { ...next, runId: 'different-run' }, + { ...next, evidenceSha256: { ...hashes, events: 'changed' } }, + ]) + expect(durableRunRecord(previous, changed, true)).toBe(changed); + expect(durableRunRecord({ ...previous, evidenceSha256: { ...hashes, proof: 'different' } }, next, true)).toBe(next); + }); it('requires a successful recorded worktree cleanup', () => { expect(completedCleanupRecord({ cleanedAt: 'now', actions: ['stim worktree remove --force'] })).toBe(true); expect(completedCleanupRecord({ cleanedAt: 'now', actions: ['verified agent-device sessions empty'] })).toBe(false); diff --git a/scripts/launch-crash-benchmark.mjs b/scripts/launch-crash-benchmark.mjs index 1c070fed..85945719 100644 --- a/scripts/launch-crash-benchmark.mjs +++ b/scripts/launch-crash-benchmark.mjs @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { topLevelShellCommand } from './agent-benchmark/run-guards.mjs'; export function launchCrashToken(runId) { const digest = createHash('sha256').update(runId).digest('hex').slice(0, 12).toUpperCase(); @@ -35,6 +36,8 @@ function successful(command) { function shellCommand(command) { const value = String(command ?? '').trim(); + const normalized = topLevelShellCommand(value); + if (normalized !== value) return normalized; const body = value.replace(/^\/bin\/(?:zsh|bash|sh) -lc\s+/, ''); return body.replace(/^["']/, '').replace(/["']$/, '').trim(); } @@ -58,7 +61,9 @@ function errorCaptureCommand(command, arm, platform) { command = shellCommand(command); if (arm === 'stim') return /(?:^|\s)stim\s+logs\s+--errors(?:\s|$)/.test(command); const explicitLogFile = - /\b(?:tail|rg|grep)\b[\s\S]*(?:\.log\b|(?:^|[\s'"])(?:\.?\/)?(?:tmp|logs?|\.expo\/dev\/logs)\/)/.test(command); + /\b(?:tail|rg|grep|sed|cat)\b[\s\S]*(?:\.log\b|(?:^|[\s'"])(?:\.?\/)?(?:tmp|logs?|\.expo\/dev\/logs)\/)/.test( + command, + ); if (platform === 'android') return /\badb\s+logcat\b/.test(command) || explicitLogFile; return /\bxcrun\s+simctl\s+spawn\b|\blog\s+(?:show|stream)\b/.test(command) || explicitLogFile; } @@ -103,6 +108,16 @@ function allowedBeforeErrorCapture(command, arm, platform) { return true; } if (sourceInspectionBeforeCapture(value, arm, platform)) return false; + if (/^(?:\.\/)?node_modules\/\.bin\/expo\s+--version$/.test(value)) return true; + if (/^node\s+-p\s+(?:process\.execPath|(["'])process\.execPath\1)$/.test(value)) return true; + if (/^print\s+-r\s+--\s+\d+\s*\|\s*tee\s+\/(?:private\/)?tmp\/[A-Za-z0-9_./-]+\.pid$/.test(value)) return true; + if ( + /^node\s+-p\s+(["'])require\.resolve\((["'])(?:expo|react-native)\/package\.json\2\)\1(?:\s*&&\s*(?:\.\/)?node_modules\/\.bin\/expo\s+--version)?$/.test( + value, + ) + ) { + return true; + } if ( /(?:\/(?:skills|skill)\/[^\s]+\/|(?:^|\s)workspace\/)SKILL\.md\b/.test(value) && /(?:^|\s)(?:cat|sed|head)(?:\s|$)/.test(value) @@ -131,14 +146,19 @@ function allowedBeforeErrorCapture(command, arm, platform) { ) { return true; } - if (/^cp\b/.test(value) && /(?:node_modules|ios\/Pods|ios\/build|android\/(?:\.gradle|app\/build))/.test(value)) { + if ( + /^(?:cp|rsync)\b/.test(value) && + /(?:node_modules|ios\/Pods|ios\/build|android\/(?:\.gradle|app\/build))/.test(value) + ) { return true; } if (arm === 'stim') { - return new RegExp(`^stim\\s+(?:worktree\\s+create|start|${platform}|logs\\s+--errors)(?:\\s|$)`).test(value); + return new RegExp( + `^stim\\s+(?:guide|doctor|worktree\\s+(?:warm|create)|start|${platform}|logs\\s+--errors)(?:\\s|$)`, + ).test(value); } return ( - /^(?:(?:[A-Za-z_][A-Za-z0-9_]*=(?:\S+|\$\([^)]*\))[;\s]+)*)(?:open\s+-a\s+Simulator|xcrun\s+simctl\s+|npx\s+expo\s+|xcodebuild\b|\.\/gradlew\b|adb\b|nohup\b|launchctl\b|ps\b|sleep\b|tail\b|cat\s+\/?tmp\/|wc\b|lsof\b|command\s+-v\b|test\b|kill\b)/.test( + /^(?:(?:[A-Za-z_][A-Za-z0-9_]*=(?:\S+|\$\([^)]*\))[;\s]+)*)(?:open\s+-a\s+Simulator|xcrun\s+simctl\s+|npx\s+expo\s+|xcodebuild\b|\.\/gradlew\b|adb\b|nohup\b|launchctl\b|ps\b|pgrep\b|sleep\b|tail\b|cat\s+\/?tmp\/|wc\b|lsof\b|command\s+-v\b|test\b|kill\b)/.test( value, ) || launchCommand(value, arm, platform) || @@ -150,10 +170,7 @@ export function launchCrashDiagnosis(commands, { dispatchAt, token, arm = 'stim' const ordered = orderedCommands(commands); const sourceMarkers = ['app/_layout.tsx', 'RootLayout']; const initialLaunchIndex = ordered.findIndex( - (command) => - successful(command) && - launchCommand(command.command, arm, platform) && - (arm !== 'stim' || (typeof command.output === 'string' && command.output.includes(token))), + (command) => successful(command) && launchCommand(command.command, arm, platform), ); if (initialLaunchIndex === -1) { return { valid: false, reason: 'launch-crash-initial-launch-evidence-missing' }; diff --git a/scripts/launch-crash-benchmark.test.mjs b/scripts/launch-crash-benchmark.test.mjs index b10fb658..30f5b8ed 100644 --- a/scripts/launch-crash-benchmark.test.mjs +++ b/scripts/launch-crash-benchmark.test.mjs @@ -46,7 +46,7 @@ describe('launch crash benchmark', () => { { id: 'launch', command: 'stim ios', - output: token, + output: 'launch com.example.app\n1 error-level record during launch (logs --errors --source device)', exitCode: 0, startedAt: '2026-09-04T12:00:01.000Z', endedAt: '2026-09-04T12:00:10.000Z', @@ -108,6 +108,60 @@ describe('launch crash benchmark', () => { }); }); + it('allows current Stim setup and narrow dependency resolution before separately captured errors', () => { + const token = launchCrashToken('setup'); + const setup = [ + 'stim guide agent', + 'git worktree add -b bench/run /tmp/bench-run HEAD', + 'stim worktree warm', + 'stim doctor --platform ios', + `node -p "require.resolve('expo/package.json')" && node_modules/.bin/expo --version`, + ].map((command, index) => ({ + id: `setup-${index}`, + command: `/bin/zsh -lc ${JSON.stringify(command)}`, + exitCode: 0, + endedAt: `2026-09-04T12:00:0${index + 1}Z`, + })); + const evidence = [ + { + id: 'launch', + command: 'stim ios', + output: 'launched com.example.app', + exitCode: 0, + endedAt: '2026-09-04T12:00:10Z', + }, + { + id: 'logs', + command: 'stim logs --errors', + output: `${token}\napp/_layout.tsx in RootLayout`, + exitCode: 0, + startedAt: '2026-09-04T12:00:11Z', + endedAt: '2026-09-04T12:00:12Z', + }, + ]; + const options = { dispatchAt: '2026-09-04T12:00:00Z', token }; + expect(launchCrashDiagnosis([...setup, ...evidence], options)).toMatchObject({ + valid: true, + commandId: 'logs', + dispatchToDiagnosisSeconds: 12, + }); + for (const command of [ + `node -p "require('fs').readFileSync('app/_layout.tsx', 'utf8')"`, + `node -p "require.resolve('expo/package.json'); require('./app/_layout.tsx')"`, + 'stim guide agent && cat app/_layout.tsx', + 'stim doctor --platform ios; git diff', + ]) { + expect(launchCrashDiagnosis([{ ...setup[0], command }, ...evidence], options)).toMatchObject({ + valid: false, + reason: 'launch-crash-pre-capture-command-not-allowed', + commandId: 'setup-0', + }); + } + expect( + launchCrashDiagnosis([...setup, evidence[0], { ...evidence[1], output: 'unrelated error' }], options), + ).toMatchObject({ valid: false, reason: 'launch-crash-error-capture-missing' }); + }); + it('rejects source inspection hidden after an allowed compound-command prefix', () => { const token = launchCrashToken('run'); const tail = [ @@ -191,6 +245,46 @@ describe('launch crash benchmark', () => { ).toMatchObject({ valid: true, commandId: 'logs' }); }); + it('allows dependency copying and PID/log diagnostics without treating source reads as logs', () => { + const token = launchCrashToken('control-setup'); + const setup = [ + 'rsync -aR node_modules ios/Pods ios/build /tmp/worktree/', + './node_modules/.bin/expo --version', + 'node -p process.execPath', + 'pgrep -P 35182 -fl .', + 'print -r -- 35182 | tee /tmp/run-metro.pid', + "sed -n '1,160p' /tmp/run-metro.log", + ].map((command, index) => ({ + id: `setup-${index}`, + command, + exitCode: 0, + endedAt: `2026-09-04T12:00:0${index + 1}Z`, + })); + const launch = { + id: 'launch', + command: 'npx expo run:ios --device SIMULATOR', + output: 'launched', + exitCode: 0, + endedAt: '2026-09-04T12:00:10Z', + }; + const logs = { + id: 'logs', + command: "sed -n '1,160p' /tmp/run-runtime.log", + output: `${token}\napp/_layout.tsx in RootLayout`, + exitCode: 0, + startedAt: '2026-09-04T12:00:11Z', + endedAt: '2026-09-04T12:00:12Z', + }; + const options = { dispatchAt: '2026-09-04T12:00:00Z', token, arm: 'control' }; + expect(launchCrashDiagnosis([...setup, launch, logs], options)).toMatchObject({ valid: true, commandId: 'logs' }); + expect( + launchCrashDiagnosis([launch, { ...logs, command: "sed -n '1,160p' app/_layout.tsx" }], options), + ).toMatchObject({ valid: false, reason: 'launch-crash-error-capture-missing' }); + expect( + launchCrashDiagnosis([launch, { ...logs, command: 'cat app/_layout.tsx /tmp/run-runtime.log' }], options), + ).toMatchObject({ valid: false, reason: 'launch-crash-pre-capture-command-not-allowed' }); + }); + it('unwraps a shell command whose nested quoting changes the closing quote', () => { const token = launchCrashToken('run'); expect( From 02aca9224be5c0ed70bb6e7f4dd6101d5da5f818 Mon Sep 17 00:00:00 2001 From: Janic Duplessis Date: Sun, 6 Sep 2026 11:12:56 -0400 Subject: [PATCH 2/2] fix: align crash export and setup command auditing --- scripts/export-benchmark-viewer.mjs | 8 +++++--- scripts/export-benchmark-viewer.test.mjs | 13 ++++++++++--- scripts/launch-crash-benchmark.mjs | 6 +++++- scripts/launch-crash-benchmark.test.mjs | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/scripts/export-benchmark-viewer.mjs b/scripts/export-benchmark-viewer.mjs index 857f2ec8..76c63721 100644 --- a/scripts/export-benchmark-viewer.mjs +++ b/scripts/export-benchmark-viewer.mjs @@ -14,6 +14,7 @@ import { userInfo } from 'node:os'; import { fileURLToPath } from 'node:url'; import { stripVTControlCharacters } from 'node:util'; import { launchCrashDiagnosis, launchCrashRecovery } from './launch-crash-benchmark.mjs'; +import { topLevelShellCommand } from './agent-benchmark/run-guards.mjs'; const modelPricing = { 'gpt-5.6-luna': { @@ -156,9 +157,10 @@ function replacementLabel(path) { } function unwrapShellCommand(command) { - return String(command) - .replace(/^\/bin\/(?:zsh|bash|sh) -lc /, '') - .replace(/^(['"])([\s\S]*)\1$/, '$2'); + const value = String(command); + const normalized = topLevelShellCommand(value); + if (normalized !== value) return normalized; + return value.replace(/^\/bin\/(?:zsh|bash|sh) -lc /, '').replace(/^(['"])([\s\S]*)\1$/, '$2'); } export function sanitizeBenchmarkText(value, replacements = []) { diff --git a/scripts/export-benchmark-viewer.test.mjs b/scripts/export-benchmark-viewer.test.mjs index 2bb5d0fc..99f063ec 100644 --- a/scripts/export-benchmark-viewer.test.mjs +++ b/scripts/export-benchmark-viewer.test.mjs @@ -998,7 +998,14 @@ describe('benchmark viewer export', () => { 'skill', ], ['worktree', '2026-09-04T12:00:03.000Z', '2026-09-04T12:00:05.000Z', 'stim worktree create bench/run', 'ready'], - ['launch', '2026-09-04T12:00:10.000Z', '2026-09-04T12:00:20.000Z', 'stim ios', token], + [ + 'metadata', + '2026-09-04T12:00:06.000Z', + '2026-09-04T12:00:07.000Z', + String.raw`/bin/zsh -lc "node -p \"require.resolve('expo/package.json')\" && node_modules/.bin/expo --version"`, + '58.0.0-canary', + ], + ['launch', '2026-09-04T12:00:10.000Z', '2026-09-04T12:00:20.000Z', 'stim ios', 'app launched'], ['logs', '2026-09-04T12:00:25.000Z', '2026-09-04T12:00:30.000Z', 'stim logs --errors', token], [ 'diagnosis', @@ -1067,7 +1074,7 @@ describe('benchmark viewer export', () => { valid: true, invalidReasons: [], dispatchToDiagnosisSeconds: 90, - diagnosisCommandCount: 5, + diagnosisCommandCount: 6, diagnosisUsage: { input_tokens: 100_000, cached_input_tokens: 80_000, @@ -1118,7 +1125,7 @@ describe('benchmark viewer export', () => { id: 'launch-crash-stim', platform: 'ios', diagnosisSeconds: 90, - diagnosisCommandCount: 5, + diagnosisCommandCount: 6, launchCrashAudit: { initialLaunchCommandId: 'launch', errorCaptureCommandId: 'logs', diff --git a/scripts/launch-crash-benchmark.mjs b/scripts/launch-crash-benchmark.mjs index 85945719..253a56b4 100644 --- a/scripts/launch-crash-benchmark.mjs +++ b/scripts/launch-crash-benchmark.mjs @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { topLevelShellCommand } from './agent-benchmark/run-guards.mjs'; +import { shellCommandSegments, topLevelShellCommand } from './agent-benchmark/run-guards.mjs'; export function launchCrashToken(runId) { const digest = createHash('sha256').update(runId).digest('hex').slice(0, 12).toUpperCase(); @@ -97,6 +97,10 @@ function sourceInspectionBeforeCapture(command, arm, platform) { function allowedBeforeErrorCapture(command, arm, platform) { const value = shellCommand(command); + if (/^(?:stim\s+(?:guide|doctor|worktree\s+warm)\b|rsync\b|pgrep\b|sed\b|cat\b)/.test(value)) { + const segments = shellCommandSegments(value); + if (segments.length > 1) return segments.every((segment) => allowedBeforeErrorCapture(segment, arm, platform)); + } if (/^tool:todo_list\b/.test(value)) return true; if (/^(?:env\s+)?(?:[^\s=]+=[^\s]+\s+)*agent-device\s+/.test(value)) return true; if ( diff --git a/scripts/launch-crash-benchmark.test.mjs b/scripts/launch-crash-benchmark.test.mjs index 30f5b8ed..f1c1f06e 100644 --- a/scripts/launch-crash-benchmark.test.mjs +++ b/scripts/launch-crash-benchmark.test.mjs @@ -150,6 +150,10 @@ describe('launch crash benchmark', () => { `node -p "require.resolve('expo/package.json'); require('./app/_layout.tsx')"`, 'stim guide agent && cat app/_layout.tsx', 'stim doctor --platform ios; git diff', + 'stim guide agent && rg "throw new Error" .', + 'stim doctor --platform ios; rg "throw new Error" .', + 'stim worktree warm | rg "throw new Error" .', + 'rsync -a node_modules /tmp/wt/ && rg "throw new Error" .', ]) { expect(launchCrashDiagnosis([{ ...setup[0], command }, ...evidence], options)).toMatchObject({ valid: false, @@ -277,6 +281,16 @@ describe('launch crash benchmark', () => { }; const options = { dispatchAt: '2026-09-04T12:00:00Z', token, arm: 'control' }; expect(launchCrashDiagnosis([...setup, launch, logs], options)).toMatchObject({ valid: true, commandId: 'logs' }); + for (const command of [ + 'pgrep -fl Metro && rg "throw new Error" .', + 'sed -n "1,160p" /tmp/run-metro.log && rg "throw new Error" .', + 'cat /tmp/run-metro.log; rg "throw new Error" .', + ]) { + expect(launchCrashDiagnosis([{ ...setup[0], command }, launch, logs], options)).toMatchObject({ + valid: false, + reason: 'launch-crash-pre-capture-command-not-allowed', + }); + } expect( launchCrashDiagnosis([launch, { ...logs, command: "sed -n '1,160p' app/_layout.tsx" }], options), ).toMatchObject({ valid: false, reason: 'launch-crash-error-capture-missing' });