fix: audit separate crash launch and error capture

docs/agent-benchmark.md

@@ -261,6 +261,10 @@ a repaired Settings screenshot. The Stim arm must preserve `stim ios` launch
 output and `stim logs --errors`; control collects the equivalent Metro and
 simulator logs manually. The injected error text is unique per run so the
 collector can prove that the reported stack and repair refer to this failure.
+The unique token and source location must appear in captured runtime errors;
+the earlier successful launch command does not have to print the token inline.
+Before capture, normal guide, doctor, worktree warming, and narrowly scoped
+installed-dependency resolution are setup, not application-source inspection.
 
 The coordinator creates a per-run fixture branch, injects and commits the
 exception, and checks out that fixture before dispatch, outside the timed

scripts/agent-benchmark/run-record.mjs

@@ -22,6 +22,28 @@ export function completedCleanupRecord(record) {
 }
 
 export function durableRunRecord(previous, next, cleanupCompleted = false) {
+  if (
+    cleanupCompleted &&
+    previous?.variant === 'launch-crash' &&
+    next.variant === 'launch-crash' &&
+    typeof previous.runId === 'string' &&
+    previous.runId === next.runId &&
+    previous.proof?.valid === true &&
+    previous.proof.kind === 'launch-crash-source-repair' &&
+    previous.proof.sourceSha256 &&
+    previous.proof.sourceSha256 === previous.evidenceSha256?.proof &&
+    next.proof?.reason === 'launch-crash-source-missing' &&
+    sameEvidence(previous, next)
+  ) {
+    const invalidReasons = next.invalidReasons.filter((reason) => reason !== 'launch-crash-source-missing');
+    return {
+      ...next,
+      proof: previous.proof,
+      evidenceSha256: { ...next.evidenceSha256, proof: previous.evidenceSha256.proof },
+      invalidReasons,
+      valid: invalidReasons.length === 0,
+    };
+  }
   if (
     cleanupCompleted &&
     previous?.valid === true &&

scripts/agent-benchmark/run-record.test.mjs

@@ -5,6 +5,39 @@ const hashes = { events: 'events', settingsPng: 'settings', transcript: 'transcr
 const valid = { valid: true, invalidReasons: [], evidenceSha256: hashes, collectedAt: 'first' };
 
 describe('durable benchmark run records', () => {
+  it('retains verified crash repair evidence after cleanup without retaining a stale audit verdict', () => {
+    const previous = {
+      runId: 'crash-run',
+      variant: 'launch-crash',
+      valid: false,
+      invalidReasons: ['launch-crash-initial-launch-missing'],
+      proof: { valid: true, kind: 'launch-crash-source-repair', sourceSha256: 'source' },
+      evidenceSha256: { ...hashes, proof: 'source' },
+    };
+    const next = {
+      runId: 'crash-run',
+      variant: 'launch-crash',
+      valid: false,
+      invalidReasons: ['launch-crash-source-missing'],
+      proof: { valid: false, reason: 'launch-crash-source-missing' },
+      evidenceSha256: hashes,
+    };
+    expect(durableRunRecord(previous, next, true)).toMatchObject({
+      valid: true,
+      proof: previous.proof,
+      invalidReasons: [],
+    });
+    expect(
+      durableRunRecord(previous, { ...next, invalidReasons: [...next.invalidReasons, 'timeout'] }, true),
+    ).toMatchObject({ valid: false, invalidReasons: ['timeout'] });
+    expect(durableRunRecord(previous, next)).toBe(next);
+    for (const changed of [
+      { ...next, runId: 'different-run' },
+      { ...next, evidenceSha256: { ...hashes, events: 'changed' } },
+    ])
+      expect(durableRunRecord(previous, changed, true)).toBe(changed);
+    expect(durableRunRecord({ ...previous, evidenceSha256: { ...hashes, proof: 'different' } }, next, true)).toBe(next);
+  });
   it('requires a successful recorded worktree cleanup', () => {
     expect(completedCleanupRecord({ cleanedAt: 'now', actions: ['stim worktree remove --force'] })).toBe(true);
     expect(completedCleanupRecord({ cleanedAt: 'now', actions: ['verified agent-device sessions empty'] })).toBe(false);

scripts/export-benchmark-viewer.mjs

@@ -14,6 +14,7 @@ import { userInfo } from 'node:os';
 import { fileURLToPath } from 'node:url';
 import { stripVTControlCharacters } from 'node:util';
 import { launchCrashDiagnosis, launchCrashRecovery } from './launch-crash-benchmark.mjs';
+import { topLevelShellCommand } from './agent-benchmark/run-guards.mjs';
 
 const modelPricing = {
   'gpt-5.6-luna': {
@@ -156,9 +157,10 @@ function replacementLabel(path) {
 }
 
 function unwrapShellCommand(command) {
-  return String(command)
-    .replace(/^\/bin\/(?:zsh|bash|sh) -lc /, '')
-    .replace(/^(['"])([\s\S]*)\1$/, '$2');
+  const value = String(command);
+  const normalized = topLevelShellCommand(value);
+  if (normalized !== value) return normalized;
+  return value.replace(/^\/bin\/(?:zsh|bash|sh) -lc /, '').replace(/^(['"])([\s\S]*)\1$/, '$2');
 }
 
 export function sanitizeBenchmarkText(value, replacements = []) {

scripts/export-benchmark-viewer.test.mjs

@@ -998,7 +998,14 @@ describe('benchmark viewer export', () => {
         'skill',
       ],
       ['worktree', '2026-09-04T12:00:03.000Z', '2026-09-04T12:00:05.000Z', 'stim worktree create bench/run', 'ready'],
-      ['launch', '2026-09-04T12:00:10.000Z', '2026-09-04T12:00:20.000Z', 'stim ios', token],
+      [
+        'metadata',
+        '2026-09-04T12:00:06.000Z',
+        '2026-09-04T12:00:07.000Z',
+        String.raw`/bin/zsh -lc "node -p \"require.resolve('expo/package.json')\" && node_modules/.bin/expo --version"`,
+        '58.0.0-canary',
+      ],
+      ['launch', '2026-09-04T12:00:10.000Z', '2026-09-04T12:00:20.000Z', 'stim ios', 'app launched'],
       ['logs', '2026-09-04T12:00:25.000Z', '2026-09-04T12:00:30.000Z', 'stim logs --errors', token],
       [
         'diagnosis',
@@ -1067,7 +1074,7 @@ describe('benchmark viewer export', () => {
         valid: true,
         invalidReasons: [],
         dispatchToDiagnosisSeconds: 90,
-        diagnosisCommandCount: 5,
+        diagnosisCommandCount: 6,
         diagnosisUsage: {
           input_tokens: 100_000,
           cached_input_tokens: 80_000,
@@ -1118,7 +1125,7 @@ describe('benchmark viewer export', () => {
       id: 'launch-crash-stim',
       platform: 'ios',
       diagnosisSeconds: 90,
-      diagnosisCommandCount: 5,
+      diagnosisCommandCount: 6,
       launchCrashAudit: {
         initialLaunchCommandId: 'launch',
         errorCaptureCommandId: 'logs',

scripts/launch-crash-benchmark.mjs

@@ -1,4 +1,5 @@
 import { createHash } from 'node:crypto';
+import { shellCommandSegments, topLevelShellCommand } from './agent-benchmark/run-guards.mjs';
 
 export function launchCrashToken(runId) {
   const digest = createHash('sha256').update(runId).digest('hex').slice(0, 12).toUpperCase();
@@ -35,6 +36,8 @@ function successful(command) {
 
 function shellCommand(command) {
   const value = String(command ?? '').trim();
+  const normalized = topLevelShellCommand(value);
+  if (normalized !== value) return normalized;
   const body = value.replace(/^\/bin\/(?:zsh|bash|sh) -lc\s+/, '');
   return body.replace(/^["']/, '').replace(/["']$/, '').trim();
 }
@@ -58,7 +61,9 @@ function errorCaptureCommand(command, arm, platform) {
   command = shellCommand(command);
   if (arm === 'stim') return /(?:^|\s)stim\s+logs\s+--errors(?:\s|$)/.test(command);
   const explicitLogFile =
-    /\b(?:tail|rg|grep)\b[\s\S]*(?:\.log\b|(?:^|[\s'"])(?:\.?\/)?(?:tmp|logs?|\.expo\/dev\/logs)\/)/.test(command);
+    /\b(?:tail|rg|grep|sed|cat)\b[\s\S]*(?:\.log\b|(?:^|[\s'"])(?:\.?\/)?(?:tmp|logs?|\.expo\/dev\/logs)\/)/.test(
+      command,
+    );
   if (platform === 'android') return /\badb\s+logcat\b/.test(command) || explicitLogFile;
   return /\bxcrun\s+simctl\s+spawn\b|\blog\s+(?:show|stream)\b/.test(command) || explicitLogFile;
 }
@@ -92,6 +97,10 @@ function sourceInspectionBeforeCapture(command, arm, platform) {
 
 function allowedBeforeErrorCapture(command, arm, platform) {
   const value = shellCommand(command);
+  if (/^(?:stim\s+(?:guide|doctor|worktree\s+warm)\b|rsync\b|pgrep\b|sed\b|cat\b)/.test(value)) {
+    const segments = shellCommandSegments(value);
+    if (segments.length > 1) return segments.every((segment) => allowedBeforeErrorCapture(segment, arm, platform));
+  }
   if (/^tool:todo_list\b/.test(value)) return true;
   if (/^(?:env\s+)?(?:[^\s=]+=[^\s]+\s+)*agent-device\s+/.test(value)) return true;
   if (
@@ -103,6 +112,16 @@ function allowedBeforeErrorCapture(command, arm, platform) {
     return true;
   }
   if (sourceInspectionBeforeCapture(value, arm, platform)) return false;
+  if (/^(?:\.\/)?node_modules\/\.bin\/expo\s+--version$/.test(value)) return true;
+  if (/^node\s+-p\s+(?:process\.execPath|(["'])process\.execPath\1)$/.test(value)) return true;
+  if (/^print\s+-r\s+--\s+\d+\s*\|\s*tee\s+\/(?:private\/)?tmp\/[A-Za-z0-9_./-]+\.pid$/.test(value)) return true;
+  if (
+    /^node\s+-p\s+(["'])require\.resolve\((["'])(?:expo|react-native)\/package\.json\2\)\1(?:\s*&&\s*(?:\.\/)?node_modules\/\.bin\/expo\s+--version)?$/.test(
+      value,
+    )
+  ) {
+    return true;
+  }
   if (
     /(?:\/(?:skills|skill)\/[^\s]+\/|(?:^|\s)workspace\/)SKILL\.md\b/.test(value) &&
     /(?:^|\s)(?:cat|sed|head)(?:\s|$)/.test(value)
@@ -131,14 +150,19 @@ function allowedBeforeErrorCapture(command, arm, platform) {
   ) {
     return true;
   }
-  if (/^cp\b/.test(value) && /(?:node_modules|ios\/Pods|ios\/build|android\/(?:\.gradle|app\/build))/.test(value)) {
+  if (
+    /^(?:cp|rsync)\b/.test(value) &&
+    /(?:node_modules|ios\/Pods|ios\/build|android\/(?:\.gradle|app\/build))/.test(value)
+  ) {
     return true;
   }
   if (arm === 'stim') {
-    return new RegExp(`^stim\\s+(?:worktree\\s+create|start|${platform}|logs\\s+--errors)(?:\\s|$)`).test(value);
+    return new RegExp(
+      `^stim\\s+(?:guide|doctor|worktree\\s+(?:warm|create)|start|${platform}|logs\\s+--errors)(?:\\s|$)`,
+    ).test(value);
   }
   return (
-    /^(?:(?:[A-Za-z_][A-Za-z0-9_]*=(?:\S+|\$\([^)]*\))[;\s]+)*)(?:open\s+-a\s+Simulator|xcrun\s+simctl\s+|npx\s+expo\s+|xcodebuild\b|\.\/gradlew\b|adb\b|nohup\b|launchctl\b|ps\b|sleep\b|tail\b|cat\s+\/?tmp\/|wc\b|lsof\b|command\s+-v\b|test\b|kill\b)/.test(
+    /^(?:(?:[A-Za-z_][A-Za-z0-9_]*=(?:\S+|\$\([^)]*\))[;\s]+)*)(?:open\s+-a\s+Simulator|xcrun\s+simctl\s+|npx\s+expo\s+|xcodebuild\b|\.\/gradlew\b|adb\b|nohup\b|launchctl\b|ps\b|pgrep\b|sleep\b|tail\b|cat\s+\/?tmp\/|wc\b|lsof\b|command\s+-v\b|test\b|kill\b)/.test(
       value,
     ) ||
     launchCommand(value, arm, platform) ||
@@ -150,10 +174,7 @@ export function launchCrashDiagnosis(commands, { dispatchAt, token, arm = 'stim'
   const ordered = orderedCommands(commands);
   const sourceMarkers = ['app/_layout.tsx', 'RootLayout'];
   const initialLaunchIndex = ordered.findIndex(
-    (command) =>
-      successful(command) &&
-      launchCommand(command.command, arm, platform) &&
-      (arm !== 'stim' || (typeof command.output === 'string' && command.output.includes(token))),
+    (command) => successful(command) && launchCommand(command.command, arm, platform),
   );
   if (initialLaunchIndex === -1) {
     return { valid: false, reason: 'launch-crash-initial-launch-evidence-missing' };

scripts/launch-crash-benchmark.test.mjs

@@ -46,7 +46,7 @@ describe('launch crash benchmark', () => {
         {
           id: 'launch',
           command: 'stim ios',
-          output: token,
+          output: 'launch com.example.app\n1 error-level record during launch (logs --errors --source device)',
           exitCode: 0,
           startedAt: '2026-09-04T12:00:01.000Z',
           endedAt: '2026-09-04T12:00:10.000Z',
@@ -108,6 +108,64 @@ describe('launch crash benchmark', () => {
     });
   });
 
+  it('allows current Stim setup and narrow dependency resolution before separately captured errors', () => {
+    const token = launchCrashToken('setup');
+    const setup = [
+      'stim guide agent',
+      'git worktree add -b bench/run /tmp/bench-run HEAD',
+      'stim worktree warm',
+      'stim doctor --platform ios',
+      `node -p "require.resolve('expo/package.json')" && node_modules/.bin/expo --version`,
+    ].map((command, index) => ({
+      id: `setup-${index}`,
+      command: `/bin/zsh -lc ${JSON.stringify(command)}`,
+      exitCode: 0,
+      endedAt: `2026-09-04T12:00:0${index + 1}Z`,
+    }));
+    const evidence = [
+      {
+        id: 'launch',
+        command: 'stim ios',
+        output: 'launched com.example.app',
+        exitCode: 0,
+        endedAt: '2026-09-04T12:00:10Z',
+      },
+      {
+        id: 'logs',
+        command: 'stim logs --errors',
+        output: `${token}\napp/_layout.tsx in RootLayout`,
+        exitCode: 0,
+        startedAt: '2026-09-04T12:00:11Z',
+        endedAt: '2026-09-04T12:00:12Z',
+      },
+    ];
+    const options = { dispatchAt: '2026-09-04T12:00:00Z', token };
+    expect(launchCrashDiagnosis([...setup, ...evidence], options)).toMatchObject({
+      valid: true,
+      commandId: 'logs',
+      dispatchToDiagnosisSeconds: 12,
+    });
+    for (const command of [
+      `node -p "require('fs').readFileSync('app/_layout.tsx', 'utf8')"`,
+      `node -p "require.resolve('expo/package.json'); require('./app/_layout.tsx')"`,
+      'stim guide agent && cat app/_layout.tsx',
+      'stim doctor --platform ios; git diff',
+      'stim guide agent && rg "throw new Error" .',
+      'stim doctor --platform ios; rg "throw new Error" .',
+      'stim worktree warm | rg "throw new Error" .',
+      'rsync -a node_modules /tmp/wt/ && rg "throw new Error" .',
+    ]) {
+      expect(launchCrashDiagnosis([{ ...setup[0], command }, ...evidence], options)).toMatchObject({
+        valid: false,
+        reason: 'launch-crash-pre-capture-command-not-allowed',
+        commandId: 'setup-0',
+      });
+    }
+    expect(
+      launchCrashDiagnosis([...setup, evidence[0], { ...evidence[1], output: 'unrelated error' }], options),
+    ).toMatchObject({ valid: false, reason: 'launch-crash-error-capture-missing' });
+  });
+
   it('rejects source inspection hidden after an allowed compound-command prefix', () => {
     const token = launchCrashToken('run');
     const tail = [
@@ -191,6 +249,56 @@ describe('launch crash benchmark', () => {
     ).toMatchObject({ valid: true, commandId: 'logs' });
   });
 
+  it('allows dependency copying and PID/log diagnostics without treating source reads as logs', () => {
+    const token = launchCrashToken('control-setup');
+    const setup = [
+      'rsync -aR node_modules ios/Pods ios/build /tmp/worktree/',
+      './node_modules/.bin/expo --version',
+      'node -p process.execPath',
+      'pgrep -P 35182 -fl .',
+      'print -r -- 35182 | tee /tmp/run-metro.pid',
+      "sed -n '1,160p' /tmp/run-metro.log",
+    ].map((command, index) => ({
+      id: `setup-${index}`,
+      command,
+      exitCode: 0,
+      endedAt: `2026-09-04T12:00:0${index + 1}Z`,
+    }));
+    const launch = {
+      id: 'launch',
+      command: 'npx expo run:ios --device SIMULATOR',
+      output: 'launched',
+      exitCode: 0,
+      endedAt: '2026-09-04T12:00:10Z',
+    };
+    const logs = {
+      id: 'logs',
+      command: "sed -n '1,160p' /tmp/run-runtime.log",
+      output: `${token}\napp/_layout.tsx in RootLayout`,
+      exitCode: 0,
+      startedAt: '2026-09-04T12:00:11Z',
+      endedAt: '2026-09-04T12:00:12Z',
+    };
+    const options = { dispatchAt: '2026-09-04T12:00:00Z', token, arm: 'control' };
+    expect(launchCrashDiagnosis([...setup, launch, logs], options)).toMatchObject({ valid: true, commandId: 'logs' });
+    for (const command of [
+      'pgrep -fl Metro && rg "throw new Error" .',
+      'sed -n "1,160p" /tmp/run-metro.log && rg "throw new Error" .',
+      'cat /tmp/run-metro.log; rg "throw new Error" .',
+    ]) {
+      expect(launchCrashDiagnosis([{ ...setup[0], command }, launch, logs], options)).toMatchObject({
+        valid: false,
+        reason: 'launch-crash-pre-capture-command-not-allowed',
+      });
+    }
+    expect(
+      launchCrashDiagnosis([launch, { ...logs, command: "sed -n '1,160p' app/_layout.tsx" }], options),
+    ).toMatchObject({ valid: false, reason: 'launch-crash-error-capture-missing' });
+    expect(
+      launchCrashDiagnosis([launch, { ...logs, command: 'cat app/_layout.tsx /tmp/run-runtime.log' }], options),
+    ).toMatchObject({ valid: false, reason: 'launch-crash-pre-capture-command-not-allowed' });
+  });
+
   it('unwraps a shell command whose nested quoting changes the closing quote', () => {
     const token = launchCrashToken('run');
     expect(