fix: preserve complete CI test output and exit status

packages/wb/README.md

@@ -178,3 +178,5 @@ when the Markdown parser treats their source as prose (for example,
 `{{ "ﾃｽﾄ" }}` or `[visible]{title="ﾃｽﾄ"}`). Half-width kana is checked in plain paragraph and list text, but not in headings
 or bold/italic spans or blockquotes. The other rules also inspect headings and
 blockquotes. Markdown coverage is not exhaustive.
+
+`wb test-on-ci` streams raw output and overwrites `.wb/test-ci.log` in the selected project on each run. Failed commands retain their exit status after output has been flushed; `--dry-run` preserves the previous log.

packages/wb/src/commands/testOnCi.ts

@@ -4,12 +4,15 @@ import path from 'node:path';
 import chalk from 'chalk';
 import type { ArgumentsCamelCase, CommandModule, InferredOptionTypes } from 'yargs';
 
+import type { Project } from '../project.js';
 import { findDescendantProjects } from '../project.js';
 import { toDevNull } from '../scripts/builder.js';
 import { dockerScripts } from '../scripts/dockerScripts.js';
 import { selectScripts } from '../scripts/execution/selectScripts.js';
 import { runWithSpawn, runWithSpawnInParallel } from '../scripts/run.js';
 import type { sharedOptionsBuilder } from '../sharedOptionsBuilder.js';
+import { PackageCommandError } from '../utils/packageCommand.js';
+import { startVerificationOutput } from '../utils/verificationOutput.js';
 import { promisePool } from '../utils/promisePool.js';
 import { findTestStructureViolations, printTestStructureViolations } from '../utils/testStructure.js';
 
@@ -47,7 +50,25 @@ export async function testOnCi(
     process.exit(1);
   }
 
-  for (const project of projects.descendants) {
+  const reporter = argv.dryRun
+    ? undefined
+    : startVerificationOutput(path.join(projects.self.dirPath, '.wb', 'test-ci.log'), true);
+  try {
+    await runTests(projects.descendants, argv);
+    if (!process.exitCode) reporter?.succeed();
+  } catch (error) {
+    if (!(error instanceof PackageCommandError)) console.error(error);
+    process.exitCode = error instanceof PackageCommandError ? error.exitCode : 1;
+  } finally {
+    await reporter?.finish(Number(process.exitCode ?? 0));
+  }
+}
+
+async function runTests(
+  projects: Project[],
+  argv: ArgumentsCamelCase<InferredOptionTypes<typeof testOnCiBuilder & typeof sharedOptionsBuilder>>
+): Promise<void> {
+  for (const project of projects) {
     project.env.CI ||= '1';
     // Overwrite, not ||=: project.env already carries the dotenv-derived value.
     project.env.WB_ENV = process.env.WB_ENV;
@@ -65,21 +86,21 @@ export async function testOnCi(
 
     const hasDockerfile = project.hasDockerfile;
     if (hasDockerfile) {
-      await runWithSpawnInParallel(dockerScripts.stopAll(), project, argv);
+      await runCiStep(dockerScripts.stopAll(), project, argv);
     }
     const defaultUnitTargets = getDefaultUnitTargets(project);
     if (defaultUnitTargets !== false) {
       // CI mode disallows `only` to avoid including debug tests
       const unitArgv = { ...argv, targets: defaultUnitTargets };
-      await runWithSpawnInParallel(scripts.testUnit(project, unitArgv).replaceAll(' --allowOnly', ''), project, argv);
+      await runCiStep(scripts.testUnit(project, unitArgv).replaceAll(' --allowOnly', ''), project, argv);
     }
     if (fs.existsSync(path.join(project.dirPath, 'test', 'e2e'))) {
       // Confirm dev server startup for consistency across projects with E2E tests.
-      await runWithSpawnInParallel(await scripts.testStart(project, argv), project, argv);
+      await runCiStep(await scripts.testStart(project, argv), project, argv);
       await promisePool.promiseAll();
       if (hasDockerfile) {
         project.env.WB_DOCKER ||= '1';
-        await runWithSpawn(`${scripts.buildDocker(project, 'test')}${toDevNull(argv)}`, project, argv);
+        await runCiStep(`${scripts.buildDocker(project, 'test')}${toDevNull(argv)}`, project, argv);
       }
       const script = hasDockerfile
         ? await scripts.testE2EDocker(project, argv, {})
@@ -99,8 +120,17 @@ export async function testOnCi(
         process.exitCode = e2eExitCode;
       }
       if (hasDockerfile) {
-        await runWithSpawn(dockerScripts.stop(project), project, argv);
+        await runCiStep(dockerScripts.stop(project), project, argv);
       }
     }
   }
 }
+
+async function runCiStep(
+  script: string,
+  project: Project,
+  argv: ArgumentsCamelCase<InferredOptionTypes<typeof testOnCiBuilder & typeof sharedOptionsBuilder>>
+): Promise<void> {
+  const exitCode = await runWithSpawnInParallel(script, project, argv, { exitIfFailed: false });
+  if (exitCode !== 0) throw new PackageCommandError(exitCode);
+}

packages/wb/src/utils/verificationOutput.ts

@@ -10,8 +10,11 @@ export function isCapturingVerificationOutput(): boolean {
   return capturingVerificationOutput;
 }
 
-/** Saves output as it arrives; successful verification exposes only its final recap. */
-export function startVerificationOutput(logPath: string): {
+/** Saves output as it arrives, optionally streaming it instead of showing only a verification recap. */
+export function startVerificationOutput(
+  logPath: string,
+  streamOutput = false
+): {
   startStep: (name?: string) => void;
   succeed: () => void;
   finish: (exitCode: number) => Promise<void>;
@@ -37,7 +40,7 @@ export function startVerificationOutput(logPath: string): {
           : chunk;
       logSize += fs.writeSync(logFile, buffer);
       const done = typeof encodingOrCallback === 'function' ? encodingOrCallback : callback;
-      if (succeeded) return original.call(stream, buffer, undefined, done);
+      if (succeeded || streamOutput) return original.call(stream, buffer, undefined, done);
       if (done) queueMicrotask(done);
       return true;
     }) as typeof original;
@@ -59,16 +62,22 @@ export function startVerificationOutput(logPath: string): {
     process.stderr.write = stderrWrite;
     globalThis.console = originalConsole;
     process.removeListener('exit', onExit);
-    const tail = succeeded ? '' : readFailureTail(logFile, stepStart, logSize);
+    const tail = succeeded || streamOutput ? '' : readFailureTail(logFile, stepStart, logSize);
     fs.closeSync(logFile);
-    const message = `${succeeded ? 'Full log' : 'Verification failed. Full log'}: ${logPath}\n`;
+    const message = `${succeeded || streamOutput ? 'Full log' : 'Verification failed. Full log'}: ${logPath}\n`;
     fs.appendFileSync(logPath, message);
-    const output = succeeded
-      ? message
-      : `Failed step: ${stepName ?? 'verification setup'} (exit code ${exitCode})\n${tail}${message}`;
-    await new Promise<void>((resolve, reject) => {
-      stdoutWrite(output, (error) => (error ? reject(error) : resolve()));
-    });
+    const output =
+      succeeded || streamOutput
+        ? message
+        : `Failed step: ${stepName ?? 'verification setup'} (exit code ${exitCode})\n${tail}${message}`;
+    await Promise.all([
+      new Promise<void>((resolve, reject) => {
+        stdoutWrite(output, (error) => (error ? reject(error) : resolve()));
+      }),
+      new Promise<void>((resolve, reject) => {
+        stderrWrite('', (error) => (error ? reject(error) : resolve()));
+      }),
+    ]);
   };
   // An unexpected process.exit() still closes the saved log. Normal failures await the flush.
   const onExit = (exitCode: number): void => {

packages/wb/test/unit/verifyOutput.test.ts

@@ -152,6 +152,35 @@ it('preserves the previous log during dry-run and keeps standalone tests verbose
   expect(result.stdout).toContain('RAW_TEST_STDOUT');
 });
 
+it.each([0, 7])('saves and flushes complete CI output with exit code %s', async (exitCode) => {
+  const dir = await createFixture();
+  const logPath = path.join(dir, '.wb/test-ci.log');
+  await fs.mkdir(path.dirname(logPath), { recursive: true });
+  await fs.writeFile(logPath, 'PREVIOUS_RUN');
+  const dryRun = runCli(dir, ['test-on-ci', '--dry-run']);
+  expect(dryRun.status, dryRun.stderr).toBe(0);
+  expect(await fs.readFile(logPath, 'utf8')).toBe('PREVIOUS_RUN');
+  await fs.writeFile(
+    path.join(dir, 'test/unit/example.test.ts'),
+    `import fs from 'node:fs';
+import { test } from 'bun:test';
+test('large output', () => {
+  fs.writeFileSync(1, 'CI_STDOUT_α😀\\n'.repeat(20_000));
+  fs.writeFileSync(2, 'CI_STDERR_α😀\\n'.repeat(20_000));
+  ${exitCode ? `process.exit(${exitCode});` : ''}
+});`
+  );
+  const result = runCli(dir, ['test-on-ci']);
+  expect(result.status, result.stderr).toBe(exitCode);
+  const log = await fs.readFile(logPath, 'utf8');
+  for (const output of [log, result.stdout + result.stderr]) {
+    expect(output.match(/CI_STDOUT_α😀/g)).toHaveLength(20_000);
+    expect(output.match(/CI_STDERR_α😀/g)).toHaveLength(20_000);
+    expect(output).not.toContain('PREVIOUS_RUN');
+  }
+  expect(result.stdout).toContain(logPath);
+});
+
 async function createFixture(): Promise<string> {
   const tmp = path.resolve('.tmp');
   await fs.mkdir(tmp, { recursive: true });
@@ -178,5 +207,10 @@ async function createFixture(): Promise<string> {
 }
 
 function runCli(dir: string, args: string[]): SpawnSyncReturns<string> {
-  return spawnSync('node', [cliPath, ...args], { cwd: dir, encoding: 'utf8', timeout: 30_000 });
+  return spawnSync('node', [cliPath, ...args], {
+    cwd: dir,
+    encoding: 'utf8',
+    timeout: 30_000,
+    maxBuffer: 4 * 1024 * 1024,
+  });
 }

packages/wbfy/src/generators/selfContainedWorkflow.ts

@@ -190,7 +190,29 @@ function buildTestWorkflow(config: PackageConfig, allPackageConfigs: PackageConf
       : []),
     ...(hasTypecheck ? [{ run: 'bun run typecheck', ...fnoxEnv }] : []),
     { run: 'bun run lint', ...fnoxEnv },
-    { run: 'bun run test/ci', ...fnoxEnv },
+    {
+      name: 'Test',
+      id: 'test',
+      run: `log_dir=$(mktemp -d "$RUNNER_TEMP/test-output.XXXXXX")
+echo "log_path=$log_dir/test.log" >> "$GITHUB_OUTPUT"
+echo "artifact_name=$(basename "$log_dir")" >> "$GITHUB_OUTPUT"
+set +e
+bun run test/ci 2>&1 | tee "$log_dir/test.log"
+test_exit=\${PIPESTATUS[0]}
+exit "$test_exit"`,
+      ...fnoxEnv,
+    },
+    {
+      name: 'Upload test log',
+      if: "${{ always() && steps.test.outputs.log_path != '' }}",
+      uses: uploadArtifactAction,
+      with: {
+        name: '${{ steps.test.outputs.artifact_name }}',
+        path: '${{ steps.test.outputs.log_path }}',
+        'retention-days': 14,
+        'if-no-files-found': 'warn',
+      },
+    },
     ...(playwrightDirPaths.length > 0
       ? [
           {

packages/wbfy/test/unit/selfContainedWorkflow.test.ts

@@ -1,3 +1,4 @@
+import { spawnSync } from 'node:child_process';
 import fs from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
@@ -46,7 +47,6 @@ test('generates self-contained test and semantic-pr workflows without reusable-w
     expect(testContent).not.toContain('reusable-workflows');
     const testWorkflow = YAML.parse(testContent) as ParsedWorkflow;
     const runCommands = testWorkflow.jobs.test?.steps.map((step) => step.run).filter(Boolean);
-    expect(runCommands).toContain('bun run test/ci');
     // No TypeScript and no Playwright in this repository.
     expect(runCommands).not.toContain('bun run typecheck');
     expect(testContent).not.toContain('playwright');
@@ -82,7 +82,7 @@ test('includes typecheck, Playwright caching and step-scoped FNOX_AGE_KEY when t
     // Step-scoped, not job-wide: `bun install` must not see the age identity.
     const installStep = steps.find((step) => step.name === 'Install dependencies');
     expect(installStep?.env?.FNOX_AGE_KEY).toBeUndefined();
-    const testStep = steps.find((step) => step.run === 'bun run test/ci');
+    const testStep = steps.find((step) => step.name === 'Test');
     expect(testStep?.env?.FNOX_AGE_KEY).toBe('${{ secrets.FNOX_AGE_KEY }}');
   });
 });
@@ -120,11 +120,45 @@ test('installs Playwright browsers from the declaring workspace package in a mon
     expect(cacheStep?.with?.key).toBe(
       'playwright-${{ runner.os }}-${{ steps.playwright-version-0.outputs.version }}-${{ steps.playwright-version-1.outputs.version }}'
     );
-    const uploadStep = steps.find((step) => step.uses?.startsWith('actions/upload-artifact@'));
+    const uploadStep = steps.find((step) => step.name === 'Upload test results');
     expect(uploadStep?.with?.path).toBe('packages/app/test-results\npackages/web/test-results');
   });
 });
 
+test('generated test step preserves full logs and failing exit codes', async () => {
+  await withTempRepo(async (dirPath) => {
+    await generateSelfContainedWorkflows(createConfig({ dirPath, isRoot: true, isWillBoosterRepo: false }));
+    await promisePool.promiseAll();
+    const workflow = YAML.parse(
+      await fs.readFile(path.join(dirPath, '.github/workflows/test.yml'), 'utf8')
+    ) as ParsedWorkflow;
+    const script = workflow.jobs.test!.steps.find((step) => step.name === 'Test')!.run!;
+    await fs.writeFile(
+      path.join(dirPath, 'emit.js'),
+      String.raw`require('node:fs').writeFileSync(1, 'evidence\n'.repeat(100_000)); process.exit(Number(process.argv[2]));`
+    );
+    for (const exitCode of [0, 7]) {
+      await fs.writeFile(
+        path.join(dirPath, 'package.json'),
+        JSON.stringify({ scripts: { 'test/ci': `node emit.js ${exitCode}` } })
+      );
+      const outputPath = path.join(dirPath, `outputs-${exitCode}`);
+      const result = spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], {
+        cwd: dirPath,
+        env: { ...process.env, RUNNER_TEMP: dirPath, GITHUB_OUTPUT: outputPath },
+        encoding: 'utf8',
+        maxBuffer: 2 * 1024 * 1024,
+      });
+      expect(result.status, result.stderr).toBe(exitCode);
+      const outputs = await fs.readFile(outputPath, 'utf8');
+      const logPath = outputs.match(/^log_path=(.+)$/m)![1]!;
+      const log = await fs.readFile(logPath, 'utf8');
+      expect(log.match(/evidence/g)).toHaveLength(100_000);
+      expect(result.stdout.match(/evidence/g)).toHaveLength(100_000);
+    }
+  });
+});
+
 test('semantic-pr workflow grants the permissions the action needs', async () => {
   await withTempRepo(async (dirPath) => {
     const config = createConfig({