From 1b34e7058e7388a39202db57d7f6f1043cce89c7 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Mon, 13 Jul 2026 11:44:18 +0200 Subject: [PATCH] perf(ci): add focused benchmark commands --- .github/workflows/benchmarks.yml | 65 ++++++++++++++++++++++++-------- docs/BENCHMARKS.md | 16 +++++++- package.json | 3 +- scripts/run-quick-bench.mjs | 64 +++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 18 deletions(-) create mode 100644 scripts/run-quick-bench.mjs diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7570efad..1358cad2 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -12,11 +12,12 @@ on: suite: description: Benchmark suite required: true - default: full + default: quick type: choice options: - - full + - quick - web + - full permissions: contents: read @@ -28,7 +29,7 @@ jobs: if: | (github.event_name == 'issue_comment' && github.event.issue.pull_request && - startsWith(github.event.comment.body, '/bench') && + contains(fromJSON('["/bench","/bench quick","/bench web","/bench full"]'), github.event.comment.body) && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) ) || (github.event_name == 'workflow_dispatch') runs-on: ubuntu-22.04 @@ -54,10 +55,21 @@ jobs: throw new Error('Unable to determine pull request number'); } + const commentSuites = { + '/bench': 'quick', + '/bench quick': 'quick', + '/bench web': 'web', + '/bench full': 'full', + }; + const commentBody = context.payload.comment?.body?.trim().toLowerCase(); const suite = context.eventName === 'workflow_dispatch' - ? context.payload.inputs?.suite ?? 'full' - : 'full'; - const benchmarkScripts = { full: 'benchmark', web: 'benchmark:web' }; + ? context.payload.inputs?.suite ?? 'quick' + : commentSuites[commentBody]; + const benchmarkScripts = { + quick: 'benchmark:quick', + full: 'benchmark', + web: 'benchmark:web', + }; if (!Object.hasOwn(benchmarkScripts, suite)) { throw new Error(`Unsupported benchmark suite: ${suite}`); } @@ -124,7 +136,6 @@ jobs: with: ref: ${{ steps.pr.outputs.base_sha }} path: bench-base - fetch-depth: 0 submodules: recursive - name: Checkout head @@ -133,9 +144,16 @@ jobs: with: ref: ${{ steps.pr.outputs.head_sha }} path: bench-head - fetch-depth: 0 submodules: recursive + - name: Checkout benchmark tools + if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.suite == 'quick' + uses: actions/checkout@v4 + with: + ref: ${{ github.workflow_sha }} + path: bench-tools + sparse-checkout: scripts + - name: Use Node.js 20.x if: steps.pr.outputs.same_repo == 'true' uses: actions/setup-node@v4 @@ -151,8 +169,6 @@ jobs: - name: Install Rust toolchain if: steps.pr.outputs.same_repo == 'true' uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - name: Add wasm targets if: steps.pr.outputs.same_repo == 'true' @@ -219,8 +235,16 @@ jobs: run: | set -o pipefail status=0 - env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$? - pnpm run benchmark:aggregate || status=$? + if [ "$BENCHMARK_SCRIPT" = "benchmark:quick" ]; then + env -u CI TREECRDT_BENCH_ROOT="$PWD" \ + node "$GITHUB_WORKSPACE/bench-tools/scripts/run-quick-bench.mjs" \ + 2>&1 | tee benchmark.log || status=$? + else + env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$? + fi + if [ ! -f benchmarks/summary.json ]; then + pnpm run benchmark:aggregate || status=$? + fi exit "$status" - name: Run benchmarks (head) @@ -234,8 +258,16 @@ jobs: run: | set -o pipefail status=0 - env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$? - pnpm run benchmark:aggregate || status=$? + if [ "$BENCHMARK_SCRIPT" = "benchmark:quick" ]; then + env -u CI TREECRDT_BENCH_ROOT="$PWD" \ + node "$GITHUB_WORKSPACE/bench-tools/scripts/run-quick-bench.mjs" \ + 2>&1 | tee benchmark.log || status=$? + else + env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$? + fi + if [ ! -f benchmarks/summary.json ]; then + pnpm run benchmark:aggregate || status=$? + fi exit "$status" - name: Upload benchmark artifacts @@ -342,8 +374,9 @@ jobs: const baseOutcome = '${{ steps.benchmark_base.outcome }}'; const headOutcome = '${{ steps.benchmark_head.outcome }}'; const benchmarkFailed = [baseOutcome, headOutcome].includes('failure'); - const triggerLabel = - context.eventName === 'workflow_dispatch' ? '`workflow_dispatch`' : '`/bench`'; + const triggerLabel = context.eventName === 'workflow_dispatch' + ? '`workflow_dispatch`' + : `\`${context.payload.comment.body.trim()}\``; const parts = [ marker, diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index efa6a855..f88f47bc 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -14,6 +14,7 @@ pnpm benchmark:sync:help Useful top-level entrypoints: ```sh +pnpm benchmark:quick pnpm benchmark pnpm benchmark:sqlite-node pnpm benchmark:sqlite-node:ops @@ -29,7 +30,20 @@ pnpm benchmark:wasm pnpm benchmark:postgres ``` -`pnpm benchmark` writes JSON results under `benchmarks/`. +Both `pnpm benchmark:quick` and `pnpm benchmark` write JSON results under `benchmarks/`. +The quick suite samples the main local implementations at a representative scale; the full suite retains the exhaustive size and iteration matrix. + +### Pull Request Comparisons + +Repository members can trigger base/head comparisons from a pull request: + +- `/bench` or `/bench quick`: representative browser/OPFS, native SQLite, direct sync, WASM, and Rust core coverage +- `/bench web`: full browser memory and OPFS coverage +- `/bench full`: the exhaustive suite + +Use quick for routine feedback and full when confirming a broad or performance-sensitive change. +Quick keeps browser memory/OPFS operation coverage and native direct-sync coverage; use +`/bench web` for the browser-specific sync matrix. ## Which Benchmark Answers What? diff --git a/package.json b/package.json index 264aed3d..21840bc8 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "build": "pnpm -r --if-present run build", "test:browser": "pnpm -C packages/treecrdt-wa-sqlite/e2e test:e2e", "test:native-node": "pnpm -C packages/sync-protocol/material/sqlite run test && pnpm -C packages/treecrdt-sqlite-node run test && pnpm -C packages/treecrdt-wa-sqlite run test && pnpm -C packages/treecrdt-benchmark run test", - "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm run benchmark:web && pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sync:direct && pnpm run benchmark:wasm && pnpm run benchmark:postgres && pnpm run benchmark:core && pnpm run benchmark:aggregate", + "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm run benchmark:web && pnpm run benchmark:sqlite-node && pnpm run benchmark:wasm && pnpm run benchmark:postgres && pnpm run benchmark:core && pnpm run benchmark:aggregate", + "benchmark:quick": "node scripts/run-quick-bench.mjs", "benchmark:sqlite-node": "pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sqlite-node:sync", "benchmark:core": "cargo bench -p treecrdt-core --bench core --features bench", "benchmark:aggregate": "node scripts/aggregate-bench.mjs", diff --git a/scripts/run-quick-bench.mjs b/scripts/run-quick-bench.mjs new file mode 100644 index 00000000..080d7273 --- /dev/null +++ b/scripts/run-quick-bench.mjs @@ -0,0 +1,64 @@ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { repoRootFromImportMeta } from './repo-root.mjs'; + +const repoRoot = process.env.TREECRDT_BENCH_ROOT + ? path.resolve(process.env.TREECRDT_BENCH_ROOT) + : repoRootFromImportMeta(import.meta.url, 1); +const quickEnv = { + BENCH_ITERATIONS: '3', + BENCH_WARMUP: '1', +}; + +const commands = [ + ['pnpm', ['run', 'benchmark:web', '--grep-invert=sync']], + ['pnpm', ['-C', 'packages/treecrdt-sqlite-node', 'run', 'benchmark:ops', '--', '--count=1000']], + [ + 'pnpm', + ['-C', 'packages/treecrdt-sqlite-node', 'run', 'benchmark:note-paths', '--', '--count=10000'], + ], + [process.execPath, ['scripts/run-sync-bench.mjs', 'direct', '--count=1000']], + [ + 'pnpm', + [ + '-C', + 'packages/treecrdt-wasm-js', + 'run', + 'benchmark', + '--', + '--sizes=1000', + '--workloads=insert-move,insert-chain', + ], + ], + [ + 'cargo', + [ + 'bench', + '-p', + 'treecrdt-core', + '--bench', + 'core', + '--features', + 'bench', + '--', + '--count=1000', + ], + ], + ['pnpm', ['run', 'benchmark:aggregate']], +]; + +fs.rmSync(path.join(repoRoot, 'benchmarks'), { recursive: true, force: true }); +fs.mkdirSync(path.join(repoRoot, 'benchmarks'), { recursive: true }); + +for (const [command, args] of commands) { + console.log(`\n[benchmark:quick] ${command} ${args.join(' ')}`); + const result = spawnSync(command, args, { + cwd: repoRoot, + env: { ...process.env, ...quickEnv }, + stdio: 'inherit', + }); + if (result.error) throw result.error; + if (result.status !== 0) process.exit(result.status ?? 1); +}