From 036e9e04d4ce913ab7cf5b7c676cb25fe2053c96 Mon Sep 17 00:00:00 2001
From: Kira <kira@ariaki.me>
Date: Tue, 21 Jul 2026 01:24:53 -0400
Subject: [PATCH] bench: add generic runner and M1 Max Qwen3.6 results

---
 .github/workflows/ci.yml                     |   9 ++
 benchmarks/README.md                         |  46 +++++++-
 benchmarks/results/m1-max-64gb_baseRT.csv    |  13 +++
 benchmarks/results/m1-max-64gb_baseRT.md     |  51 ++++++++
 benchmarks/results/m1-max-64gb_baseRT.raw.md |  40 +++++++
 benchmarks/scripts/parse_bench.py            |  89 ++++++++++++++
 benchmarks/scripts/run_bench.sh              | 116 +++++++++++++++++++
 benchmarks/tests/test_parse_bench.py         |  72 ++++++++++++
 benchmarks/tests/test_run_bench.py           |  89 ++++++++++++++
 docs/guides/benchmarking.md                  |  10 +-
 10 files changed, 525 insertions(+), 10 deletions(-)
 create mode 100644 benchmarks/results/m1-max-64gb_baseRT.csv
 create mode 100644 benchmarks/results/m1-max-64gb_baseRT.md
 create mode 100644 benchmarks/results/m1-max-64gb_baseRT.raw.md
 create mode 100755 benchmarks/scripts/parse_bench.py
 create mode 100755 benchmarks/scripts/run_bench.sh
 create mode 100644 benchmarks/tests/test_parse_bench.py
 create mode 100644 benchmarks/tests/test_run_bench.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 790aff0..8be0a27 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,6 +15,15 @@ on:
 # Also turn on "Require approval for all outside collaborators" in repo Actions
 # settings before making this repo public.
 jobs:
+  benchmark-tools:
+    name: benchmark tooling (unit tests)
+    if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && (github.base_ref == 'main' || github.base_ref == 'dev'))
+    runs-on: [self-hosted, macOS, ARM64, m4pro]
+    steps:
+      - uses: actions/checkout@v4
+      - run: bash -n benchmarks/scripts/run_bench.sh
+      - run: python3 -m unittest discover -s benchmarks/tests -v
+
   base-convert:
     name: base-convert (build + test)
     if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && (github.base_ref == 'main' || github.base_ref == 'dev'))
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 7c0d7c1..594581c 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -7,6 +7,7 @@ Reproduce baseRT throughput numbers on your own hardware.
 1. Download an engine release and unpack it into `build/` at the repo root
    (see the top-level README's "Getting the engine"). You need `build/basert-bench`.
 2. Convert a model to `.base` (see `base-convert/`) into `models/`.
+3. Python 3 (the output parser uses only the standard library).
 
 ## Run
 
@@ -18,17 +19,50 @@ benchmarks/scripts/run_bench.sh
 PP_VALS="128 512" REPS=3 benchmarks/scripts/run_bench.sh models/your-model.base
 ```
 
+The runner accepts one or more explicit `.base` paths. With no arguments it
+benchmarks every `models/*.base` file. These environment variables control a
+run:
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `PP_VALS` | `128 256 512` | Space-separated prefill lengths. |
+| `TG_VAL` | `128` | Decode length. |
+| `REPS` | `5` | Measured repetitions. |
+| `WARMUP` | `1` | Warm-up iterations. |
+| `BASERT` | `build/basert-bench` | Benchmark executable. |
+| `RESULTS` | `benchmarks/results/<arch>_baseRT.csv` | Output CSV path. |
+
+The output parser requires exactly one expected `ppN` and `tgN` row from every
+invocation. A benchmark or output-format failure therefore stops the run instead
+of publishing a partial CSV.
+
 Results are written to `benchmarks/results/<arch>_baseRT.csv` with columns
 `model,size_mb,engine,test,tok_per_sec,stddev`. `ppN` rows are prefill
-throughput at prompt length N; `tgN` rows are decode (token-generation)
-throughput.
+throughput at prompt length N; `tgN` rows are independent decode
+(token-generation) microbenchmarks. A `tgN` result is not decode-after-prefill
+for the adjacent `ppN` row, even though both rows come from the same invocation.
+These single-stream microbenchmarks are not a production server workload; use
+concurrent clients against `basert serve --continuous-batching` to measure
+server capacity and scheduling behavior.
 
 ## Example results
 
-`results/m4-pro_baseRT.csv` and `results/m3-base_baseRT.csv` hold reference
-numbers from earlier runs (dated by commit). They are not continuously
-republished — regenerate locally for an apples-to-apples comparison with your
-build and models.
+`results/m1-max-64gb_baseRT.csv` contains BaseRT 0.1.7 results for
+Qwen3.6-35B-A3B `default-q4` on an M1 Max with 64 GB unified memory; its adjacent
+[metadata file](results/m1-max-64gb_baseRT.md) records the host, model artifact,
+engine, and methodology. `results/m4-pro_baseRT.csv` and
+`results/m3-base_baseRT.csv` hold reference numbers from earlier runs (dated by
+commit). They are not continuously republished — regenerate locally for an
+apples-to-apples comparison with your build and models.
+
+## Tests
+
+The parser and runner tests use only the Python standard library and a fake
+benchmark executable; they do not require an engine release or model:
+
+```sh
+python3 -m unittest discover -s benchmarks/tests -v
+```
 
 ## Legacy / comparison scripts
 
diff --git a/benchmarks/results/m1-max-64gb_baseRT.csv b/benchmarks/results/m1-max-64gb_baseRT.csv
new file mode 100644
index 0000000..c06a641
--- /dev/null
+++ b/benchmarks/results/m1-max-64gb_baseRT.csv
@@ -0,0 +1,13 @@
+model,size_mb,engine,test,tok_per_sec,stddev
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,pp128,552.07,1.29
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,tg128,96.77,0.72
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,pp256,692.73,1.88
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,tg128,96.37,0.66
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,pp512,851.39,0.41
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,tg128,97.15,0.77
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,pp2048,1025.34,10.61
+Qwen/Qwen3.6-35B-A3B · default-q4,20889,basert,tg128,96.48,0.62
+Qwen/Qwen3.6-35B-A3B · default-q4,20928,basert,pp6000,878.60,3.18
+Qwen/Qwen3.6-35B-A3B · default-q4,20928,basert,tg128,95.99,0.86
+Qwen/Qwen3.6-35B-A3B · default-q4,21651,basert,pp43000,574.93,1.00
+Qwen/Qwen3.6-35B-A3B · default-q4,21651,basert,tg128,96.53,0.52
diff --git a/benchmarks/results/m1-max-64gb_baseRT.md b/benchmarks/results/m1-max-64gb_baseRT.md
new file mode 100644
index 0000000..3179755
--- /dev/null
+++ b/benchmarks/results/m1-max-64gb_baseRT.md
@@ -0,0 +1,51 @@
+# M1 Max 64 GB — Qwen3.6-35B-A3B
+
+Metadata for [`m1-max-64gb_baseRT.csv`](m1-max-64gb_baseRT.csv). The complete
+second-run console output is preserved in
+[`m1-max-64gb_baseRT.raw.md`](m1-max-64gb_baseRT.raw.md).
+
+## Environment
+
+- **Run date:** 2026-07-21
+- **Host:** Apple M1 Max, 64 GB unified memory
+- **OS:** macOS 26.4.1
+- **Engine:** BaseRT 0.1.7, official macOS arm64 release
+- **Engine binary SHA-256:** `f81d0ddd3c3eae1ff43f734aa0c76786b8b2e9b9fe3e1c5ec28f8d581a972721`
+- **Base repository revision:** `1e923026`
+- **Model:** `Qwen/Qwen3.6-35B-A3B`
+- **Source revision:** `995ad96eacd98c81ed38be0c5b274b04031597b0`
+- **Conversion profile:** `default-q4`
+- **Conversion command:** `basert pull Qwen/Qwen3.6-35B-A3B`
+- **Converter version:** `base-convert v0.1.6` (invoked by `basert pull v0.1.6`)
+- **`.base` file size:** 20,699,643,904 bytes
+- **`.base` SHA-256:** `74c4a9256e64971e919c41b78b03ff8e0ec74b4330058113f5785097706ed1d0`
+
+## Invocation
+
+```sh
+PP_VALS="128 256 512 2048 6000 43000" \
+TG_VAL=128 REPS=5 WARMUP=1 \
+benchmarks/scripts/run_bench.sh /path/to/qwen3.6-35b-a3b-default-q4.base
+```
+
+Each `tok_per_sec` value is the mean throughput over five measured repetitions
+after one warm-up iteration. The complete sweep was repeated independently
+through `run_bench.sh`; all throughput values reproduced within 1.2%, and the
+CSV below records that second end-to-end run. `stddev` is copied from the
+engine-reported standard deviation. `size_mb` is copied verbatim from the
+corresponding `basert-bench` output row's `size` column; it is not the `.base`
+file size. The reported value grows from 20,889 MiB
+to 21,651 MiB for the longest prompt sweep, so it is intentionally retained per
+row rather than replaced with one file-level value.
+
+## Interpretation
+
+`basert-bench` reports an independent prefill microbenchmark (`ppN`) and decode
+microbenchmark (`tgN`) in each invocation. A `tg128` row does **not** measure 128
+decode tokens after the prompt length named by the adjacent `ppN` row. The
+repeated `tg128` rows are retained as separate observations from each invocation.
+
+These are single-stream engine microbenchmarks. They do not measure concurrent
+requests, continuous batching, scheduling, HTTP overhead, time to first token,
+or end-to-end latency. Use a concurrent client workload against `basert serve
+--continuous-batching` for production server capacity claims.
diff --git a/benchmarks/results/m1-max-64gb_baseRT.raw.md b/benchmarks/results/m1-max-64gb_baseRT.raw.md
new file mode 100644
index 0000000..78362ee
--- /dev/null
+++ b/benchmarks/results/m1-max-64gb_baseRT.raw.md
@@ -0,0 +1,40 @@
+# Raw BaseRT benchmark output — M1 Max 64 GB
+
+This is the complete stdout from the second end-to-end sweep recorded in
+`m1-max-64gb_baseRT.csv`. The invocation and artifact provenance are documented
+in `m1-max-64gb_baseRT.md`.
+
+```text
+[1/1] Benchmarking: /Users/ariaki-m1max/Library/Caches/baseRT/models/Qwen/Qwen3.6-35B-A3B/default-q4/model.base
+  pp128/tg128, reps=5, warmup=1
+| model | size | test | t/s |
+| --- | ---: | ---: | ---: |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | pp128 | 552.07 ± 1.29 |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | tg128 | 96.77 ± 0.72 |
+  pp256/tg128, reps=5, warmup=1
+| model | size | test | t/s |
+| --- | ---: | ---: | ---: |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | pp256 | 692.73 ± 1.88 |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | tg128 | 96.37 ± 0.66 |
+  pp512/tg128, reps=5, warmup=1
+| model | size | test | t/s |
+| --- | ---: | ---: | ---: |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | pp512 | 851.39 ± 0.41 |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | tg128 | 97.15 ± 0.77 |
+  pp2048/tg128, reps=5, warmup=1
+| model | size | test | t/s |
+| --- | ---: | ---: | ---: |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | pp2048 | 1025.34 ± 10.61 |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | tg128 | 96.48 ± 0.62 |
+  pp6000/tg128, reps=5, warmup=1
+| model | size | test | t/s |
+| --- | ---: | ---: | ---: |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20928 MiB | pp6000 | 878.60 ± 3.18 |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20928 MiB | tg128 | 95.99 ± 0.86 |
+  pp43000/tg128, reps=5, warmup=1
+| model | size | test | t/s |
+| --- | ---: | ---: | ---: |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 21651 MiB | pp43000 | 574.93 ± 1.00 |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 21651 MiB | tg128 | 96.53 ± 0.52 |
+Wrote results to /tmp/upstream-run-m1max.csv
+```
diff --git a/benchmarks/scripts/parse_bench.py b/benchmarks/scripts/parse_bench.py
new file mode 100755
index 0000000..4301ff0
--- /dev/null
+++ b/benchmarks/scripts/parse_bench.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Parse basert-bench's table output into benchmark CSV rows."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import re
+import sys
+from typing import Iterable, TextIO
+
+FIELDNAMES = [
+    "model",
+    "size_mb",
+    "engine",
+    "test",
+    "tok_per_sec",
+    "stddev",
+]
+NUMBER = r"[0-9]+(?:\.[0-9]+)?"
+SIZE_RE = re.compile(rf"^(?P<size>{NUMBER})\s+MiB$")
+RATE_RE = re.compile(
+    rf"^(?P<tok_per_sec>{NUMBER})\s*±\s*(?P<stddev>{NUMBER})$"
+)
+TEST_RE = re.compile(r"^(?:pp|tg)[0-9]+$")
+
+
+def parse_output(output: str, expected_tests: Iterable[str]) -> list[dict[str, str]]:
+    """Return expected benchmark rows in the requested order.
+
+    Non-table lines and table headers are ignored. Every expected test must occur
+    exactly once so a changed or truncated basert-bench table cannot silently
+    produce an incomplete results file.
+    """
+    expected = list(expected_tests)
+    if len(set(expected)) != len(expected):
+        raise ValueError("expected test names must be unique")
+    parsed: dict[str, dict[str, str]] = {}
+    for line in output.splitlines():
+        cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
+        if len(cells) != 4 or not TEST_RE.fullmatch(cells[2]):
+            continue
+        size = SIZE_RE.fullmatch(cells[1])
+        rate = RATE_RE.fullmatch(cells[3])
+        if size is None or rate is None:
+            continue
+        if cells[2] in parsed:
+            raise ValueError(f"duplicate benchmark row: {cells[2]}")
+        parsed[cells[2]] = {
+            "model": cells[0],
+            "size_mb": size.group("size"),
+            "engine": "basert",
+            "test": cells[2],
+            "tok_per_sec": rate.group("tok_per_sec"),
+            "stddev": rate.group("stddev"),
+        }
+
+    for test in expected:
+        if test not in parsed:
+            raise ValueError(f"missing expected row: {test}")
+    return [parsed[test] for test in expected]
+
+
+def write_rows(rows: Iterable[dict[str, str]], output: TextIO) -> None:
+    """Write headerless rows for run_bench.sh to append to its result file."""
+    writer = csv.DictWriter(output, fieldnames=FIELDNAMES, lineterminator="\n")
+    writer.writerows(rows)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--expect",
+        action="append",
+        required=True,
+        metavar="TEST",
+        help="expected test row (repeat for each row, for example pp128 and tg128)",
+    )
+    args = parser.parse_args()
+    try:
+        rows = parse_output(sys.stdin.read(), args.expect)
+    except ValueError as error:
+        parser.error(str(error))
+    write_rows(rows, sys.stdout)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/benchmarks/scripts/run_bench.sh b/benchmarks/scripts/run_bench.sh
new file mode 100755
index 0000000..3fcb126
--- /dev/null
+++ b/benchmarks/scripts/run_bench.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# Run basert-bench for one or more .base models and write normalized CSV results.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+MODELS_DIR="${MODELS_DIR:-$REPO_ROOT/models}"
+BASERT="${BASERT:-$REPO_ROOT/build/basert-bench}"
+PYTHON="${PYTHON:-python3}"
+PP_VALS="${PP_VALS:-128 256 512}"
+TG_VAL="${TG_VAL:-128}"
+REPS="${REPS:-5}"
+WARMUP="${WARMUP:-1}"
+ARCH="$(uname -m | tr '[:upper:]' '[:lower:]')"
+RESULTS="${RESULTS:-$REPO_ROOT/benchmarks/results/${ARCH}_baseRT.csv}"
+PARSER="$SCRIPT_DIR/parse_bench.py"
+
+usage() {
+    echo "Usage: $0 [model.base ...]" >&2
+    echo "With no model arguments, benchmarks every models/*.base file." >&2
+}
+
+is_positive_integer() {
+    [[ "$1" =~ ^[1-9][0-9]*$ ]]
+}
+
+is_nonnegative_integer() {
+    [[ "$1" =~ ^[0-9]+$ ]]
+}
+
+if [[ ! -x "$BASERT" ]]; then
+    echo "error: basert-bench is missing or not executable: $BASERT" >&2
+    exit 1
+fi
+if [[ ! -r "$PARSER" ]]; then
+    echo "error: benchmark output parser is missing or unreadable: $PARSER" >&2
+    exit 1
+fi
+if ! command -v "$PYTHON" >/dev/null 2>&1; then
+    echo "error: Python interpreter not found: $PYTHON" >&2
+    exit 1
+fi
+if ! is_positive_integer "$TG_VAL"; then
+    echo "error: TG_VAL must be a positive integer: $TG_VAL" >&2
+    exit 2
+fi
+if ! is_positive_integer "$REPS"; then
+    echo "error: REPS must be a positive integer: $REPS" >&2
+    exit 2
+fi
+if ! is_nonnegative_integer "$WARMUP"; then
+    echo "error: WARMUP must be a non-negative integer: $WARMUP" >&2
+    exit 2
+fi
+
+IFS=' ' read -r -a prompt_values <<< "$PP_VALS"
+if [[ ${#prompt_values[@]} -eq 0 ]]; then
+    echo "error: PP_VALS must contain at least one prompt length" >&2
+    exit 2
+fi
+for prompt in "${prompt_values[@]}"; do
+    if ! is_positive_integer "$prompt"; then
+        echo "error: PP_VALS must contain positive integers: $PP_VALS" >&2
+        exit 2
+    fi
+done
+
+if [[ $# -gt 0 ]]; then
+    models=("$@")
+else
+    shopt -s nullglob
+    models=("$MODELS_DIR"/*.base)
+    shopt -u nullglob
+fi
+if [[ ${#models[@]} -eq 0 ]]; then
+    usage
+    echo "error: no .base models found in $MODELS_DIR" >&2
+    exit 1
+fi
+for model in "${models[@]}"; do
+    if [[ ! -f "$model" ]]; then
+        echo "error: model not found: $model" >&2
+        exit 1
+    fi
+done
+
+mkdir -p "$(dirname "$RESULTS")"
+temporary_results="$(mktemp "${RESULTS}.tmp.XXXXXX")"
+trap 'rm -f "$temporary_results"' EXIT
+printf 'model,size_mb,engine,test,tok_per_sec,stddev\n' > "$temporary_results"
+
+total=${#models[@]}
+index=0
+for model in "${models[@]}"; do
+    index=$((index + 1))
+    echo "[$index/$total] Benchmarking: $model"
+    for prompt in "${prompt_values[@]}"; do
+        echo "  pp${prompt}/tg${TG_VAL}, reps=${REPS}, warmup=${WARMUP}"
+        if output=$("$BASERT" "$model" \
+            -p "$prompt" -n "$TG_VAL" -r "$REPS" -w "$WARMUP" 2>&1); then
+            printf '%s\n' "$output"
+        else
+            status=$?
+            printf '%s\n' "$output" >&2
+            echo "error: basert-bench failed for $model (pp${prompt}/tg${TG_VAL})" >&2
+            exit "$status"
+        fi
+        printf '%s\n' "$output" | "$PYTHON" "$PARSER" \
+            --expect "pp${prompt}" --expect "tg${TG_VAL}" \
+            >> "$temporary_results"
+    done
+done
+
+mv "$temporary_results" "$RESULTS"
+trap - EXIT
+echo "Wrote results to $RESULTS"
diff --git a/benchmarks/tests/test_parse_bench.py b/benchmarks/tests/test_parse_bench.py
new file mode 100644
index 0000000..d7f412e
--- /dev/null
+++ b/benchmarks/tests/test_parse_bench.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Tests for parsing basert-bench's human-readable result table."""
+
+import sys
+import unittest
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts"
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from parse_bench import parse_output  # noqa: E402
+
+
+SAMPLE_OUTPUT = """\
+BaseRT benchmark
+| model | size | test | tok/s |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | pp128 | 553.17 ± 1.74 |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | tg128 | 97.04 ± 0.46 |
+"""
+
+
+class ParseOutputTests(unittest.TestCase):
+    def test_parses_model_label_size_and_throughput(self):
+        rows = parse_output(SAMPLE_OUTPUT, ["pp128", "tg128"])
+
+        self.assertEqual(
+            rows,
+            [
+                {
+                    "model": "Qwen/Qwen3.6-35B-A3B · default-q4",
+                    "size_mb": "20889",
+                    "engine": "basert",
+                    "test": "pp128",
+                    "tok_per_sec": "553.17",
+                    "stddev": "1.74",
+                },
+                {
+                    "model": "Qwen/Qwen3.6-35B-A3B · default-q4",
+                    "size_mb": "20889",
+                    "engine": "basert",
+                    "test": "tg128",
+                    "tok_per_sec": "97.04",
+                    "stddev": "0.46",
+                },
+            ],
+        )
+
+    def test_rejects_output_missing_an_expected_row(self):
+        output = """\
+| model | size | test | tok/s |
+| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | pp128 | 553.17 ± 1.74 |
+"""
+
+        with self.assertRaisesRegex(ValueError, "missing expected row: tg128"):
+            parse_output(output, ["pp128", "tg128"])
+
+    def test_rejects_duplicate_expected_rows(self):
+        output = SAMPLE_OUTPUT + (
+            "| Qwen/Qwen3.6-35B-A3B · default-q4 | 20889 MiB | "
+            "tg128 | 98.00 ± 0.50 |\n"
+        )
+
+        with self.assertRaisesRegex(ValueError, "duplicate benchmark row: tg128"):
+            parse_output(output, ["pp128", "tg128"])
+
+    def test_rejects_duplicate_expectations(self):
+        with self.assertRaisesRegex(ValueError, "expected test names must be unique"):
+            parse_output(SAMPLE_OUTPUT, ["pp128", "pp128"])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/benchmarks/tests/test_run_bench.py b/benchmarks/tests/test_run_bench.py
new file mode 100644
index 0000000..108e91c
--- /dev/null
+++ b/benchmarks/tests/test_run_bench.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Integration tests for the generic benchmark entrypoint."""
+
+import csv
+import os
+import stat
+import subprocess
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RUN_BENCH = REPO_ROOT / "benchmarks" / "scripts" / "run_bench.sh"
+
+
+class RunBenchTests(unittest.TestCase):
+    def test_specific_model_honors_sweep_environment_and_writes_csv(self):
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            temporary = Path(temporary_directory)
+            model = temporary / "model with spaces.base"
+            model.touch()
+            bench = temporary / "fake-basert-bench"
+            log = temporary / "calls.txt"
+            results = temporary / "results" / "bench.csv"
+            bench.write_text(
+                textwrap.dedent(
+                    """\
+                    #!/usr/bin/env python3
+                    import os
+                    import sys
+
+                    args = sys.argv[1:]
+                    with open(os.environ["FAKE_BENCH_LOG"], "a") as handle:
+                        handle.write(repr(args) + "\\n")
+                    prompt = args[args.index("-p") + 1]
+                    decode = args[args.index("-n") + 1]
+                    print("| model | size | test | tok/s |")
+                    print(f"| Model label with spaces · q4 | 123 MiB | pp{prompt} | 10.50 ± 0.25 |")
+                    print(f"| Model label with spaces · q4 | 123 MiB | tg{decode} | 20.75 ± 0.50 |")
+                    """
+                )
+            )
+            bench.chmod(bench.stat().st_mode | stat.S_IXUSR)
+            environment = os.environ.copy()
+            environment.update(
+                {
+                    "BASERT": str(bench),
+                    "RESULTS": str(results),
+                    "PP_VALS": "128 512",
+                    "TG_VAL": "64",
+                    "REPS": "3",
+                    "WARMUP": "2",
+                    "FAKE_BENCH_LOG": str(log),
+                }
+            )
+
+            completed = subprocess.run(
+                ["bash", str(RUN_BENCH), str(model)],
+                cwd=REPO_ROOT,
+                env=environment,
+                text=True,
+                capture_output=True,
+            )
+
+            self.assertEqual(completed.returncode, 0, completed.stderr)
+            with results.open(newline="") as handle:
+                rows = list(csv.DictReader(handle))
+            self.assertEqual(
+                [(row["test"], row["model"]) for row in rows],
+                [
+                    ("pp128", "Model label with spaces · q4"),
+                    ("tg64", "Model label with spaces · q4"),
+                    ("pp512", "Model label with spaces · q4"),
+                    ("tg64", "Model label with spaces · q4"),
+                ],
+            )
+            calls = log.read_text().splitlines()
+            self.assertEqual(len(calls), 2)
+            for prompt, call in zip(("128", "512"), calls):
+                self.assertIn(repr(str(model)), call)
+                self.assertIn(f"'-p', '{prompt}'", call)
+                self.assertIn("'-n', '64'", call)
+                self.assertIn("'-r', '3'", call)
+                self.assertIn("'-w', '2'", call)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/docs/guides/benchmarking.md b/docs/guides/benchmarking.md
index 542d844..0fe3695 100644
--- a/docs/guides/benchmarking.md
+++ b/docs/guides/benchmarking.md
@@ -35,7 +35,9 @@ engine bundle unpacked into `build/` (so `build/basert-bench` exists) — see
 - Use controlled token counts (`-p`/`-n`) and several repetitions (`-r`) with a
   warm-up (`-w`) so the first-call allocation cost doesn't skew results.
 - Prefill and decode are reported separately; they stress different parts of the
-  engine (compute-bound matmuls vs. memory-bound single-token steps).
-- For server-style throughput, benchmark under `basert serve
-  --continuous-batching` with concurrent clients rather than the single-stream
-  `bench` tool.
+  engine (compute-bound matmuls vs. memory-bound single-token steps). The decode
+  row is an independent microbenchmark, not generation after the prompt length
+  in the prefill row printed beside it.
+- `basert bench` is a single-stream engine microbenchmark. For production
+  server-style throughput, benchmark under `basert serve --continuous-batching`
+  with concurrent clients so request scheduling and batching are included.