chore(bench): fixtures + analysis/transport benchmarks + baseline

.github/workflows/ci.yml

@@ -49,6 +49,17 @@ jobs:
         run: opam exec -- dune build
       - name: Test
         run: opam exec -- dune runtest
+      # Perf gate: regenerate fixtures and assert the LSP E2E path stays
+      # within the ADR-010 hard-fail budgets. Linux only — shared runners
+      # are too noisy for the tight p95 targets, but a hard-fail breach is
+      # a real regression. See bench/BASELINE.md.
+      - name: Benchmark budget gate
+        if: runner.os == 'Linux'
+        run: |
+          python3 bench/gen_fixtures.py
+          opam exec -- dune build bench/bench_analysis.exe
+          python3 bench/lsp_roundtrip.py \
+            _build/default/bin/httui-lsp/httui_lsp.exe --check
       - name: Format check
         if: runner.os == 'Linux'
         run: |

Makefile

@@ -52,7 +52,9 @@ test-grammars: ## Run grammar corpus tests
 	cd $(TS_REFS_DIR) && npx tree-sitter test
 	cd $(TS_HTTP_DIR) && npx tree-sitter test
 
-bench: build-ocaml ## Run the LSP transport benchmark
+bench: build-ocaml ## Run perf benchmarks (analysis in-process + LSP transport)
+	python3 bench/gen_fixtures.py
+	dune exec bench/bench_analysis.exe -- bench/fixtures
 	python3 bench/lsp_roundtrip.py
 
 lint: lint-ocaml lint-js ## Run all linters

bench/BASELINE.md

@@ -0,0 +1,76 @@
+# Performance baseline — Slice 2.5
+
+Measured on the canonical ADR-010 fixtures (`bench/fixtures/`, generated
+by `gen_fixtures.py`): **medium** (466 lines, 10 blocks, ~30 refs) and
+**large** (5226 lines, 50 blocks, ~190 refs). Reproduce with `make bench`.
+
+Machine: darwin/arm64 (dev). Numbers are indicative, not absolute — the
+gates that matter are the ADR-010 budgets and regression between runs.
+
+## Analysis path — in-process (`bench_analysis.ml`)
+
+Pure OCaml, no transport. p95 on the **large** fixture:
+
+| Operation | p50 | p95 |
+|---|---|---|
+| `Fence_scanner.scan` | 0.21ms | 0.23ms |
+| `Analyze.diagnostics` | 0.62ms | 0.70ms |
+| `Semantic_tokens.of_blocks` | 0.79ms | 0.83ms |
+| `Analyze.completion_at` | ~0ms | ~0ms |
+| `Analyze.hover_at` | ~0ms | ~0ms |
+| didChange (scan + diagnostics) | 0.89ms | 0.92ms |
+
+The whole analysis is **sub-millisecond on 50 blocks** — two orders of
+magnitude under the ADR-010 diagnostics budget (<100ms p95). The
+suspected hot spots (the O(n²) `aliases_above`, re-parsing every block's
+tree-sitter tree per edit) cost <1ms combined. **The analysis is not a
+bottleneck.**
+
+## Transport path — E2E over stdio (`lsp_roundtrip.py`)
+
+| Operation | p50 | p95 | ADR-010 p95 budget |
+|---|---|---|---|
+| request round-trip floor | 0.011ms | 0.015ms | — |
+| didChange→diagnostics [medium 12KB] | 0.21ms | 0.35ms | <100ms |
+| didChange→diagnostics [large 147KB] | 1.83ms | 1.89ms | <100ms |
+| semanticTokens/full [medium] | 1.27ms | 1.34ms | <80ms |
+| **semanticTokens/full [large]** | **67.8ms** | **69.7ms** | **<80ms** |
+
+`didChange→diagnostics` on the large doc is **1.9ms** end to end —
+excellent. The one number near a budget is **`semanticTokens/full` on
+the large doc: ~70ms p95**. Note `Semantic_tokens.of_blocks` itself is
+0.83ms — so the other ~67ms is **LSP delta-position encoding + JSON
+serialization of the large token array, plus stdio transfer**, not the
+token computation. This is exactly what `semanticTokens/full/delta`
+targets (send the diff, not the whole array).
+
+## Frontend path — per-keystroke (`httui-desktop`, vitest bench, jsdom)
+
+One keystroke = insert + delete (two `view.dispatch` calls):
+
+| Fixture | mean | p99 |
+|---|---|---|
+| medium (10 blocks) | 0.59ms | 1.04ms |
+| large (50 blocks) | 6.18ms | 6.89ms |
+
+**10.5x medium→large.** That linear-with-doc-size scaling is the
+signature of the full-document scanners (`createFencedBlockExtension`
+`findBlocks`, `cm-tables`, `cm-merge-conflict`) that re-walk every line
+on every `docChanged`. The viewport-scoped `referenceHighlight` would be
+roughly constant. jsdom caveat: `visibleRanges` covers the whole doc, so
+this is a worst-case upper bound for the viewport plugins and excludes
+browser layout/paint.
+
+## Conclusions (what the data says to optimize)
+
+1. **LSP analysis (planned Fase 2: memo + subtree cache) — not justified
+   by the bench.** Everything is sub-ms; optimizing it would tune work
+   already 100x under budget (the ADR-010 anti-pattern: don't optimize
+   what the bench doesn't flag).
+2. **Semantic tokens encoding (Fase 3 delta) — justified**, but as the
+   one transport hot spot (~70ms on large) and a future token-consuming
+   client concern. The desktop does not consume server semantic tokens
+   today (Lezer first-paint), so it pays 0ms of this now.
+3. **Frontend full-doc scanners (Fase 4) — the real desktop win.** They
+   scale linearly with doc size and dominate the keystroke cost. Early-out
+   by sentinel + incrementalizing the fence scanner cut this directly.

bench/README.md

@@ -1,30 +1,33 @@
 # Benchmark fixtures and harness
 
-Synthetic vault fixtures used to validate performance budgets:
+Synthetic vault fixtures used to validate the ADR-010 performance
+budgets, generated deterministically by `gen_fixtures.py`:
 
-- **Medium**: ~500 lines of markdown, 10 executable blocks (5 HTTP + 5 SQL),
-  ~20 references.
-- **Large**: ~5000 lines, 50 blocks, ~200 references.
+- **medium.md**: ~500 lines, 10 executable blocks (5 HTTP + 5 SQL), ~30 refs.
+- **large.md**: ~5000 lines, 50 blocks (mixed), ~190 refs.
 
-Operations measured per fixture include incremental retokenize, semantic
-tokens delta, completion popup latency, hover latency, diagnostic publish,
-rename, find-all-references, format on save, and cancellation latency.
-
-Cold-start metrics tracked separately: LSP spawn-to-ready, time-to-first-
-completion after `didOpen`, first parse + semantic tokens, and schema cache
-lookup. Memory budget (RSS) tracked on both medium and large fixtures.
+Refs only point at blocks declared above (DAG by construction), so they
+exercise real scope resolution. Regenerate with `python3 bench/gen_fixtures.py`.
 
 ## Harness
 
-- `lsp_roundtrip.py` — transport baseline against the built `httui-lsp`
-  binary: spawn-to-initialize, request round-trip (framing + JSON +
-  dispatch), and didChange ingestion. No external dependencies.
+Two complementary benchmarks, both run by `make bench`:
+
+- `bench_analysis.ml` — **in-process** analysis cost (no transport): scan,
+  diagnostics, semantic tokens, completion, hover over each fixture. This
+  isolates the pure-OCaml algorithm cost.
+- `lsp_roundtrip.py` — **E2E over stdio**: spawn-to-initialize, request
+  round-trip floor, and per-fixture `didChange→diagnostics` +
+  `semanticTokens/full`. The gap between this and the in-process numbers
+  is the LSP encoding + transport cost. No external dependencies.
 
 ```bash
-dune build
-make bench          # or: python3 bench/lsp_roundtrip.py
+make bench
+# or individually:
+python3 bench/gen_fixtures.py
+dune exec bench/bench_analysis.exe -- bench/fixtures
+python3 bench/lsp_roundtrip.py
 ```
 
-Feature-level operations (hover, completion, semantic tokens,
-diagnostics) gain sections here as the server implements them; the
-fixtures above become their inputs.
+Current numbers and the optimization conclusions they drive live in
+[`BASELINE.md`](BASELINE.md).

bench/bench_analysis.ml

@@ -0,0 +1,95 @@
+(* In-process analysis benchmark: measures the pure-OCaml analysis path
+   (no stdio, no JSON) over the canonical fixtures, so the numbers are
+   the algorithm cost alone — the transport floor lives in
+   lsp_roundtrip.py.
+
+   Run via [make bench] or:
+     dune exec bench/bench_analysis.exe -- [fixtures-dir]
+   (default fixtures-dir: bench/fixtures)
+
+   Reports p50/p95/p99 per operation per fixture. ADR-010 budgets:
+   retoken viewport <16ms p95, diagnostics publish <100ms p95. *)
+
+let read_file path =
+  let ic = open_in_bin path in
+  let n = in_channel_length ic in
+  let s = really_input_string ic n in
+  close_in ic;
+  s
+
+let now () = Unix.gettimeofday ()
+
+let percentile sorted p =
+  let n = Array.length sorted in
+  sorted.(int_of_float (Float.round (float_of_int (n - 1) *. p)))
+
+let report name samples =
+  Array.sort compare samples;
+  let ms x = x *. 1e3 in
+  Printf.printf "%-40s n=%5d  p50=%.4fms  p95=%.4fms  p99=%.4fms  max=%.4fms\n"
+    name (Array.length samples)
+    (ms (percentile samples 0.50))
+    (ms (percentile samples 0.95))
+    (ms (percentile samples 0.99))
+    (ms samples.(Array.length samples - 1))
+
+(* Time [f ()] [iters] times after [warmup] discarded runs. [f] returns a
+   value we keep via [Sys.opaque_identity] so the optimizer cannot hoist
+   the work out of the loop. *)
+let bench ?(warmup = 50) ?(iters = 1000) f =
+  for _ = 1 to warmup do
+    ignore (Sys.opaque_identity (f ()))
+  done;
+  let samples = Array.make iters 0.0 in
+  for i = 0 to iters - 1 do
+    let t = now () in
+    ignore (Sys.opaque_identity (f ()));
+    samples.(i) <- now () -. t
+  done;
+  samples
+
+(* Offset just past the first [{{] in the doc — a realistic completion
+   trigger point. Falls back to 0 if the doc has no refs. *)
+let first_ref_open doc =
+  match Str.search_forward (Str.regexp_string "{{") doc 0 with
+  | i -> i + 2
+  | exception Not_found -> 0
+
+(* Offset inside the first ref name — a realistic hover point. *)
+let first_ref_name doc = first_ref_open doc
+
+let run_fixture path =
+  let doc = read_file path in
+  Printf.printf "\n# %s (%d bytes, %d lines)\n" (Filename.basename path)
+    (String.length doc)
+    (String.fold_left (fun n c -> if c = '\n' then n + 1 else n) 0 doc);
+  let blocks = Httui_lang.Fence_scanner.scan doc in
+  let comp_off = first_ref_open doc in
+  let hover_off = first_ref_name doc in
+  report "Fence_scanner.scan"
+    (bench (fun () -> Httui_lang.Fence_scanner.scan doc));
+  report "Analyze.diagnostics"
+    (bench (fun () -> Httui_lang.Analyze.diagnostics blocks));
+  report "Semantic_tokens.of_blocks"
+    (bench (fun () -> Httui_lang.Semantic_tokens.of_blocks blocks));
+  report "Analyze.completion_at"
+    (bench (fun () ->
+         Httui_lang.Analyze.completion_at doc blocks ~offset:comp_off));
+  report "Analyze.hover_at"
+    (bench (fun () -> Httui_lang.Analyze.hover_at blocks ~offset:hover_off));
+  (* didChange cost = scan + diagnostics (what publish_diagnostics does). *)
+  report "didChange (scan + diagnostics)"
+    (bench (fun () ->
+         let b = Httui_lang.Fence_scanner.scan doc in
+         Httui_lang.Analyze.diagnostics b))
+
+let () =
+  let dir =
+    if Array.length Sys.argv > 1 then Sys.argv.(1) else "bench/fixtures"
+  in
+  List.iter
+    (fun f ->
+      let path = Filename.concat dir f in
+      if Sys.file_exists path then run_fixture path
+      else Printf.eprintf "missing fixture: %s\n" path)
+    [ "medium.md"; "large.md" ]

bench/dune

@@ -0,0 +1,3 @@
+(executable
+ (name bench_analysis)
+ (libraries httui_lang unix str))