From 370fbd8a21adf2c67665a69dea580b50e4e85a12 Mon Sep 17 00:00:00 2001 From: Julien Cornebise Date: Fri, 24 Jul 2026 03:01:37 +0200 Subject: [PATCH] =?UTF-8?q?python-math=20#30:=20feat(delphi):=20poller-equ?= =?UTF-8?q?ivalence=20harness=20(goal=20condition=202)=20=E2=80=94=20PASSE?= =?UTF-8?q?S=20live=20on=20vw=20+=20pc-meta-02?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What The condition-(2) harness from `GOAL_R1_PARITY.md`, specified in `MATH_POLLER_EQUIV_SPEC.md`: one throwaway Postgres, the REAL Clojure math container (`clojure -M:run full`, writing under `MATH_ENV=clj-ref`) and the Python `MathPollerService` (`MATH_ENV=py-shadow`) fed identical timed vote batches — `math_env` being the column that keys each math result row, so the two engines write side by side in the same database. Written rows are compared per batch with the battery acceptance criteria (`StepComparer` projection, plus the quirk-Q7 exclusion: subgroup stats that Clojure computes and persists but nothing downstream ever consumes), extended with a measured Clojure self-jitter envelope for floats — two independent Clojure runs establish the envelope, the float threshold is max(2x envelope, 1e-9), and structural divergences are never excused. The protocol includes a kill+restart seam mid-schedule, and checks `caching_tick` MAX+1 per env, `math_ticks` monotonic, and exactly-once watermark coverage. Guards: NO-COVERAGE (0 aligned batches / `ready:false` / empty store => FAIL — a vacuous pass is structurally impossible), a fail-fast feeder with runner-log evidence capture, and a startup gate closing the quirk-Q19 race window (see below). ## Live results 2026-07-24; stores + verdict JSONs under `real_data/.local/replays/poller_equiv/`: - `vw`: verdict MATCH, 8/8 batches, restart seam after batch 4, ticks OK, 0 envelope-excused divergences. - `pc-meta-02`: verdict MATCH, 6/6 batches, seam after batch 3, moderation stream live (146/146 moderation events woven, matching the Clojure side). ## Real bugs found and fixed by the harness arc - The Python poller stringified pid/tid/zid at ingestion (`postgres.py` `poll_votes`/`poll_votes_since`/`poll_moderation`, `service.py` `str(zid)`) — native DB ints now flow end-to-end, matching Clojure. The `poll_moderation` fix also prevents a would-be regression: int vote-tids vs str mod-tids would have silently disabled comment moderation. - `derive_ptptstats` wrote `conv.participant_info` — a DIFFERENT STATISTIC from Clojure's `prep-ptpt-stats`. It is now the verbatim geometric port (`repness.clj:383-413`: centricness/coreness/extremeness in PCA space, columnized per `conv_man.clj:79-88`, group order via `clojure_hash_map_key_order`). - Quirk Q19 (`CLOJURE_QUIRKS.md`): a GENUINE Clojure production bug — `conv_man.clj` `queue-message-batch!` races on discovery of a brand-new zid (conversation ID): two conv-actors get spawned, one is orphaned, and its accumulated votes are silently lost (reproduced 3x). The harness carves it out via test-input sequencing only: wait for the container's first poll cycle before feeding (the acceptance criteria are untouched). Flagged for an upstream fix. - Five harness-environment findings (the Clojure `postgres://` URL regex, `POLL_FROM_DAYS_AGO` long parse, SSL mode, shared-millisecond cut ties vs strict-`>` watermarks, per-row insert visibility) — fixed in the harness, with the Clojure behaviors documented. ## Testing Harness + poller: 497 passed. Full delphi suite: 1134 passed / 22 skipped / 46 xfailed, zero failures. The live protocol is opt-in (it needs the real services); the unit surface covers every pure decision point. Co-Authored-By: Claude Fable 5 commit-id:1530cd63 --- delphi/docs/MATH_POLLER_EQUIV_SPEC.md | 100 + delphi/polismath/conversation/conversation.py | 2 +- delphi/polismath/database/postgres.py | 56 +- delphi/polismath/poller/__init__.py | 102 +- delphi/polismath/poller/math_writer.py | 160 +- delphi/polismath/poller/service.py | 24 +- delphi/polismath/replay/poller_equiv.py | 2774 +++++++++++++++++ delphi/polismath/utils/clj_hash.py | 17 +- delphi/scripts/poller_equiv.py | 282 ++ delphi/tests/poller/test_load_or_init.py | 36 + delphi/tests/poller/test_math_writer.py | 237 +- .../poller/test_postgres_client_pid_types.py | 183 ++ .../test_poller_equiv_compare.py | 1155 +++++++ .../test_poller_equiv_envelope.py | 724 +++++ .../replay_harness/test_poller_equiv_seed.py | 1018 ++++++ 15 files changed, 6805 insertions(+), 65 deletions(-) create mode 100644 delphi/docs/MATH_POLLER_EQUIV_SPEC.md create mode 100644 delphi/polismath/replay/poller_equiv.py create mode 100644 delphi/scripts/poller_equiv.py create mode 100644 delphi/tests/poller/test_postgres_client_pid_types.py create mode 100644 delphi/tests/replay_harness/test_poller_equiv_compare.py create mode 100644 delphi/tests/replay_harness/test_poller_equiv_envelope.py create mode 100644 delphi/tests/replay_harness/test_poller_equiv_seed.py diff --git a/delphi/docs/MATH_POLLER_EQUIV_SPEC.md b/delphi/docs/MATH_POLLER_EQUIV_SPEC.md new file mode 100644 index 0000000000..b67bea13f3 --- /dev/null +++ b/delphi/docs/MATH_POLLER_EQUIV_SPEC.md @@ -0,0 +1,100 @@ +# Poller-equivalence harness — spec (goal condition 2) + +**Status:** spec — 2026-07-24 (session 5). Companion to `MATH_POLLER_DESIGN.md` +(the py poller under test) and `GOAL_R1_PARITY.md` ("DONE means": *poller +equivalence: identical math_main/bidToPid/ptptstats rows and tick/watermark +semantics vs the Clojure math container on the same vote streams, including a +restart-mid-schedule seam*). + +## 1. Shape + +One Postgres, two writers, one comparer: + +``` +scripts/poller_equiv.py (CLI orchestrator) + ├─ seed: create throwaway DB (polis_equiv) with the polis schema subset + │ (conversations, votes, comments, math_main, math_bidtopid, + │ math_ptptstats, math_ticks); insert conversation + comments; + │ votes are inserted in TIMED BATCHES by the driver loop below + ├─ clj: the REAL container loop — clojure -M:run full (math/), env + │ DATABASE_URL=…/polis_equiv, MATH_ENV=clj-ref, + │ POLL_FROM_DAYS_AGO=10000 (historical vote timestamps) + ├─ py: scripts/math_poller.py, same DB, MATH_ENV=py-shadow, + │ POLISMATH_ENGINE_MODE=clojure-legacy, same poll window + ├─ feed: insert vote batch k → wait until BOTH math_envs' math_main + │ rows advance past batch k's votes (poll by caching_tick / + │ lastVoteTimestamp in the blob) → next batch. Batches mirror a + │ battery schedule's cuts (vw uniform8; pc-meta-02 uniform6-mod + │ for the moderation stream — comments.modified drives mod polls) + ├─ seam: after batch R (mid-schedule), SIGKILL the py poller process, + │ restart it (load-or-init warm path — the from_dict restore + │ fixed 2026-07-24), continue feeding. Also restart the clj + │ container at the same seam for symmetry (its load-or-init). + └─ compare per batch k and per table: + math_main.data → the SAME acceptance as certify (StepComparer: + structural identity on memberships/ids/ + selections/priorities; declared float + tolerances; subgroup-* excluded per Q7) + math_bidtopid.data → EXACT equality (bid→pids map) + math_ptptstats.data→ structural + tolerances + caching_tick → per-env MAX+1 monotonicity (not cross-env equal — + each env has its own sequence) + math_ticks → equals the number of completed recomputes per env + watermark semantics→ each batch processed exactly once (no vote + reprocessing: assert vote_counts in the blob + match cumulative inserts at each cut) +``` + +## 2. Acceptance — the float bar + +The PRODUCTION clj container cannot be Q10/Q12-pinned (no Clojure source +edits allowed): its cold-tick PCA start is unseeded-random, so even two clj +container runs differ in float tails. The bar is therefore: + +1. **clj self-jitter envelope first**: run the clj side TWICE on the same + stream (fresh DB each). Per compared key, record the max cross-run + delta — the envelope. (H-A/H-B self-jitter pattern, journal 2026-07-18.) +2. **py must sit within the envelope** (per key: |py − clj| ≤ envelope × + safety factor 2, floor 1e-9) AND be STRUCTURALLY identical (memberships, + cluster ids, repness/consensus selections, priority ordering) to the clj + reference run. +3. Report the envelope + verdict per (dataset, batch, table) in a compact + JSON verdict file (terse-output contract: ≤40 lines to stdout). + +Datasets: vw (knife-edge-free warm chain, certified 8/8) + pc-meta-02 +(mod/meta warm chain, certified 6/6). Both small → container runtime fine. + +## 3. Build plan (delegable, in order) + +- **A. schema + seeder** — smallest schema subset the clj container's + queries touch (poller.clj/postgres.clj: votes, comments, conversations, + math_main, math_bidtopid, math_ptptstats, math_ticks; check + db/load-conv's SELECT for exact columns). Seeder loads a replay dataset + (real_data loaders) and inserts conversation+comments; vote inserts + exposed as `insert_votes(conn, dataset, from_slot, to_slot)`. +- **B. runners** — subprocess wrappers: clj container (env as §1; verify + `clojure -M:run full` works headless from math/ — bin/run wraps it), + py poller CLI. Health = row appears in math_main for the env. +- **C. feeder + comparer** — batch loop, per-batch row snapshots, the + StepComparer adapter (math_main.data JSON ≈ the certify blob surface — + verify key overlap first; bidToPid exact; tick/watermark assertions). +- **D. seam + envelope** — restart choreography, two clj runs, envelope + computation, verdict JSON. + +Integration gates at the top level after each stage; the harness lives in +`polismath/replay/` + `scripts/` next to certify (same store/report +conventions). Tests: unit-test the comparer adapter + watermark assertions +with canned rows (no containers); the full harness is an opt-in script +(RUN_POLLER_EQUIV=1), like the RUN_CLJ_INTEGRATION certify tests. + +## 4. Known hazards (from session-5 recon) + +- Postgres reachable at localhost:15432 via OrbStack pgproxy (socat → + polis-dev-postgres-1:5432); create polis_equiv there, NEVER touch + polis-dev / polis_prodclone. +- Vote timestamps are historical → POLL_FROM_DAYS_AGO=10000 on BOTH sides. +- The clj container writes ALL zids it sees in the window — the throwaway + DB isolates this. +- 4h JVM self-reboot (bin/run timeout) — irrelevant at harness timescales. +- polismath/poller/__init__.py "load-or-init finding" docstring is stale + (base_clusters DO restore since 2026-07-24) — refresh it in stage B. diff --git a/delphi/polismath/conversation/conversation.py b/delphi/polismath/conversation/conversation.py index de58092d49..865441a18d 100644 --- a/delphi/polismath/conversation/conversation.py +++ b/delphi/polismath/conversation/conversation.py @@ -175,7 +175,7 @@ class Conversation: """ def __init__(self, - conversation_id: str, + conversation_id: Union[str, int], last_updated: Optional[int] = None, votes: Optional[Dict[str, Any]] = None): """ diff --git a/delphi/polismath/database/postgres.py b/delphi/polismath/database/postgres.py index 076b2cb8a0..f37be7c868 100644 --- a/delphi/polismath/database/postgres.py +++ b/delphi/polismath/database/postgres.py @@ -520,11 +520,30 @@ def poll_votes( # Execute query votes = self.query(sql, params) - # Format votes for processing, flipping sign at PostgreSQL boundary + # Format votes for processing, flipping sign at PostgreSQL boundary. + # pid AND tid are kept as the DB's native int (votes.pid/tid are both + # INTEGER) — NOT str()-wrapped. Found live (2026-07-24, poller- + # equivalence harness, session 2): the pid cast was the ONLY source + # of a Type-mismatch divergence in math_main.base-clusters.members + # against Clojure (which holds an int pid throughout) — + # Conversation.update_votes is deliberately type-agnostic at ingress + # ("Preserve original type", both pid AND tid) and + # raw_rating_mat/rating_mat are ALWAYS rebuilt fresh from these two + # methods on every load-or-init (never restored via from_dict — see + # polismath/poller/__init__.py's "load-or-init finding" docstring), + # so removing the cast is a one-point fix with no other code changes + # needed. Session 3 (same day): fixing pid alone left tid's OWN + # str() cast unmasked — a live vw full-run then showed the SAME + # Type-mismatch pattern on zid/tids[]/repness.*.tid, traced to this + # same cast. The certified/CSV replay driver never cast tid either, + # and matched clj int-for-int across 20 cross-validated entries — + # the evidence that authorized this fix. See also poll_moderation + # below, which needed the SAME fix for mod_out_tids/mod_in_tids/ + # meta_tids/mod_out_ptpts to stay type-consistent with these two. return [ { - "pid": str(v["pid"]), - "tid": str(v["tid"]), + "pid": v["pid"], + "tid": v["tid"], "vote": postgres_vote_to_delphi(int(v["vote"])), "created": v["created"], } @@ -555,11 +574,14 @@ def poll_votes_since(self, since: int) -> List[Dict[str, Any]]: """, {"since": since}, ) + # pid AND tid kept as the DB's native int — see poll_votes's + # docstring/comment above for the full root-cause rationale + # (2026-07-24 live findings, sessions 2-3). return [ { "zid": int(v["zid"]), - "pid": str(v["pid"]), - "tid": str(v["tid"]), + "pid": v["pid"], + "tid": v["tid"], "vote": postgres_vote_to_delphi(int(v["vote"])), "created": v["created"], } @@ -679,13 +701,22 @@ def poll_moderation( # Execute query mods = self.query(sql_mods, params) - # Format moderation data + # Format moderation data. tid is kept as the DB's native int — NOT + # str()-wrapped (2026-07-24 live finding, session 3): mod_out_tids + # feeds Conversation._apply_moderation's + # ``[c for c in self.mod_out_tids if c in self.rating_mat.columns]`` + # intersection UNCONDITIONALLY (no engine-mode branch, unlike the + # participant-ban check below) — left str while poll_votes/ + # poll_votes_since's tid became int, that intersection would ALWAYS + # be empty, silently disabling moderated-out comment zeroing in the + # live poller. poll_moderation_since (the OTHER, global-watermark + # variant) already used int(m["tid"]) and was never affected. mod_out_tids = [] mod_in_tids = [] meta_tids = [] for m in mods: - tid = str(m["tid"]) + tid = m["tid"] # Check moderation status with support for string values mod_value = m["mod"] @@ -712,8 +743,15 @@ def poll_moderation( # Execute query mod_ptpts = self.query(sql_ptpts, params) - # Format moderated participants - mod_out_ptpts = [str(p["pid"]) for p in mod_ptpts] + # Format moderated participants. pid kept as the DB's native int — + # NOT str()-wrapped (2026-07-24 live finding, session 3): keeps this + # consistent with poll_votes/poll_votes_since's (also-int) pid, for + # Conversation._apply_moderation's ``p not in self.mod_out_ptpts`` + # check ('improved' engine mode only — 'clojure-legacy' intentionally + # leaks bans and skips this check entirely, so this specific fix has + # no observable effect in the mode this harness runs in, but matters + # for 'improved' mode elsewhere). + mod_out_ptpts = [p["pid"] for p in mod_ptpts] return { "mod_out_tids": mod_out_tids, diff --git a/delphi/polismath/poller/__init__.py b/delphi/polismath/poller/__init__.py index 302634ddf0..ddc064a00b 100644 --- a/delphi/polismath/poller/__init__.py +++ b/delphi/polismath/poller/__init__.py @@ -48,39 +48,79 @@ (conversation.py:789) and ``_fold_base_clusters`` writes ``base-clusters.id`` / ``.members`` in that order (conversation.py:1643-1649), so ``[c['members'] for c in conv.base_clusters]`` is the exact alignment the server -needs. ``derive_bidtopid`` (math_writer.py) implements this. Pids are strings -Python-side (poll_votes casts ``str(pid)``) vs ints Clojure-side; the server -parseInt()s them (participants.ts:53-55), so a parity comparer needs int/str -tolerance on this one field. - -load-or-init finding (from_dict restoration is PARTIAL) -------------------------------------------------------- -``Conversation.from_dict`` (conversation.py:2249-2303) restores from a dict with -underscore/nested keys: ``last_updated, participant_count, comment_count, -vote_stats, moderation{...}, pca{center,comps}, proj, group_clusters, repness, -participant_info, comment_priorities``. ``Conversation.to_dict`` (used as the -math_main ``data`` blob) is a SUPERSET that carries those same underscore keys -alongside the hyphenated Clojure keys, so ``from_dict(to_dict(conv))`` round-trips -the listed fields — notably the PCA warm-start vectors and prior moderation. - -But ``from_dict`` does NOT restore: ``raw_rating_mat`` / ``rating_mat`` (the vote -matrices), ``base_clusters``, ``subgroup_clusters``, ``group_clusterings`` / -``group_k_smoother`` (warm smoother state), ``consensus`` or ``group_votes``. +needs. ``derive_bidtopid`` (math_writer.py) implements this. + +UPDATE 2026-07-24 (poller-equivalence harness live debugging, quirk finding): +until this date, ``PostgresClient.poll_votes``/``poll_votes_since`` cast +``str(pid)`` at ingress, while Clojure holds the DB's native int pid +throughout — the server's ``parseInt()`` (participants.ts:53-55) papered over +it, but it made ``bidToPid``/``base-clusters.members`` diverge bit-for-bit +from a live clj container (confirmed: the CSV/certify replay driver never +cast pid at all, and its blobs already matched clj int-for-int). +``Conversation.update_votes`` is deliberately type-agnostic at ingress +(``ptpt_id = vote.get('pid')``/``comment_id = vote.get('tid') # Preserve +original type``) and raw_rating_mat/rating_mat are ALWAYS rebuilt fresh from +these two methods on load-or-init (never restored via ``from_dict`` — see +this file's "load-or-init finding" section), so removing the ``str()`` cast +was a one-point fix with no other code changes needed: pids are now native +ints end-to-end, Python-side AND Clojure-side, and ``derive_bidtopid``'s +``_normalize_bidtopid``-style int/str tolerance is now redundant +defensive-coding for this field rather than a load-bearing requirement +(kept — harmless, and guards a future regression). + +UPDATE 2026-07-24, same day (session 3): ``tid`` (and ``zid``) had the +IDENTICAL bug, just masked by the sheer volume of pid divergences until +session 2's fix above landed — a follow-up live vw full-run then showed +``Type mismatch: golden=int, current=str`` on the top-level ``zid``, every +``tids[i]``, and every ``repness.[i].tid``. Fixed the same way, same +day: ``poll_votes``/``poll_votes_since`` no longer cast ``str(tid)`` either, +``poll_moderation`` (the single-zid full-state variant — NOT +``poll_moderation_since``, which already used int) no longer casts +``str()`` on tid OR pid (needed for internal consistency once votes-side +ids became int — see ``postgres.py``'s ``poll_moderation`` docstring for +why a stale str-tid there would have silently DISABLED moderated-out +comment zeroing), and ``polismath/poller/service.py``'s cold-start +``Conversation(str(zid), ...)`` construction now passes the int through. +The certified/CSV replay driver never cast tid (or zid) either, and matched +clj int-for-int across 20 cross-validated entries — the evidence that +authorized this follow-up fix. The scattered ``int(tid) if +isinstance(tid, str) and tid.isdigit()`` idioms elsewhere in +conversation.py are DEFENSIVE normalizers (no-ops on an already-int input), +not evidence tid needed to stay a string. + +load-or-init finding (from_dict restoration is PARTIAL — updated 2026-07-24) +----------------------------------------------------------------------------- +``Conversation.from_dict`` (conversation.py:2818-2966) restores from a dict with +underscore/nested keys: ``zid, last_updated, participant_count, comment_count, +vote_stats, moderation{...}, pca{center,comps}, proj, group_clusters, +base_clusters, group_votes, repness, participant_info, comment_priorities``. +``Conversation.to_dict`` (used as the math_main ``data`` blob) is a SUPERSET +that carries those same underscore keys alongside the hyphenated Clojure keys, +so ``from_dict(to_dict(conv))`` round-trips the listed fields — notably the PCA +warm-start vectors, prior moderation, base-cluster LINEAGE (id/members, unfolded +exactly as Clojure's restructure-json-conv, conv_man.clj:171-186 -> +clusters.clj unfold-clusters), and group-votes (needed by the recovery tick's +comment-priorities calc, Q2, conversation.clj:658). + +As of 2026-07-24, ``base_clusters`` / ``zid`` / ``group_votes`` ARE restored +(conversation.py:2905-2921 base_clusters, :2923-2952 group_votes) — this note +previously said they were NOT; that was fixed to mirror Clojure's +restructure-json-conv (conv_man.clj:173 keeps ``:base-clusters`` in the +subset, :180 unfolds them) instead of re-deriving base-cluster lineage cold. + +``from_dict`` still does NOT restore: ``raw_rating_mat`` / ``rating_mat`` (the +vote matrices — never touched anywhere in ``from_dict``) or +``group_clusterings`` / ``group_k_smoother`` (warm smoother state), nor the +dead ``subgroup_clusters`` / ``consensus`` paths (CLOJURE_QUIRKS.md Q7). Therefore load-or-init ALWAYS rebuilds the rating matrices from the full vote history (``poll_votes(zid)`` ordered by zid,tid,pid,created — parity with -conv-poll offset 0) and recomputes base_clusters; the non-persisted smoother -state cold-starts. This is CLOSE TO — but not byte-identical with — a Clojure -worker restart: on restart Clojure ``restructure-json-conv`` RESTORES -``base-clusters`` (and the PCA) from the persisted blob before its ``:reboot`` -recompute (conv_man.clj:173 keeps ``:base-clusters`` in the subset, :180 unfolds -them), whereas Python re-derives base_clusters cold -from the vote matrices. The rating-matrix rebuild itself matches -(conv_man.clj:188-207 rebuilds ``raw-rating-mat`` the same way), and we -opportunistically seed the warm PCA start from ``from_dict`` when a row exists -(low-risk, literally what ``restructure-json-conv`` does). The base-cluster -lineage difference is a KNOWN divergence to trace against Clojure's ``:reboot`` -semantics before the parity gate; a full cold rebuild is otherwise correct — -just without Clojure's restored-lineage warm start. +conv-poll offset 0); the non-persisted smoother state cold-starts. The +rating-matrix rebuild itself matches Clojure (conv_man.clj:188-207 rebuilds +``raw-rating-mat`` the same way on restart). The remaining gap versus a true +Clojure worker restart is narrower than before: only the non-persisted warm +smoother state (group_clusterings/group_k_smoother) cold-starts — tracked as a +KNOWN divergence to trace against Clojure's ``:reboot`` semantics before the +parity gate. Config var mapping (config.py names PREFERRED, design aliases accepted) ----------------------------------------------------------------------- diff --git a/delphi/polismath/poller/math_writer.py b/delphi/polismath/poller/math_writer.py index 2e05d4a41e..f36c381ede 100644 --- a/delphi/polismath/poller/math_writer.py +++ b/delphi/polismath/poller/math_writer.py @@ -22,6 +22,10 @@ import uuid from typing import Any, Dict, List, Optional +import numpy as np + +from polismath.utils.clj_hash import clojure_hash_map_key_order + logger = logging.getLogger(__name__) @@ -46,10 +50,16 @@ def derive_bidtopid(conv: Any, zid: int) -> Dict[str, Any]: ``[c['members'] for c in conv.base_clusters]`` is positionally aligned with ``base-clusters.id`` — the exact alignment the server relies on. - Note on element type: Python pids are strings (poll_votes casts ``str(pid)``), - whereas Clojure emits integer pids. The server parseInt()s them - (participants.ts:53-55) so both work; a parity comparer needs int/str - tolerance on this field. Members are left as-is so that + Note on element type: as of 2026-07-24 (poller-equivalence harness live + debugging), ``PostgresClient.poll_votes``/``poll_votes_since`` no longer + cast ``str(pid)`` — pids are native ints Python-side, matching Clojure's + integer pids, end-to-end. (Before that date this docstring said Python + pids were strings; that was a real, unintentional divergence — the CSV/ + certify replay driver never cast pid at all and already matched clj + int-for-int, so the live poller path was the outlier, not the norm.) The + server still ``parseInt()``s either form defensively + (participants.ts:53-55), so this is stronger-than-required parity, not a + behavior change for it. Members are left as-is so that math_bidtopid.bidToPid and math_main.base-clusters.members stay identical. Args: @@ -71,16 +81,138 @@ def derive_bidtopid(conv: Any, zid: int) -> Dict[str, Any]: } -def derive_ptptstats(conv: Any, zid: int) -> Dict[str, Any]: - """Derive the prep-ptpt-stats blob (conv_man.clj:90-94). +def _unfold_group_members(conv: Any) -> List[Dict[str, Any]]: + """``[{"id": gid, "members": [pid, ...]}, ...]`` — each group's + base-cluster members (bids) expanded to participant ids via + ``conv.base_clusters``. Reimplemented locally rather than calling + ``Conversation._unfolded_group_clusters`` (a private method) so this + module stays testable against lightweight ``SimpleNamespace`` fakes + exposing only public attrs — the same pattern :func:`derive_bidtopid` + already uses (it re-sorts ``base_clusters`` itself rather than calling + a conv method too).""" + base_clusters = getattr(conv, "base_clusters", None) or [] + bid_to_pids = {c["id"]: list(c.get("members", [])) for c in base_clusters} + unfolded = [] + for g in getattr(conv, "group_clusters", None) or []: + members: List[Any] = [] + for bid in g.get("members", []): + members.extend(bid_to_pids.get(bid, [])) + unfolded.append({"id": g["id"], "members": members}) + return unfolded + + +def _group_iteration_order(groups: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Clojure's ``group-data`` map (``conv_man.clj``'s ``(into {} (map (fn + [{:keys [id members]}] [id {...}]) group-clusters))``) is an ARRAY-map + (insertion / ``group-clusters`` order) for <=8 groups but a + ``PersistentHashMap`` (HAMT id-hash order) for >8 groups — the EXACT same + threshold ``legacy_kmeans.py``'s ``cleared-clusters`` scan order already + documents and relies on (same :func:`clojure_hash_map_key_order` + utility). Polis "groups" (as opposed to the finer base-clusters) are + almost always a handful, so this only matters in pathological cases — + but getting it right costs one function call.""" + if len(groups) <= 8: + return groups + order = clojure_hash_map_key_order([g["id"] for g in groups]) + by_id = {g["id"]: g for g in groups} + return [by_id[gid] for gid in order] + + +def _columnize(rows: List[Dict[str, Any]]) -> Dict[str, List[Any]]: + """Mirrors Clojure's ``columnize`` (conv_man.clj:79-88): transpose a list + of per-participant stat dicts into ``{key: [val, val, ...]}`` using the + FIRST row's key set (every row shares the same keys by construction + here). An EMPTY ``rows`` returns ``{}`` — NOT a dict of empty-array + columns — matching Clojure's own empty-seq behavior (``(-> stats first + keys)`` on ``()`` is ``nil``, so ``columnize`` degenerates to + ``(into {} nil)`` = ``{}``).""" + if not rows: + return {} + keys = list(rows[0].keys()) + return {k: [r[k] for r in rows] for k in keys} + + +def derive_ptptstats( + conv: Any, zid: int, user_vote_counts: Optional[Dict[Any, int]] = None, +) -> Dict[str, Any]: + """Derive the prep-ptpt-stats blob (conv_man.clj:90-94), matching + Clojure's COLUMNAR shape verbatim — a REAL py-poller bug fix, 2026-07-24 + (poller-equivalence harness live debugging session 2): production + consumers read the clj shape, and what this function emitted before this + date (a bare wrap of ``conv.participant_info``) was not merely + differently-SHAPED but a COMPLETELY DIFFERENT STATISTIC — Python's + ``participant_info`` is vote-correlation-based (n_agree/n_disagree/ + n_pass/group_correlations, ``_compute_participant_info_optimized``), + while Clojure's ``ptptstats`` is GEOMETRIC (distance-to-center in the + PCA-projected plane, ``repness/participant-stats``, math/repness.clj: + 383-413). ``participant_info`` is left UNTOUCHED — it's still consumed + elsewhere (run_math_pipeline.py, narrative reporting) under its own, + Python-only semantics (crosslang.py explicitly excludes it from the + clj-parity acceptance surface); this function no longer reads it at all. + + Verbatim port of ``repness/participant-stats``: - ptptstats is a secondary consumer (scoped "replace", not fidelity-critical - like math_main / math_bidtopid). We wrap the conversation's public - ``participant_info`` under the same envelope keys Clojure uses. + bid->pid = base-clusters id -> members (participant ids) + global-center = mean of ALL in-conv participants' proj positions + for each group (base-cluster ids expanded to participant ids): + center = mean of THIS group's participants' proj positions + extreme-direction = normalise(center - global-center) + for each participant pid in the group: + centricness = 1 - |proj[pid] - global-center| + coreness = 1 - |proj[pid] - center| + extremeness = dot(proj[pid] - center, extreme-direction) + n-votes = user_vote_counts.get(pid) (None if missing, like + Clojure's (get ptpt-vote-counts pid) -> nil) + + then COLUMNIZED (:func:`_columnize`) into ``{pid, gid, n-votes, + centricness, coreness, extremeness}``, each a same-length, positionally- + aligned array — group visitation order via :func:`_group_iteration_order` + (Clojure array-map vs hash-map, threshold 8). + + ``user_vote_counts`` is the caller's ALREADY-COMPUTED + ``data["user-vote-counts"]`` (from ``conv.to_dict()``, needed for + math_main anyway) rather than recomputed here — keeps this function + testable against lightweight fakes with no pandas dependency, and avoids + a second full vote-count pass per write cycle. + + Structural fidelity verified against a REAL clj-ref row captured live + (real_data/.local/replays/poller_equiv/vw/main/clj-ref/batch-000/ + math_ptptstats.json, 2026-07-24 vw full-run) — see + tests/poller/test_math_writer.py::TestDerivePtptstatsMatchesLiveClj. """ + user_vote_counts = user_vote_counts or {} + groups = _group_iteration_order(_unfold_group_members(conv)) + proj = getattr(conv, "proj", None) or {} + + rows: List[Dict[str, Any]] = [] + if groups and proj: + positions = np.array(list(proj.values()), dtype=float) + global_center = positions.mean(axis=0) + + for g in groups: + members = [pid for pid in g["members"] if pid in proj] + if not members: + continue + member_positions = np.array([proj[pid] for pid in members], dtype=float) + center = member_positions.mean(axis=0) + direction = center - global_center + norm = float(np.linalg.norm(direction)) + extreme_direction = direction / norm if norm > 0 else direction + + for pid in members: + pos = np.asarray(proj[pid], dtype=float) + rows.append({ + "pid": pid, + "gid": g["id"], + "n-votes": user_vote_counts.get(pid), + "centricness": float(1 - np.linalg.norm(pos - global_center)), + "coreness": float(1 - np.linalg.norm(pos - center)), + "extremeness": float(np.dot(pos - center, extreme_direction)), + }) + return { "zid": zid, - "ptptstats": getattr(conv, "participant_info", {}) or {}, + "ptptstats": _columnize(rows), "lastVoteTimestamp": getattr(conv, "last_updated", None), } @@ -114,9 +246,13 @@ def write_conv_updates(self, zid: int, conv: Any) -> int: self._pg.write_math_bidtopid( zid, data=derive_bidtopid(conv, zid), math_tick=math_tick ) - # 3. math_ptptstats — participant stats + # 3. math_ptptstats — participant stats (clj-shaped, 2026-07-24 fix). + # Reuses data["user-vote-counts"] (already computed above for + # math_main) rather than recomputing it a second time. self._pg.write_participant_stats( - zid, data=derive_ptptstats(conv, zid), math_tick=math_tick + zid, + data=derive_ptptstats(conv, zid, data.get("user-vote-counts", {})), + math_tick=math_tick, ) logger.info( diff --git a/delphi/polismath/poller/service.py b/delphi/polismath/poller/service.py index 1c1b1fe88c..6285e42724 100644 --- a/delphi/polismath/poller/service.py +++ b/delphi/polismath/poller/service.py @@ -394,10 +394,12 @@ def _load_or_init(self, zid: int) -> Conversation: """Mirror Clojure load-or-init (conv_man.clj:188-207). Restores warm state from math_main via ``Conversation.from_dict`` when a - row exists, then ALWAYS rebuilds the rating matrices from the full vote - history and applies the full moderation state (from_dict restores neither - the matrices nor base_clusters — see the poller package docstring's - "load-or-init finding"). Non-persisted warm smoother state cold-starts, + row exists (as of 2026-07-24 this includes base_clusters/zid/group_votes, + mirroring Clojure's restructure-json-conv — from_dict still does NOT + restore the rating matrices or the warm smoother state; see the poller + package docstring's "load-or-init finding"), then ALWAYS rebuilds the + rating matrices from the full vote history and applies the full + moderation state. Non-persisted warm smoother state cold-starts, exactly like a Clojure worker restart. last_updated is seeded NONZERO-but-low (not wall-clock): ``Conversation``'s @@ -444,7 +446,19 @@ def _load_or_init(self, zid: int) -> Conversation: # conversation.clj:161-165) so a zero-votes conversation emits # lastVoteTimestamp=0, not the internal seed. Any real vote advances # it via max() in update_votes. - conv = Conversation(str(zid), last_updated=1) + # + # `zid` (int) passed through AS-IS — NOT str(zid) — since + # 2026-07-24 (live poller-equivalence harness finding, session + # 3): Conversation.__init__ just does a bare + # `self.conversation_id = conversation_id` (no string-specific + # logic anywhere on that attribute — every `.conversation_id` + # use site was grepped; the only str() casts are at the + # DynamoDB boundary, database/dynamodb.py, which already + # handles either type defensively) and Clojure holds zid as an + # int throughout, so this was a real, live, one-point Type- + # mismatch divergence (to_dict()['zid'], every tids[i], every + # repness.*.tid all trace back to this same conversation_id). + conv = Conversation(zid, last_updated=1) conv.last_updated = 0 votes = self._pg.poll_votes(zid, None) # full history, ordered, sign-flipped diff --git a/delphi/polismath/replay/poller_equiv.py b/delphi/polismath/replay/poller_equiv.py new file mode 100644 index 0000000000..7b8802c6d6 --- /dev/null +++ b/delphi/polismath/replay/poller_equiv.py @@ -0,0 +1,2774 @@ +"""Poller-equivalence harness — schema/seeder (Stage A) + runners (Stage B) + +feeder/comparer (Stage C). + +See ``delphi/docs/MATH_POLLER_EQUIV_SPEC.md`` for the full design (goal +condition 2: poller equivalence — identical math_main/bidToPid/ptptstats rows +and tick/watermark semantics between the Clojure math container and the +Python poller replaying the SAME vote stream against one throwaway Postgres). +This module implements spec §3 stages A (schema + seeder), B (runners), +C (feeder + comparer, see the "Stage C" section near the bottom of this file), +and D (self-jitter envelope + full-run orchestration, see the "Stage D" +section at the very bottom): :func:`compute_self_jitter_envelope` measures +clj-vs-clj float jitter across two independent runs of the same stream; +:func:`compare_batch`/:func:`compare_snapshots` accept an optional +``envelope`` parameter that accepts (and separately counts) float mismatches +within the envelope while never excusing structural divergences; +:func:`run_full_equiv_protocol` orchestrates the complete spec protocol +(two clj-only self-jitter runs -> envelope -> one paired clj+py restart-seam +run -> envelope-aware compare -> verdict), with its decision logic split into +the pure, canned-dir-testable :func:`assemble_full_run_verdict`. + +Schema derivation (spec item A.1 — "do not guess; quote the clj SQL") +----------------------------------------------------------------------- +The Clojure ``full`` subcommand (``clojure -M:run full``, ``deps.edn:65-66`` +``-m polismath.runner``, subcommand table ``runner.clj:71-78`` ``"full"`` -> +``system/full-system``) is defined as ``(merge (poller-system +config-overrides))`` (``system.clj:47-52``), and ``poller-system`` is +``base-system`` (config, logger, core-matrix-boot, postgres, +conversation-manager) plus a votes poller and a moderation poller +(``system.clj:29-33``). Notably this does NOT include ``task-system`` +(worker_tasks) or ``export-system``/darwin (``participants`` table reads +live ONLY in ``darwin/export.clj`` — grepped, confirmed absent from +poller.clj/conv_man.clj/postgres.clj) — so those tables are OUT of scope for +the clj side of this harness. + +Tables touched, with the exact query/columns (clj file:line, then py +file:line): + +* ``votes`` — clj ``postgres/poll`` (postgres.clj:132-145, global watermark + loop) and ``postgres/conv-poll`` (postgres.clj:197-212, load-or-init full + history) both ``SELECT * ... ORDER BY zid, tid, pid, created WHERE created > + ts``; only ``:pid :tid :vote`` are actually destructured downstream + (conv_man.clj:202-203), ``:zid``/``:created`` drive grouping/watermark + (poller.clj:18-27). Py: ``PostgresClient.poll_votes_since`` + (postgres.py:534-567) and ``.poll_votes`` (postgres.py:474-532) — both + ``SELECT zid, tid, pid, vote, created ... ORDER BY zid, tid, pid, created``. + Columns: ``zid, pid, tid, vote, created`` (+ ``weight_x_32767`` kept for + shape-fidelity with the real ``SELECT *`` — never read by either poller). +* ``comments`` — clj ``postgres/mod-poll`` (postgres.clj:148-161, global) and + ``postgres/conv-mod-poll`` (postgres.clj:214-225, load-or-init) both + ``SELECT * ... ORDER BY zid, tid, modified WHERE modified > ts``; consumed + by ``conv/mod-update`` (math/conversation.clj:846-884) which destructures + ``:tid :is_meta :mod :modified``. Py: ``poll_moderation_since`` + (postgres.py:569-604) ``SELECT zid, tid, modified, mod, is_meta`` and + ``poll_moderation`` (postgres.py:645-723) ``SELECT tid, modified, mod, + is_meta``. Columns: ``zid, tid, modified, mod, is_meta`` (+ ``pid, uid, + created, txt`` kept NOT NULL with placeholders — neither poller reads them). +* ``conversations`` — FK target only. Neither poller SELECTs a column off it + (grepped ``math/src/polismath`` for ``conversations``/``strict_moderation`` + outside ``darwin/export.clj`` — no hits); it exists purely so + ``math_main``/``math_ticks``/``math_bidtopid``/``math_ptptstats`` FKs + resolve (``server/postgres/migrations/000000_initial.sql:658-667`` etc, all + ``zid INTEGER [NOT NULL] REFERENCES conversations(zid)``). Columns: ``zid``. +* ``math_ticks`` — clj ``inc-math-tick`` (postgres.clj:292-295): ``insert into + math_ticks (zid, math_env) values (?,?) on conflict (zid, math_env) do + update set modified = now_as_millis(), math_tick = (math_ticks.math_tick + + 1) returning math_tick``. Py: ``increment_math_tick`` (postgres.py:910-939), + byte-identical SQL text. +* ``math_main`` — clj ``upload-math-main`` (postgres.clj:323-338) and + ``load-conv`` (postgres.clj:419-434, ``SELECT * FROM math_main WHERE zid=? + AND math_env=?``). Py: ``write_math_main`` (postgres.py:757-817), + ``load_math_main`` (postgres.py:725-755). Columns: ``zid, math_env, data, + last_vote_timestamp, caching_tick, math_tick, modified``. +* ``math_bidtopid`` — clj ``upload-math-bidtopid`` (postgres.clj:369-380). Py: + ``write_math_bidtopid`` (postgres.py:819-848). Columns: ``zid, math_env, + math_tick, data, modified``. +* ``math_ptptstats`` — clj ``upload-math-ptptstats`` (postgres.clj:350-361). + Py: ``write_participant_stats`` (postgres.py:850-879). Columns: ``zid, + math_env, math_tick, data, modified``. + +Two extra tables that are NOT in the spec's headline list, added after +tracing both poller code paths (the "do not guess" instruction cuts both +ways): + +* ``participants`` (pid, zid, mod) — a SHIM, needed ONLY by the PYTHON side. + ``poll_moderation`` (postgres.py:701-713) unconditionally runs ``SELECT pid + FROM participants WHERE zid=:zid AND (mod=-1 OR mod='-1')`` for + ``mod_out_ptpts`` (the participant-ban leak fix, 2026-06-10) on EVERY + load-or-init and every moderation batch — a missing table raises and parks + the zid (service.py's per-zid exception boundary, ``_handle_zid``). The + Clojure ``full`` poller never queries ``participants`` at all (only + ``darwin/export.clj`` does, out of scope). We create the table but never + seed rows into it: no participant-ban scenario is in scope for this harness. +* ``math_profile`` — clj writes it every vote-batch cycle via + ``handle-profile-data`` (conv_man.clj:97-113) -> ``upload-math-profile`` + (postgres.clj:340-348), including the actor-startup ``react-to-messages! + ... :votes []`` call (conv_man.clj:387). A missing table is non-fatal + (caught + logged, conv_man.clj:104-112) but noisy; included so the clj + container's stderr stays clean and its write-side footprint is + production-shaped. Python never touches this table. + +Vote sign convention (spec item A.2 — "the DB must hold RAW-DB-convention +signs") +----------------------------------------------------------------------- +``votes.vote`` in production is RAW-DB convention: AGREE=-1, DISAGREE=+1 +(``server/postgres/migrations/000000_initial.sql:742-747``). The Clojure +poller consumes this value AS-IS (no flip anywhere in postgres.clj/conv_man.clj +— grepped). The Python poller flips it AT INGRESS +(``polismath.utils.general.postgres_vote_to_delphi``, ``vote * -1``) inside +``poll_votes``/``poll_votes_since`` (postgres.py:474-567) to Delphi convention +(AGREE=+1). Meanwhile ``ReplayDataset.votes[i].sign`` (the seeder's INPUT) is +ALREADY in Delphi convention — ``driver.py``'s module docstring: "export CSVs +are ALREADY in Delphi convention (AGREE=+1)"; ``VOTE_SIGN_CONVENTION = +"delphi"`` (driver.py:56). So the seeder must flip dataset sign -> raw DB sign +via ``polismath.utils.general.delphi_vote_to_postgres`` (the documented +inverse, general.py:43-56) before INSERTing — the opposite direction from the +py poller's ingress flip, landing the DB in the same RAW convention production +holds, which BOTH pollers then read exactly as they read production data. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional, Sequence +from urllib.parse import urlsplit, urlunsplit + +import sqlalchemy as sa + +from polismath.replay import real_data +from polismath.replay import schedule as sched +from polismath.replay.certify import _acceptance_projecting_comparer, normalize_path +from polismath.replay.stepcompare import DEFAULT_TOLERANT_STAT_KEYS, StepComparer +from polismath.replay.store import _safe_path_component +from polismath.replay.types import ModEvent, ReplayDataset +from polismath.utils.engine_mode import ENGINE_MODE_ENV_VAR, ENGINE_MODE_LEGACY +from polismath.utils.general import delphi_vote_to_postgres + +# poller_equiv.py -> replay -> polismath -> delphi -> repo root (mirrors +# certify.py / store.py). +_DELPHI_ROOT = Path(__file__).resolve().parents[2] +_REPO_ROOT = _DELPHI_ROOT.parents[0] +_MATH_ROOT = _REPO_ROOT / "math" + +DEFAULT_DBNAME = "polis_equiv" +DEFAULT_ZID = 1 +# The comment placeholder text is NEVER real content (private-data policy, +# CLAUDE.local.md — never commit vote/comment CONTENT); a bare "comment {tid}" +# also satisfies production's UNIQUE(zid, txt) if that constraint is ever +# reintroduced, though our subset schema does not declare it. +_PLACEHOLDER_TXT_FMT = "comment {tid}" + + +# --------------------------------------------------------------------------- +# Schema (Stage A.1). +# --------------------------------------------------------------------------- +# Verbatim from server/postgres/migrations/000000_initial.sql:22-30 — every +# BIGINT ``modified``/``created``/tick column in the real schema (and both +# pollers' literal SQL, e.g. postgres.clj:295/328/338/... and +# postgres.py's "now_as_millis()" call-sites) depends on this function +# existing; Postgres has no builtin equivalent. +NOW_AS_MILLIS_FN = """ +CREATE OR REPLACE FUNCTION now_as_millis() RETURNS BIGINT AS $$ + DECLARE + temp TIMESTAMP := now(); + BEGIN + RETURN 1000*FLOOR(EXTRACT(EPOCH FROM temp)) + FLOOR(EXTRACT(MILLISECONDS FROM temp)) - 1000*FLOOR(EXTRACT(SECOND FROM temp)); + END; +$$ LANGUAGE plpgsql; +""" + +# conversations: FK target only (see module docstring) — no other column is +# ever read by either poller's "full"/py-poller code path. +CREATE_CONVERSATIONS = """ +CREATE TABLE conversations ( + zid SERIAL PRIMARY KEY +); +""" + +# votes: postgres.clj:132-145 (poll) / :197-212 (conv-poll); postgres.py:474-567 +# (poll_votes / poll_votes_since). No PK/FK — matches production +# (migrations.sql:737-755): a revote is simply a new row, latest-created wins. +CREATE_VOTES = """ +CREATE TABLE votes ( + zid INTEGER NOT NULL, + pid INTEGER NOT NULL, + tid INTEGER NOT NULL, + -- RAW DB convention: -1=agree, +1=disagree, 0=pass/unsure (migrations.sql:742-747). + vote SMALLINT, + -- Present because both pollers' SELECT * would include it in production; + -- never read downstream (nm/update-nmat / poll_votes only touch pid/tid/vote). + weight_x_32767 SMALLINT DEFAULT 0, + created BIGINT NOT NULL +); +""" + +# comments: postgres.clj:148-161 (mod-poll) / :214-225 (conv-mod-poll); +# math/conversation.clj:846-884 (mod-update, destructures tid/is_meta/mod/modified); +# postgres.py:569-604 (poll_moderation_since) / :645-723 (poll_moderation). +# UNIQUE(zid, tid) backs our seeder's idempotent ON CONFLICT upsert. No FK to +# participants (production has one, migrations.sql:505) — this subset schema +# does not model participants as a real per-comment-author table (see the +# `participants` shim below for why one exists at all). +CREATE_COMMENTS = """ +CREATE TABLE comments ( + tid INTEGER NOT NULL, + zid INTEGER NOT NULL REFERENCES conversations(zid), + pid INTEGER NOT NULL DEFAULT 0, + uid INTEGER NOT NULL DEFAULT 0, + created BIGINT NOT NULL DEFAULT 0, + modified BIGINT NOT NULL, + txt VARCHAR(1000) NOT NULL DEFAULT '', + mod INTEGER NOT NULL DEFAULT 0, + is_meta BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE (zid, tid) +); +""" + +# SHIM — needed ONLY by the Python poller (see module docstring): postgres.py +# poll_moderation:701-713 unconditionally selects pid from here. Never seeded +# with rows (no participant-ban scenario is in this harness's scope). +CREATE_PARTICIPANTS = """ +CREATE TABLE participants ( + pid INTEGER NOT NULL, + zid INTEGER NOT NULL REFERENCES conversations(zid), + mod INTEGER NOT NULL DEFAULT 0, + UNIQUE (zid, pid) +); +""" + +# math_ticks: postgres.clj:292-295 inc-math-tick; postgres.py:910-939 +# increment_math_tick. Verbatim column shape from migrations.sql:647-654. +CREATE_MATH_TICKS = """ +CREATE TABLE math_ticks ( + zid INTEGER REFERENCES conversations(zid), + math_tick BIGINT NOT NULL DEFAULT 0, + caching_tick BIGINT NOT NULL DEFAULT 0, + math_env VARCHAR(999) NOT NULL, + modified BIGINT NOT NULL DEFAULT now_as_millis(), + UNIQUE (zid, math_env) +); +""" + +# math_main: postgres.clj:323-338 upload-math-main, :419-434 load-conv; +# postgres.py:757-817 write_math_main, :725-755 load_math_main. Verbatim +# column shape from migrations.sql:658-667. +CREATE_MATH_MAIN = """ +CREATE TABLE math_main ( + zid INTEGER NOT NULL REFERENCES conversations(zid), + math_env VARCHAR(999) NOT NULL, + data jsonb NOT NULL, + last_vote_timestamp BIGINT NOT NULL, + caching_tick BIGINT NOT NULL DEFAULT 0, + math_tick BIGINT NOT NULL DEFAULT -1, + modified BIGINT DEFAULT now_as_millis(), + UNIQUE (zid, math_env) +); +""" + +# math_ptptstats: postgres.clj:350-361 upload-math-ptptstats; postgres.py:850-879 +# write_participant_stats. Verbatim column shape from migrations.sql:679-687. +CREATE_MATH_PTPTSTATS = """ +CREATE TABLE math_ptptstats ( + zid INTEGER NOT NULL REFERENCES conversations(zid), + math_env VARCHAR(999) NOT NULL, + math_tick BIGINT NOT NULL DEFAULT -1, + data jsonb NOT NULL, + modified BIGINT DEFAULT now_as_millis(), + UNIQUE (zid, math_env) +); +""" + +# math_bidtopid: postgres.clj:369-380 upload-math-bidtopid; postgres.py:819-848 +# write_math_bidtopid. Verbatim column shape from migrations.sql:698-706. +CREATE_MATH_BIDTOPID = """ +CREATE TABLE math_bidtopid ( + zid INTEGER NOT NULL REFERENCES conversations(zid), + math_env VARCHAR(999) NOT NULL, + math_tick BIGINT NOT NULL DEFAULT -1, + data jsonb NOT NULL, + modified BIGINT DEFAULT now_as_millis(), + UNIQUE (zid, math_env) +); +""" + +# math_profile: conv_man.clj:97-113 handle-profile-data -> postgres.clj:340-348 +# upload-math-profile (see module docstring — clj write-side only, never read +# by the harness comparer). Verbatim column shape from migrations.sql:670-677. +CREATE_MATH_PROFILE = """ +CREATE TABLE math_profile ( + zid INTEGER NOT NULL REFERENCES conversations(zid), + math_env VARCHAR(999) NOT NULL, + data jsonb NOT NULL, + modified BIGINT DEFAULT now_as_millis(), + UNIQUE (zid, math_env) +); +""" + +# Order matters: FK targets (conversations) before their referrers. +SCHEMA_STATEMENTS: list[str] = [ + NOW_AS_MILLIS_FN, + CREATE_CONVERSATIONS, + CREATE_VOTES, + CREATE_COMMENTS, + CREATE_PARTICIPANTS, + CREATE_MATH_TICKS, + CREATE_MATH_MAIN, + CREATE_MATH_PTPTSTATS, + CREATE_MATH_BIDTOPID, + CREATE_MATH_PROFILE, +] + +# Joined form for introspection/documentation (parse_schema_columns operates +# on this; the function DDL has no "CREATE TABLE" in it so it's harmless here). +SCHEMA_DDL = "\n\n".join(SCHEMA_STATEMENTS) + + +# --------------------------------------------------------------------------- +# DDL introspection — "parse your own DDL" (Stage A test requirement). +# --------------------------------------------------------------------------- +_CREATE_TABLE_START_RE = re.compile(r"CREATE TABLE (\w+)\s*\(", re.IGNORECASE) +_CONSTRAINT_KEYWORDS = frozenset({"UNIQUE", "PRIMARY", "FOREIGN", "CHECK", "CONSTRAINT"}) + + +def _strip_line_comments(sql: str) -> str: + return re.sub(r"--[^\n]*", "", sql) + + +def _split_top_level_commas(body: str) -> list[str]: + """Split ``body`` on commas that are NOT nested inside parens (e.g. the + comma-free ``VARCHAR(999)`` argument list must not count as a split).""" + parts: list[str] = [] + depth = 0 + current: list[str] = [] + for ch in body: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if ch == "," and depth == 0: + parts.append("".join(current)) + current = [] + else: + current.append(ch) + parts.append("".join(current)) + return parts + + +def parse_schema_columns(ddl: str = SCHEMA_DDL) -> dict[str, list[str]]: + """Parse ``CREATE TABLE name (...)`` column names out of ``ddl``. + + NOT a general SQL parser — depth-counts parens to find each table's + closing ``)`` (robust to ``VARCHAR(999)``-style nested parens) and drops + bare constraint lines (``UNIQUE (...)`` etc). Good enough for a + self-consistency check that our own DDL declares the columns each + poller's SELECT/INSERT statement actually needs (module docstring). + """ + text = _strip_line_comments(ddl) + tables: dict[str, list[str]] = {} + for m in _CREATE_TABLE_START_RE.finditer(text): + name = m.group(1) + start = m.end() # just past the opening '(' consumed by the regex + depth = 1 + i = start + while depth > 0 and i < len(text): + if text[i] == "(": + depth += 1 + elif text[i] == ")": + depth -= 1 + i += 1 + body = text[start : i - 1] + cols = [] + for part in _split_top_level_commas(body): + stripped = part.strip() + if not stripped: + continue + first_word = stripped.split()[0].upper() + if first_word in _CONSTRAINT_KEYWORDS: + continue + cols.append(stripped.split()[0]) + tables[name] = cols + return tables + + +# --------------------------------------------------------------------------- +# Seeder (Stage A.2). +# --------------------------------------------------------------------------- +# Never point create_equiv_db at one of these — belt-and-braces guard for +# MATH_POLLER_EQUIV_SPEC.md hazard 1 ("NEVER touch polis-dev / polis_prodclone"). +_PRECIOUS_DBNAMES = frozenset( + {"postgres", "polis-dev", "polis_dev", "polis_prodclone", "polis-prod", "polis_prod"} +) + + +def _url_with_dbname(url: str, dbname: str) -> str: + parts = urlsplit(url) + return urlunsplit((parts.scheme, parts.netloc, f"/{dbname}", parts.query, parts.fragment)) + + +def _url_with_scheme(url: str, scheme: str) -> str: + """Swap ``url``'s scheme, preserving user/pass/host/port/path/query. + + Needed because the clj and py runners require OPPOSITE, MUTUALLY + INCOMPATIBLE schemes for the SAME connection string (root cause #1, + 2026-07-24 live debug — see :func:`build_clj_env`'s docstring): + Clojure's Hikari datasource regex (``postgres.clj``'s + ``create-hikari-datasource``) only matches a literal ``postgres://`` + prefix, while SQLAlchemy/psycopg2 (the py side) reject that exact scheme + (dropped in SQLAlchemy 1.4+) and require ``postgresql://``. Every + connection URL flowing through this module (``admin_url`` / + :func:`create_equiv_db`'s return value) is SQLAlchemy-style + (``postgresql://`` or ``postgresql+driver://``) — this normalizes to + whichever scheme the CALLING runner actually needs, at the last possible + moment, so neither runner ever sees the other's required scheme. + """ + parts = urlsplit(url) + return urlunsplit((scheme, parts.netloc, parts.path, parts.query, parts.fragment)) + + +def _format_days_ago(value: float) -> str: + """Format a days-ago value as a BARE INTEGER string (root cause #2, + 2026-07-24 live debug — see :func:`build_clj_env`'s docstring): Clojure's + ``->long`` config parser is ``Long/parseLong``, which throws (caught + + logged, returns nil) on ANY non-integer-literal string — including + ``"10000.0"``, exactly what ``str()`` on a Python float produces. The + CLI's ``--poll-from-days-ago`` option is ``type=float`` (so fractional + windows are technically allowed), so this ROUNDS to the nearest whole + day rather than truncating a caller-supplied fraction into a raw '.0' + string. Applied on BOTH runners' env assembly for consistency, even + though only the clj side's parser is fatally strict about it — the py + side's ``float(...)`` parses either form fine either way. + """ + return str(int(round(value))) + + +def create_equiv_db(admin_url: str, dbname: str = DEFAULT_DBNAME) -> str: + """``CREATE DATABASE dbname`` (DROP first if it exists), then create the + schema subset inside it. Returns a connection URL for the new database + (same credentials/host as ``admin_url``, dbname swapped). + + ``admin_url`` must point at an EXISTING database on the target server that + is NOT ``dbname`` itself — Postgres refuses DROP/CREATE DATABASE on the + database a connection is currently attached to (point this at the + server's ``postgres`` maintenance db, or any db other than ``dbname``). + """ + if dbname in _PRECIOUS_DBNAMES: + raise ValueError( + f"refusing to create/drop {dbname!r}: matches a known-precious " + "database name (MATH_POLLER_EQUIV_SPEC.md hazard 1)" + ) + + admin_engine = sa.create_engine(admin_url, isolation_level="AUTOCOMMIT") + try: + with admin_engine.connect() as conn: + conn.execute(sa.text(f'DROP DATABASE IF EXISTS "{dbname}"')) + conn.execute(sa.text(f'CREATE DATABASE "{dbname}"')) + finally: + admin_engine.dispose() + + target_url = _url_with_dbname(admin_url, dbname) + schema_engine = sa.create_engine(target_url) + try: + with schema_engine.begin() as conn: + for stmt in SCHEMA_STATEMENTS: + conn.execute(sa.text(stmt)) + finally: + schema_engine.dispose() + return target_url + + +def seed_conversation(conn: Any, dataset: ReplayDataset, zid: int = DEFAULT_ZID) -> None: + """Insert the conversation row + comment rows for ``dataset`` under + ``zid``. Idempotent (ON CONFLICT DO NOTHING on both inserts) — safe to + call again against an already-seeded conversation. + + Comment text is ALWAYS a placeholder (``"comment {tid}"``) — never real + content (private-data policy). Comment ``mod``/``modified`` seed at the + UNMODERATED baseline (``mod=0``, ``modified=created``); applying dataset + ``mod_events`` over time is the feeder's job (Stage C), not this seeder's. + Votes are NOT inserted here — see :func:`insert_votes` for the timed + batches the driver loop issues incrementally. + """ + conn.execute( + sa.text("INSERT INTO conversations (zid) VALUES (:zid) ON CONFLICT (zid) DO NOTHING"), + {"zid": zid}, + ) + for tid in sorted(dataset.comments): + meta = dataset.comments[tid] + conn.execute( + sa.text( + "INSERT INTO comments (tid, zid, pid, uid, created, modified, txt, mod, is_meta) " + "VALUES (:tid, :zid, 0, 0, :created, :created, :txt, 0, :is_meta) " + "ON CONFLICT (zid, tid) DO NOTHING" + ), + { + "tid": tid, + "zid": zid, + "created": meta.created_ms, + "txt": _PLACEHOLDER_TXT_FMT.format(tid=tid), + "is_meta": meta.is_meta, + }, + ) + + +def insert_votes( + conn: Any, dataset: ReplayDataset, from_slot: int, to_slot: int, zid: int = DEFAULT_ZID +) -> int: + """Insert ``dataset.votes[from_slot:to_slot]`` (plain 0-based Python slice + — consistent with :func:`polismath.replay.schedule.slice_schedule`'s own + ``dataset.votes[prev:cut]`` use of 1-based cut slots as slice bounds), + preserving ``pid``/``tid``/``created`` (``t_ms``) and flipping the vote + sign from the dataset's Delphi convention to RAW DB convention (module + docstring) via :func:`polismath.utils.general.delphi_vote_to_postgres`. + + ATOMIC per batch — ONE multi-row ``INSERT ... VALUES (...), (...), ...`` + statement, never a per-row loop (ROOT CAUSE #5, 2026-07-24 live-debug + task, found AFTER :func:`snap_cuts_past_timestamp_ties` fixed the + CROSS-batch tie boundary: a genuine INTRA-batch race remained). Both + pollers run continuously (~1s interval) REGARDLESS of harness batch + boundaries; under the harness's AUTOCOMMIT isolation level, a per-row + execute() loop lets a poller's concurrent SELECT observe a PARTIAL + batch mid-insert. If that partial snapshot's max ``created`` happens to + tie with a not-yet-committed row's ``created`` (the vw dataset's + timestamps are 1-second-granular with most seconds shared by several + votes — see the module docstring), the watermark's STRICT ``created > + ts`` comparison (both engines, byte-identical SQL) permanently drops + that row — reproduced live: pid 33's vote at dataset index 2050, + comfortably INSIDE a batch's slice (nowhere near either cut edge), + silently vanished from clj-ref's own vote count. A single multi-row + INSERT is one atomic unit under Postgres MVCC: a concurrent reader sees + either NONE or ALL of a batch's rows, never a subset — this is NOT the + same as DBAPI ``executemany`` (psycopg2's default executemany is ITSELF + a client-side loop of single-row execute calls, no atomicity gained). + + NOT idempotent by design: a repeated call over an overlapping range + inserts duplicate rows, exactly like production (``votes`` has no + unique constraint — a revote is simply a new row). Callers (the future + Stage C feeder) must call this once per NEW batch, not repeatedly for + the same range. + + Returns the number of rows inserted. Issues NO statement at all for an + empty slice (``from_slot == to_slot``) — an empty ``VALUES ()`` clause + is invalid SQL, and there is nothing to insert anyway. + """ + rows = dataset.votes[from_slot:to_slot] + if not rows: + return 0 + value_clauses = [] + params: dict[str, Any] = {"zid": zid} + for i, v in enumerate(rows): + value_clauses.append(f"(:zid, :pid{i}, :tid{i}, :vote{i}, :created{i})") + params[f"pid{i}"] = v.pid + params[f"tid{i}"] = v.tid + params[f"vote{i}"] = delphi_vote_to_postgres(v.sign) + params[f"created{i}"] = v.t_ms + stmt = ( + "INSERT INTO votes (zid, pid, tid, vote, created) VALUES " + + ", ".join(value_clauses) + ) + conn.execute(sa.text(stmt), params) + return len(rows) + + +def insert_mod_events( + conn: Any, + dataset: ReplayDataset, + prev_time_ms: int | None, + cut_time_ms: int, + zid: int = DEFAULT_ZID, +) -> int: + """Apply ``dataset.mod_events`` with ``prev_time_ms < t_ms <= cut_time_ms`` + (``prev_time_ms=None`` means no floor — the first batch) as + ``comments.mod``/``comments.modified`` UPDATEs. The moderation-stream + analogue of :func:`insert_votes`, added session 2 (2026-07-24) to + actually exercise pc-meta-02's ``"interleave-by-timestamp"`` moderation + schedule — the live feeder never had this before (only the CSV-based + driver, :func:`polismath.replay.schedule.slice_schedule`, applied + mod_events; ``seed_conversation``'s own docstring flagged this as "the + feeder's job", but Stage C never built it). + + Time-windowing is IDENTICAL to ``slice_schedule`` (schedule.py:206-225: + ``m.t_ms <= cut_time_ms and (prev_time is None or m.t_ms > prev_time)``) + — an event is attached to the FIRST batch whose cut reaches it, so both + drivers agree on which batch a given mod_event lands in. Events after + the final cut are silently excluded (same "tail" convention as tail + votes) — the CALLER controls this simply by never invoking the function + with a ``cut_time_ms`` past the schedule's last cut. + + ATOMIC — ONE ``UPDATE ... FROM (VALUES ...)`` statement, never a per-row + loop (same rationale as :func:`insert_votes`'s root cause #5: a + per-row loop under AUTOCOMMIT would let a concurrent poller observe a + partial moderation batch mid-update). Multiple events for the SAME tid + within one window are DE-DUPLICATED to the latest (highest ``t_ms``) + BEFORE the statement is built — Postgres's ``UPDATE ... FROM`` has + UNSPECIFIED behavior when the FROM subquery matches a target row more + than once, so this must never be left to the database. + + Returns the number of RAW events in the window (mirrors + :func:`insert_votes`'s row-count contract), even though fewer VALUES + rows may actually be sent due to de-duplication. Issues NO statement for + an empty window (including the common case of a dataset with zero + mod_events at all — e.g. vw). + """ + events = [ + e for e in dataset.mod_events + if e.t_ms <= cut_time_ms and (prev_time_ms is None or e.t_ms > prev_time_ms) + ] + if not events: + return 0 + latest: dict[int, ModEvent] = {} + for e in events: # dataset.mod_events is t_ms-sorted (ReplayDataset.build) -> later wins. + latest[e.tid] = e + value_clauses = [] + params: dict[str, Any] = {"zid": zid} + for i, (tid, e) in enumerate(latest.items()): + value_clauses.append(f"(:tid{i}, :mod{i}, :modified{i})") + params[f"tid{i}"] = tid + params[f"mod{i}"] = e.mod + params[f"modified{i}"] = e.t_ms + stmt = ( + "UPDATE comments SET mod = v.mod, modified = v.modified " + "FROM (VALUES " + ", ".join(value_clauses) + ") AS v(tid, mod, modified) " + "WHERE comments.tid = v.tid AND comments.zid = :zid" + ) + conn.execute(sa.text(stmt), params) + return len(events) + + +# --------------------------------------------------------------------------- +# Runners (Stage B.1/B.2) — subprocess wrappers for the clj container loop and +# the python poller CLI, sharing one lifecycle (start/is_alive/kill). +# --------------------------------------------------------------------------- +class _SubprocessRunner: + """Shared Popen lifecycle. ``kill()`` is SIGTERM-then-SIGKILL: a grace + period for cooperative shutdown (both the JVM and the python poller + install signal handling — math_poller.py:63-68 traps SIGTERM/SIGINT), then + an unconditional SIGKILL so a hung/ignoring process never blocks the + harness (spec B.1 — "kill() must terminate the JVM (SIGKILL after grace)"). + + ``log_path`` (REQUIRED FIX #3, 2026-07-24 live-debug task — "RUNNER + EVIDENCE"): when given, stdout+stderr are redirected DIRECTLY to that + file (append mode — a seam restart reusing the same path preserves the + pre-seam timeline in one file) instead of a ``subprocess.PIPE``. This is + not merely a debugging nicety: an un-drained PIPE fills its OS buffer + (~64KB) once the child writes enough output, at which point the child + BLOCKS on its next write — a classic subprocess deadlock — because the + live feeder loop (:func:`run_batch_loop`) never reads from the runners' + stdout the way the standalone ``run-clj``/``run-py`` CLI subcommands do + (``_run_and_stream`` in ``scripts/poller_equiv.py``). ``log_path=None`` + (the default) preserves the original PIPE-based contract those + subcommands rely on.""" + + def __init__( + self, cmd: list[str], *, cwd: Path, env: dict[str, str], + log_path: Optional[Path] = None, + ): + self.cmd = cmd + self.cwd = cwd + self.env = env + self.log_path = Path(log_path) if log_path is not None else None + self._proc: Optional[subprocess.Popen] = None + self._log_fh: Optional[Any] = None + + def start(self) -> subprocess.Popen: + if self.log_path is not None: + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self._log_fh = open(self.log_path, "a") + stdout_target: Any = self._log_fh + else: + stdout_target = subprocess.PIPE + self._proc = subprocess.Popen( + self.cmd, + cwd=str(self.cwd), + env=self.env, + stdout=stdout_target, + stderr=subprocess.STDOUT, + text=True, + ) + return self._proc + + @property + def pid(self) -> Optional[int]: + return self._proc.pid if self._proc is not None else None + + def is_alive(self) -> bool: + return self._proc is not None and self._proc.poll() is None + + def wait(self, timeout: Optional[float] = None) -> Optional[int]: + if self._proc is None: + return None + return self._proc.wait(timeout=timeout) + + def kill(self, grace: float = 5.0) -> None: + try: + if self._proc is None or self._proc.poll() is not None: + return + self._proc.terminate() # SIGTERM: cooperative shutdown attempt + try: + self._proc.wait(timeout=grace) + except subprocess.TimeoutExpired: + self._proc.kill() # SIGKILL: unconditional + self._proc.wait(timeout=grace) + finally: + if self._log_fh is not None: + self._log_fh.close() + self._log_fh = None + + +def build_clj_env( + *, + database_url: str, + math_env: str, + poll_from_days_ago: float = 10000, + logging_level: str = "info", + base_env: Optional[dict[str, str]] = None, +) -> dict[str, str]: + """Env for ``clojure -M:run full``, keyed exactly to what + ``polismath.components.config`` actually reads (config.clj rules map, + :63-114 — environ lower-kebabs the env var name): + + DATABASE_URL -> :database-url (config.clj:68; postgres.clj:108 + asserts non-nil at Postgres component start). + TRANSLATED to the ``postgres://`` scheme + (:func:`_url_with_scheme`) — root cause #1 + (2026-07-24 live debug): ``create-hikari-datasource`` + (postgres.clj:18) parses this with + ``#"postgres://(?:(.+):(.*)@)?([^:]+)(?::(\\d+))?/(.+)"``, + which only matches a LITERAL "postgres://" prefix. + The SQLAlchemy-style "postgresql://" URLs this + module otherwise deals in (``create_equiv_db``'s + return value) make ``re-matches`` return nil, so + user/password/host/port/db ALL destructure to + nil -> ``jdbc:postgresql://:5432/`` (empty host, + default port) -> immediate ConnectException. + Reproduced live against a real Postgres; fixed by + this translation (verified: math_main rows are + written once corrected). + MATH_ENV -> :math-env (config.clj:65; becomes + :math-env-string, the upsert key on every + math_main/math_ticks/... row) + POLL_FROM_DAYS_AGO -> :poll-from-days-ago (config.clj:106; poller.clj:15 + ``start-polling-from = now - poll-from-days-ago + days`` — 10000 days makes every historical vote + timestamp "since" the watermark). FORMATTED as a + bare integer string (:func:`_format_days_ago`) — + root cause #2 (2026-07-24 live debug): Clojure's + ``->long`` parser (``Long/parseLong``) throws on + "10000.0" (exactly what ``str()`` on the CLI's + ``type=float`` default produces), silently + becoming nil post-``deep-merge`` (config.clj's + defaults-vs-environ merge REPLACES, not + fall-backs). ``polismath.poller/poll`` then + computes ``(* nil 1000 60 60 24)`` — reproduced + live as ``NullPointerException at + polismath.poller/poll (poller.clj:15)``. + LOGGING_LEVEL -> :logging-level (config.clj:111, applied by + ``polismath.components.logger`` — its + ``:min-level`` defaults to ``:warn`` + (``defaults`` map, config.clj:53), which + SILENTLY SUPPRESSES every application-level + trace (poll cycles, conv-manager batch + processing, recompute completion). RUNNER + EVIDENCE (2026-07-24 live-debug task, REQUIRED + FIX #3): without this, a captured runner log + (:class:`_SubprocessRunner`'s ``log_path``) is + HikariCP connection-pool heartbeats and nothing + else — a stalled container is indistinguishable + from a healthy-but-quiet one. Defaults to + ``"info"`` (every poll cycle + conv-manager + timing line, still well short of ``:debug``'s + volume); overridable per-call. + """ + env = dict(base_env if base_env is not None else os.environ) + env["DATABASE_URL"] = _url_with_scheme(database_url, "postgres") + env["MATH_ENV"] = math_env + env["POLL_FROM_DAYS_AGO"] = _format_days_ago(poll_from_days_ago) + env["LOGGING_LEVEL"] = logging_level + return env + + +class CljContainerRunner(_SubprocessRunner): + """The REAL clj math container loop: ``clojure -M:run full`` + (``deps.edn:65-66`` ``:run`` alias -> ``-m polismath.runner``; + ``runner.clj`` subcommand table -> ``"full"`` -> ``system/full-system``, + ``system.clj:47-52`` = ``poller-system`` = base-system + vote-poller + + mod-poller, ``system.clj:29-33``). ``math/bin/run`` wraps this same + invocation in a 4h-reboot while-loop (MATH_POLLER_EQUIV_SPEC.md hazard 4) + that is irrelevant at harness timescales — we invoke the bare command. + """ + + def __init__( + self, + *, + database_url: str, + math_env: str, + poll_from_days_ago: float = 10000, + logging_level: str = "info", + base_env: Optional[dict[str, str]] = None, + log_path: Optional[Path] = None, + ): + env = build_clj_env( + database_url=database_url, + math_env=math_env, + poll_from_days_ago=poll_from_days_ago, + logging_level=logging_level, + base_env=base_env, + ) + super().__init__(["clojure", "-M:run", "full"], cwd=_MATH_ROOT, env=env, log_path=log_path) + + +def build_py_env( + *, + database_url: str, + math_env: str, + poll_from_days_ago: float = 10000, + engine_mode: str = ENGINE_MODE_LEGACY, + database_ssl_mode: str = "disable", + base_env: Optional[dict[str, str]] = None, +) -> dict[str, str]: + """Env for ``scripts/math_poller.py``, keyed to ``PollerConfig.from_env`` + (service.py:150-182): ``DATABASE_URL``, ``MATH_ENV``, + ``POLL_FROM_DAYS_AGO`` (same names as the clj side — see + :func:`build_clj_env`), plus ``POLISMATH_ENGINE_MODE`` (engine_mode.py:30 + ``ENGINE_MODE_ENV_VAR``), applied at service start via + ``apply_engine_mode`` (service.py:207-220). + + ``DATABASE_URL`` is normalized to the ``postgresql://`` scheme + (:func:`_url_with_scheme`) — the mirror-image guard of the clj side's + translation to ``postgres://`` (:func:`build_clj_env`'s docstring): + SQLAlchemy/psycopg2 reject the bare "postgres" dialect (removed in + SQLAlchemy 1.4+), so even a caller that already normalized for the clj + side must not leak that scheme here. + + ``DATABASE_SSL_MODE`` defaults to ``"disable"`` — root cause #3 + (2026-07-24 live debug): ``scripts/math_poller.py`` never loads a `.env` + file, so ``PostgresClient``'s own fallback + (``os.environ.get("DATABASE_SSL_MODE", "require")``, postgres.py:92) bites + whenever this env var isn't already present in the CALLING shell — + reproduced live as an infinite ``psycopg2.OperationalError: ... server + does not support SSL, but SSL was required`` retry loop against the + harness's local (non-SSL) Postgres target. This harness always targets a + local/OrbStack-proxied Postgres with no SSL layer (spec hazard 1) so + "disable" is the correct default; overridable for a caller that DOES + target an SSL-requiring server. + """ + env = dict(base_env if base_env is not None else os.environ) + env["DATABASE_URL"] = _url_with_scheme(database_url, "postgresql") + env["MATH_ENV"] = math_env + env["POLL_FROM_DAYS_AGO"] = _format_days_ago(poll_from_days_ago) + env[ENGINE_MODE_ENV_VAR] = engine_mode + env["DATABASE_SSL_MODE"] = database_ssl_mode + return env + + +class PyPollerRunner(_SubprocessRunner): + """The python math_poller CLI (``scripts/math_poller.py``), run-forever + mode — its only flag is ``--once`` (math_poller.py:43-48), which we do NOT + pass, so it runs until killed exactly like the clj container.""" + + def __init__( + self, + *, + database_url: str, + math_env: str, + poll_from_days_ago: float = 10000, + engine_mode: str = ENGINE_MODE_LEGACY, + database_ssl_mode: str = "disable", + base_env: Optional[dict[str, str]] = None, + log_path: Optional[Path] = None, + ): + env = build_py_env( + database_url=database_url, + math_env=math_env, + poll_from_days_ago=poll_from_days_ago, + engine_mode=engine_mode, + database_ssl_mode=database_ssl_mode, + base_env=base_env, + ) + super().__init__( + ["uv", "run", "python", "scripts/math_poller.py"], cwd=_DELPHI_ROOT, env=env, + log_path=log_path, + ) + + +# --------------------------------------------------------------------------- +# wait_for_tick (Stage B.3) — poll math_main until a caller predicate holds. +# --------------------------------------------------------------------------- +def wait_for_tick( + conn: Any, + math_env: str, + zid: int, + predicate: Callable[[dict[str, Any]], bool], + timeout: float, + *, + poll_interval: float = 0.5, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, +) -> Optional[dict[str, Any]]: + """Poll ``math_main`` for ``(zid, math_env)`` until ``predicate(row)`` is + True or ``timeout`` seconds elapse. Returns the matching row (as a plain + dict) or ``None`` on timeout. + + ``conn`` needs only ``.execute(text, params) -> Result`` with + ``Result.mappings().first()`` — a plain SQLAlchemy ``Connection`` (same + interface ``tests/poller/test_integration_postgres.py`` already uses) or a + stand-in double. ``sleep``/``now`` are injectable seams so unit tests never + actually sleep or depend on wall-clock time. + """ + deadline = now() + timeout + while True: + result = conn.execute( + sa.text("SELECT * FROM math_main WHERE zid = :zid AND math_env = :math_env"), + {"zid": zid, "math_env": math_env}, + ) + row = result.mappings().first() + if row is not None: + row_dict = dict(row) + if predicate(row_dict): + return row_dict + if now() >= deadline: + return None + sleep(poll_interval) + + +# --------------------------------------------------------------------------- +# Quirk Q19 harness-level mitigation (session 2, 2026-07-24) — wait-for- +# first-poll-cycle gate, approved under the goal's standing autonomy. +# +# ``conv_man.clj``'s ``queue-message-batch!`` has an unsynchronized +# check-then-act race: ``(if-let [...] (get @conversations zid) ...)`` then +# a BLIND ``(swap! conversations assoc zid conv-actor)`` with no conflict +# check. When the ``:votes`` AND ``:moderation`` pollers BOTH discover data +# for a brand-new zid on their very first poll tick, they can each +# independently decide "no actor yet" and each spin up their OWN +# conv-actor (confirmed live via duplicate "Running load or init" log +# lines) — whichever actor's swap! lands last wins the registry slot, and +# the OTHER actor (which may have ALREADY correctly processed an earlier +# batch) is silently orphaned, permanently losing that batch's votes with +# no self-healing (no periodic full recompute exists). Reproduced 3/3 live +# attempts once the earlier root causes (#1-#5) were fixed and out of the +# way — ledgered as quirk Q19, math team notified separately (math/src/ is +# off-limits to fix this here). +# +# This harness triggers the race with near-certainty because it seeds +# comments AND inserts batch 0's votes well before the JVM finishes +# booting (~20-30s) — so BOTH pollers see data on their FIRST-EVER cycle. +# The mitigation: delay feeding batch 0 until we've observed evidence that +# the clj container has ALREADY completed at least one ``:votes`` poll +# cycle. By construction that cycle found ZERO votes (none inserted yet), +# so it can NEVER call ``queue-message-batch!`` — meaning only ONE poller +# (moderation, discovering the pre-seeded comments) can EVER be first to +# create the actor, regardless of exact scheduling. This is a pure +# INPUT-sequencing change (when we feed data, not what we accept as a +# match) — it does not touch, weaken, or special-case any comparison logic. +# +# Signal choice: ``polismath.poller/poll`` (poller.clj:24) logs +# ``"Polling > "`` UNCONDITIONALLY on EVERY +# cycle, found rows or not — this is what makes it reliable (vs. e.g. +# waiting for a DB row, which wouldn't exist yet on a genuinely quiet +# cycle) and requires ``LOGGING_LEVEL=info`` (already the harness default, +# see :func:`build_clj_env`). +# --------------------------------------------------------------------------- +def _poll_cycle_signal_seen(log_text: str, *, message_type: str = "votes") -> bool: + """Pure predicate: has ``log_text`` (a runner log's current content) + shown evidence of at least one completed poll cycle for + ``message_type``. See the section docstring above for why this + specific, unconditionally-emitted log line is a reliable signal.""" + return f"Polling :{message_type} >" in log_text + + +def wait_for_first_poll_cycle( + log_path: Path | None, + timeout: float, + *, + message_type: str = "votes", + poll_interval: float = 0.5, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, + start_offset: int = 0, +) -> dict[str, Any]: + """Poll ``log_path``'s content (from byte ``start_offset`` onward) until + :func:`_poll_cycle_signal_seen` matches or ``timeout`` elapses. Mirrors + :func:`wait_for_tick`'s injectable-clock shape so it's testable with a + fake clock and no real sleeping. ``log_path=None`` (no runner log + captured — e.g. :class:`_SubprocessRunner` was built without one) + returns immediately as NOT observed, never waits — there is nothing to + poll. + + ``start_offset`` matters because :class:`_SubprocessRunner` opens its + log in APPEND mode (so a seam restart's post-restart output lands in + the SAME file as the pre-restart run — see its docstring). Without an + offset, a FRESH container's cold-start gate check would be satisfied + INSTANTLY by a "Polling ..." line left over from a PREVIOUS attempt + sitting earlier in the same ``--out`` directory's log file, silently + defeating the whole quirk-Q19 mitigation after the very first run — + reproduced live (2026-07-24 session 2): the mitigation's first + real-world run still hit Q19 because the gate read stale text. + Callers (:func:`run_batch_loop`) capture ``log_path``'s size + IMMEDIATELY BEFORE starting to wait and pass it as this offset, so only + genuinely NEW content (from THIS container instance) can satisfy the + gate. ``start_offset=0`` (the default) reads the whole file, unchanged + from before this parameter existed. + + A missing (not-yet-created) log file is treated as empty content, not + an error — the subprocess may not have flushed its first write yet. + + Returns a dict always carrying ``observed``/``elapsed_s``, plus + ``reason``/``message_type`` on a timeout — this is the SAME dict + :func:`run_batch_loop` records verbatim into the manifest's + ``startup_gate`` section, so a run is self-describing about whether the + gate fired, and whether it actually observed the signal in time. + """ + start = now() + if log_path is None: + return {"observed": False, "elapsed_s": 0.0, "reason": "no runner log path"} + log_path = Path(log_path) + deadline = start + timeout + while True: + text = "" + if log_path.exists(): + with open(log_path, "rb") as fh: + fh.seek(start_offset) + text = fh.read().decode(errors="replace") + if _poll_cycle_signal_seen(text, message_type=message_type): + return {"observed": True, "elapsed_s": now() - start, "message_type": message_type} + if now() >= deadline: + return { + "observed": False, "elapsed_s": now() - start, + "reason": "timeout", "message_type": message_type, + } + sleep(poll_interval) + + +# ============================================================================= +# Stage C — feeder + comparer. +# ============================================================================= +# The three data tables the feeder snapshots per (math_env, batch); the same +# three tables the spec's "compare" bullet names. Fixed constants ONLY — never +# interpolate a caller-supplied string into the SQL built from this tuple +# (:func:`fetch_math_row`) or the filesystem path built from it +# (:func:`snapshot_path`). +EQUIV_TABLES: tuple[str, ...] = ("math_main", "math_bidtopid", "math_ptptstats") + + +# --------------------------------------------------------------------------- +# Readiness predicate — pure logic (spec §1 "feed" bullet). +# --------------------------------------------------------------------------- +def blob_total_votes(blob: dict[str, Any]) -> int | None: + """Cumulative distinct ``(pid, tid)`` rated-cell count carried by a + math_main blob's ``user-vote-counts`` key (sum of its per-pid values). + + ``user-vote-counts`` is one of the 23 prep-main keys BOTH engines emit + (crosslang.py:44-50 ``PREP_MAIN_KEYS``); Python builds it via + ``_compute_user_vote_counts`` (conversation.py:2013-2116) from + ``raw_rating_mat`` over EVERY participant in ``rating_mat.index`` — summed + across all pids, this is exactly the number of distinct ``(pid, tid)`` + pairs with a latest vote (a revote overwrites the SAME rating-matrix cell + rather than adding a new one, so revotes never inflate the total — the + same invariant :func:`expected_cumulative_vote_count` relies on below). + Verified against the committed vw recording + (``real_data/.local/replays/vw/uniform8-clojure-legacy/py/step-000.json``): + ``sum(blob["user-vote-counts"].values()) == 585 == + blob["vote_stats"]["n_votes"] == batch_size`` at step 0. + + ``votes-base`` was deliberately NOT used here despite also being a + prep-main key: in ``clojure-legacy`` engine mode (the mode this harness + always runs — ``build_py_env``'s default) Python's ``votes-base`` is the + Clojure-exact BUCKET form (``_compute_votes_base_buckets``, + conversation.py:1743-1791), whose own docstring warns "the aggregation + domain is each bucket's member pids only — votes from unclustered + participants never appear" (FP-81fda13ef6). Summing its ``'S'`` vectors + therefore UNDERCOUNTS whenever a participant hasn't been assigned to a + base cluster yet (measured 565 vs the true 585 on the same vw step-000 + blob above) — exactly the kind of transient state a readiness predicate + would otherwise stall on forever. + + Falls back to the Python-only ``vote_stats.n_votes`` (conversation.py: + 526-561 ``_compute_vote_stats``) when ``user-vote-counts`` is + absent/malformed — the Clojure side never populates ``vote_stats``, so + this fallback only ever helps when inspecting a lone Python snapshot in + isolation. + + Returns ``None`` (never a guessed ``0``) when neither key is present in + the expected shape — callers must treat that as "not ready to judge yet", + not as "zero votes seen". + """ + if not isinstance(blob, dict): + return None + uvc = blob.get("user-vote-counts") + if isinstance(uvc, dict): + try: + return sum(uvc.values()) + except TypeError: + return None + vs = blob.get("vote_stats") + if isinstance(vs, dict) and "n_votes" in vs: + return vs["n_votes"] + return None + + +def expected_cumulative_vote_count(dataset: ReplayDataset, upto_slot: int) -> int: + """The value :func:`blob_total_votes` should reach once a math_main row + reflects every vote in ``dataset.votes[:upto_slot]``: the count of + DISTINCT ``(pid, tid)`` pairs in that prefix. + + ``VoteEvent.is_revote`` already flags exactly this (types.py:44 — "a + later occurrence of an already-seen (pid, tid) pair in sorted order", + computed incrementally over a GROWING prefix in + :meth:`ReplayDataset.build`) — so counting non-revotes within any prefix + of the sorted stream gives that prefix's distinct-pair count directly. + Pure/unit-testable: no I/O, no dataset mutation. + """ + return sum(1 for v in dataset.votes[:upto_slot] if not v.is_revote) + + +def make_batch_ready_predicate( + *, min_last_vote_ts: int, min_vote_count: int, +) -> Callable[[dict[str, Any] | None], bool]: + """Pure predicate factory (spec §1 "feed" bullet — "wait until EACH + math_env's math_main row for the zid reflects the batch"). + + The returned predicate matches a math_main ROW (the shape + :func:`wait_for_tick` / :func:`fetch_math_row` return — a dict with a + ``data`` key holding the blob, plus the persisted ``last_vote_timestamp`` + column) once BOTH: + + - the blob's ``lastVoteTimestamp`` is ``>= min_last_vote_ts`` (the + batch's cut time) — falls back to the persisted + ``last_vote_timestamp`` column if the blob is missing the key + (defensive; both are written from the same ``conv.last_updated`` + value — poller/math_writer.py:101-112), and + - :func:`blob_total_votes` has caught up to ``>= min_vote_count`` — + "vote count advanced". + + ``None`` (a cold zid — no row yet) never satisfies the predicate. + """ + + def predicate(row: dict[str, Any] | None) -> bool: + if row is None: + return False + blob = row.get("data") + if not isinstance(blob, dict): + return False + lvt = blob.get("lastVoteTimestamp") + if lvt is None: + lvt = row.get("last_vote_timestamp") + if lvt is None or lvt < min_last_vote_ts: + return False + n_votes = blob_total_votes(blob) + if n_votes is None or n_votes < min_vote_count: + return False + return True + + return predicate + + +# --------------------------------------------------------------------------- +# Batch slicing — pure logic (spec §1 "feed" bullet: "insert vote batch k"). +# --------------------------------------------------------------------------- +def batch_slices(cuts: Sequence[int]) -> list[tuple[int, int]]: + """Partition ``cuts`` into half-open, 1-based ``(prev, cut]`` batches. + + Mirrors :func:`polismath.replay.schedule.slice_schedule`'s own + ``prev = 0; for cut in slots: batch = votes[prev:cut]; prev = cut`` loop + (schedule.py:208-230), WITHOUT materializing vote/mod payloads — the live + feeder slices the actual dataset itself at insert time (see + :func:`insert_votes`), so this only needs to produce the ``(prev, cut)`` + slot pairs. + + ``cuts`` must already be resolved: strictly increasing, 1-based absolute + vote-count slots (e.g. :func:`polismath.replay.schedule.resolve_cut_slots`'s + output, or a bare preset's slot list — ``ScheduleSpec``/schedule-file + resolution is the CALLER's job, kept out of this pure function). The + first batch always starts at slot 0 (spec's "first batch from slot 0" + edge case); this function does not know the dataset's total vote count + ``n``, so a caller wanting the LAST batch to run "to n" (spec's other + edge case) must include ``n`` as ``cuts[-1]`` themselves — exactly like + schedule.py's ``"end"`` sentinel. + + Raises ``ValueError`` if ``cuts`` is not strictly increasing. + """ + if not cuts: + return [] + slices: list[tuple[int, int]] = [] + prev = 0 + for cut in cuts: + if cut <= prev: + raise ValueError( + f"cuts must be strictly increasing 1-based slots; got {cut!r} " + f"after prev={prev}" + ) + slices.append((prev, cut)) + prev = cut + return slices + + +def snap_cuts_past_timestamp_ties(dataset: ReplayDataset, cuts: Sequence[int]) -> list[int]: + """Adjust each 1-based cut slot FORWARD (never backward) so it never + falls strictly inside a run of votes sharing the SAME ``created`` + millisecond timestamp. + + ROOT CAUSE #4 (2026-07-24 live-debug task): BOTH pollers watermark with + STRICT ``created > ts`` — ``postgres/poll`` (postgres.clj:132-145, + global vote poll) and ``PostgresClient.poll_votes_since`` + (postgres.py:534-557) are byte-identical on this point (also confirmed + directly against the SQL text). If a batch cut falls in the MIDDLE of a + run of votes sharing the exact same ``created`` value, the votes AFTER + the cut with that timestamp become PERMANENTLY unreachable for BOTH + engines the instant the FIRST batch's poll advances its watermark to + that exact value — verified live against a real Postgres + real + ``clojure -M:run full``: with vw's committed uniform-8 schedule (cut=585 + at slot 585), votes at 0-based indices 585/586/587 shared + ``t_ms=1732028794000`` with index 584 (the cut boundary vote); clj-ref's + own ``user-vote-counts`` came up short by EXACTLY 1 for EXACTLY the 3 + pids owning those 3 votes (pid=2/tid=43, pid=17/tid=11, pid=22/tid=22), + stalling the feeder's readiness predicate forever (its target vote count + assumed every vote up to the cut was reachable). This is NOT a + clj-vs-py divergence — both engines drop the exact same votes, + identically, by construction (same SQL, same watermark) — it is a + structurally unreachable target the HARNESS's own batch-cut choice + created; fixing it here (rather than loosening the readiness predicate + or the acceptance bar) is the only change that doesn't touch what's + being measured. + + A cut equal to ``len(dataset.votes)`` (the dataset's own total — the + "final batch runs to n" convention :func:`batch_slices` documents) is + NEVER adjusted: there is no "next" vote to tie against, and growing past + the dataset would be nonsensical. + """ + votes = dataset.votes + n = len(votes) + adjusted: list[int] = [] + for cut in cuts: + c = cut + while 0 < c < n and votes[c - 1].t_ms == votes[c].t_ms: + c += 1 + adjusted.append(c) + return adjusted + + +# --------------------------------------------------------------------------- +# Snapshot store — mirrors store.py's per-(dataset, schedule) directory +# convention (design §7 / store.py:1-23), keyed here by (math_env, batch). +# --------------------------------------------------------------------------- +def snapshot_dir(out_dir: str | Path, math_env: str, batch_index: int) -> Path: + """``//batch-/`` — never created here (lazy, like + :func:`polismath.replay.store.write_recording`); see :func:`write_snapshot`.""" + math_env = _safe_path_component(math_env, label="math_env") + return Path(out_dir) / math_env / f"batch-{batch_index:03d}" + + +def snapshot_path(out_dir: str | Path, math_env: str, batch_index: int, table: str) -> Path: + if table not in EQUIV_TABLES: + raise ValueError(f"unknown equiv table {table!r}; expected one of {EQUIV_TABLES}") + return snapshot_dir(out_dir, math_env, batch_index) / f"{table}.json" + + +def write_snapshot( + out_dir: str | Path, math_env: str, batch_index: int, table: str, row: dict[str, Any], +) -> Path: + """Write one DB row (as returned by :func:`wait_for_tick` / + :func:`fetch_math_row`) to its snapshot path, creating parent directories + lazily. Returns the path written.""" + path = snapshot_path(out_dir, math_env, batch_index, table) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as fh: + json.dump(row, fh, indent=2, sort_keys=True, default=str) + return path + + +def load_snapshot( + out_dir: str | Path, math_env: str, batch_index: int, table: str, +) -> dict[str, Any] | None: + """The inverse of :func:`write_snapshot`; ``None`` when the snapshot was + never written (e.g. the batch never became ready for that env).""" + path = snapshot_path(out_dir, math_env, batch_index, table) + if not path.exists(): + return None + return json.loads(path.read_text()) + + +def discover_batches(out_dir: str | Path, math_env: str) -> list[int]: + """Batch indices actually snapshotted for ``math_env`` (its + ``batch-NNN/`` subdirectories under ``out_dir``), sorted ascending. + Empty when the env has no directory at all (nothing snapshotted yet, or + an unrecognized/misspelled env name — never raises for that case, unlike + :func:`snapshot_path`, since "no batches yet" is a normal state for the + comparer to report on, not a caller error).""" + math_env = _safe_path_component(math_env, label="math_env") + d = Path(out_dir) / math_env + if not d.is_dir(): + return [] + indices: list[int] = [] + for p in sorted(d.glob("batch-*")): + if not p.is_dir(): + continue + try: + indices.append(int(p.name.split("-", 1)[1])) + except (IndexError, ValueError): + continue + return sorted(indices) + + +def write_manifest(out_dir: str | Path, manifest: dict[str, Any]) -> Path: + """Write the feeder's per-batch bookkeeping (expected vote counts, cut + times, …) alongside the snapshots — the comparer's watermark check reads + this back instead of re-loading the dataset (:func:`load_manifest`).""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / "manifest.json" + with open(path, "w") as fh: + json.dump(manifest, fh, indent=2, sort_keys=True, default=str) + return path + + +def load_manifest(out_dir: str | Path) -> dict[str, Any] | None: + path = Path(out_dir) / "manifest.json" + if not path.exists(): + return None + return json.loads(path.read_text()) + + +def fetch_math_row(conn: Any, table: str, zid: int, math_env: str) -> dict[str, Any] | None: + """``SELECT * FROM WHERE zid=:zid AND math_env=:math_env`` — same + connection interface as :func:`wait_for_tick` (``.execute(text, params)`` + -> ``Result.mappings().first()``). ``table`` MUST be one of + :data:`EQUIV_TABLES` — those are the only values ever interpolated into + the SQL text (never a caller-supplied string).""" + if table not in EQUIV_TABLES: + raise ValueError(f"unknown equiv table {table!r}; expected one of {EQUIV_TABLES}") + result = conn.execute( + sa.text(f"SELECT * FROM {table} WHERE zid = :zid AND math_env = :math_env"), + {"zid": zid, "math_env": math_env}, + ) + row = result.mappings().first() + return dict(row) if row is not None else None + + +class PollerEquivStreamError(RuntimeError): + """Raised by :func:`run_batch_loop` when a batch's readiness predicate + times out for some ``math_env`` — REQUIRED FIX #2 (2026-07-24 live-debug + task) "FAIL-FAST FEEDER": aborts the WHOLE stream immediately rather than + silently recording ``{"ready": False}`` and continuing to feed more vote + batches into a runner that will never catch up (or has already crashed). + The pre-fix behavior is exactly what produced a vacuous PASS in the + 2026-07-24 live run: 8 batches fed, 0 ever became ready, nothing ever + surfaced the failure loudly. The message always names the offending env, + batch index, elapsed wait, and the LAST OBSERVED math_main state for that + env (or the literal phrase "no row ever appeared" if the zid never got a + single tick) — plus, when the failing runner has a captured log + (:attr:`_SubprocessRunner.log_path`), the last ~30 lines of it.""" + + +_RUNNER_LOG_TAIL_LINES = 30 + + +def _tail_lines(path: Path | None, n: int = _RUNNER_LOG_TAIL_LINES) -> str: + """Last ``n`` lines of ``path`` (or a placeholder when unavailable) — + the diagnostic body :class:`PollerEquivStreamError` embeds so a stream + abort is debuggable from the exception message alone, no separate log + hunt required.""" + if path is None: + return "(no runner log captured for this env)" + path = Path(path) + if not path.exists(): + return f"(runner log {path} does not exist)" + try: + lines = path.read_text(errors="replace").splitlines() + except OSError as exc: + return f"(could not read runner log {path}: {exc})" + if not lines: + return f"(runner log {path} is empty)" + return "\n".join(lines[-n:]) + + +def _describe_math_main_row(row: dict[str, Any] | None) -> str: + """Human-readable summary of the LAST observed math_main row for a + timed-out env (or the literal "no row ever appeared" when ``row`` is + ``None`` — a zid that never got a single tick from that env).""" + if row is None: + return "no row ever appeared" + blob = row.get("data") + n_votes = blob_total_votes(blob) if isinstance(blob, dict) else None + return ( + f"caching_tick={row.get('caching_tick')} math_tick={row.get('math_tick')} " + f"last_vote_timestamp={row.get('last_vote_timestamp')} blob_total_votes={n_votes}" + ) + + +# --------------------------------------------------------------------------- +# Feeder — the LIVE loop (spec §1 "feed"/"seam" bullets). Thin by +# construction: every decision above this point is a pure function; this +# loop only sequences I/O calls to them. +# --------------------------------------------------------------------------- +def run_batch_loop( + conn: Any, + dataset: ReplayDataset, + cuts: Sequence[int], + math_envs: Sequence[str], + runners: dict[str, Any], + *, + out_dir: str | Path, + zid: int = DEFAULT_ZID, + seam_after: int | None = None, + restart_envs_at_seam: Sequence[str] | None = None, + restart_builders: dict[str, Callable[[], Any]] | None = None, + wait_timeout: float = 120.0, + poll_interval: float = 0.5, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, + insert_fn: Callable[[Any, ReplayDataset, int, int, int], int] = insert_votes, + mod_insert_fn: Callable[[Any, ReplayDataset, int | None, int, int], int] = insert_mod_events, + startup_gate_envs: Sequence[str] | None = None, + startup_gate_timeout: float = 60.0, + startup_gate_message_type: str = "votes", +) -> dict[str, Any]: + """Insert vote batches one at a time, waiting for each ``math_env`` to + reflect the batch before snapshotting its three tables and moving on; + optionally restart runners at ``seam_after``. + + Every decision (readiness, batch bounds, snapshot naming) is delegated to + the pure functions above, so this loop is itself trivially exercised with + fake ``conn``/``runners`` doubles and a fake ``insert_fn`` — see + ``tests/replay_harness/test_poller_equiv_compare.py::TestRunBatchLoop`` + (no real DB, no real subprocess). + + ``runners`` is mutated in place: a seam restart replaces the entry for a + restarted env with the freshly-built runner (mirrors + :class:`_SubprocessRunner`'s kill-then-replace lifecycle — the OLD + runner object is never reused after ``kill()``). + + ``mod_insert_fn`` (session 2, 2026-07-24) applies ``dataset.mod_events`` + alongside each batch's votes, using the SAME ``(prev_time_ms, + cut_time_ms]`` time-window :func:`insert_mod_events` documents (mirrors + ``slice_schedule``'s moderation semantics) — defaults to the real + :func:`insert_mod_events`, which is a no-op (never touches ``conn``) for + any dataset with zero mod_events, so this is safe for every existing + caller/test unchanged. + + ``startup_gate_envs`` (session 2, 2026-07-24 — quirk Q19 harness-level + mitigation, approved under the goal's standing autonomy) names the + envs (if any) whose runner must show evidence of AT LEAST ONE completed + poll cycle (:func:`wait_for_first_poll_cycle`) BEFORE batch 0 is ever + inserted — see that function's module-level docstring for the full + rationale. ``None`` (the default) disables the gate entirely, keeping + the manifest's ``startup_gate`` section a fixed ``{"enabled": False, + "envs": {}}`` for every caller that doesn't opt in. A gate that never + observes its signal within ``startup_gate_timeout`` aborts the WHOLE + stream the same way a batch-readiness timeout does (persists the + partial manifest, raises :class:`PollerEquivStreamError`) — proceeding + into batch 0 without the safety net the gate exists for would be worse + than not gating at all. + + Returns (and also writes, via :func:`write_manifest`) the batch manifest. + """ + out_dir = Path(out_dir) + # ROOT CAUSE #4 (2026-07-24 live-debug task) — see + # :func:`snap_cuts_past_timestamp_ties`'s docstring: a raw cut that falls + # inside a same-millisecond vote cluster makes votes structurally + # unreachable for BOTH pollers, not just harder to reach — snap BEFORE + # slicing so the readiness predicate is never given an impossible target. + effective_cuts = snap_cuts_past_timestamp_ties(dataset, cuts) + slices = batch_slices(effective_cuts) + manifest: dict[str, Any] = { + "zid": zid, "math_envs": list(math_envs), + "cuts_requested": list(cuts), "cuts_effective": effective_cuts, + "startup_gate": {"enabled": bool(startup_gate_envs), "envs": {}}, + "batches": [], + } + + # Quirk Q19 mitigation (session 2, 2026-07-24) — see this function's + # docstring and wait_for_first_poll_cycle's module-level docstring: + # block feeding batch 0 until each gated env's runner has shown evidence + # of a completed poll cycle, so the conv_man.clj actor-creation race can + # never trigger from the votes side (a cycle observed BEFORE any votes + # exist is guaranteed to have found none). + if startup_gate_envs: + gate_envs_report: dict[str, Any] = {} + for env in startup_gate_envs: + runner = runners.get(env) + log_path = getattr(runner, "log_path", None) + # Capture the CURRENT log size BEFORE waiting — see + # wait_for_first_poll_cycle's start_offset docstring: the log is + # append-mode (seam-restart continuity), so without this a + # second-or-later run in the same --out dir would have its gate + # satisfied instantly by a PRIOR attempt's stale "Polling..." + # line, silently defeating the mitigation (found live). + start_offset = 0 + if log_path is not None and Path(log_path).exists(): + start_offset = Path(log_path).stat().st_size + result = wait_for_first_poll_cycle( + log_path, startup_gate_timeout, message_type=startup_gate_message_type, + poll_interval=poll_interval, sleep=sleep, now=now, + start_offset=start_offset, + ) + gate_envs_report[env] = result + if not result["observed"]: + manifest["startup_gate"]["envs"] = gate_envs_report + write_manifest(out_dir, manifest) + raise PollerEquivStreamError( + f"poller-equiv feeder ABORT: startup gate for env={env!r} never observed " + f"a {startup_gate_message_type!r} poll cycle within " + f"{startup_gate_timeout}s (quirk Q19 mitigation — see " + f"wait_for_first_poll_cycle's docstring).\n" + f"--- last {_RUNNER_LOG_TAIL_LINES} lines of " + f"{log_path if log_path is not None else '(no runner log)'} ---\n" + f"{_tail_lines(log_path)}" + ) + manifest["startup_gate"]["envs"] = gate_envs_report + + prev_mod_time_ms: int | None = None + for i, (prev, cut) in enumerate(slices): + n_inserted = insert_fn(conn, dataset, prev, cut, zid) + cut_time_ms = dataset.votes[cut - 1].t_ms + n_mod_applied = mod_insert_fn(conn, dataset, prev_mod_time_ms, cut_time_ms, zid) + prev_mod_time_ms = cut_time_ms + expected_votes = expected_cumulative_vote_count(dataset, cut) + + batch_record: dict[str, Any] = { + "index": i, + "prev_slot": prev, + "cut_slot": cut, + "n_inserted": n_inserted, + "n_mod_events_applied": n_mod_applied, + "cut_time_ms": cut_time_ms, + "expected_vote_count": expected_votes, + "envs": {}, + } + for env in math_envs: + predicate = make_batch_ready_predicate( + min_last_vote_ts=cut_time_ms, min_vote_count=expected_votes, + ) + row = wait_for_tick( + conn, env, zid, predicate, wait_timeout, + poll_interval=poll_interval, sleep=sleep, now=now, + ) + if row is None: + last_row = fetch_math_row(conn, "math_main", zid, env) + last_state = _describe_math_main_row(last_row) + batch_record["envs"][env] = { + "ready": False, "timeout_s": wait_timeout, "last_state": last_state, + } + # Persist whatever we have BEFORE raising — a fail-fast abort + # must still leave a post-mortem-able manifest on disk (spec's + # "keep the DB alive... for post-mortem" intent, extended to + # the manifest the comparer reads). + manifest["batches"].append(batch_record) + write_manifest(out_dir, manifest) + + runner = runners.get(env) + log_path = getattr(runner, "log_path", None) + raise PollerEquivStreamError( + f"poller-equiv feeder ABORT: env={env!r} batch={i} " + f"(prev_slot={prev}, cut_slot={cut}) never became ready within " + f"{wait_timeout}s (expected_vote_count={expected_votes}, " + f"cut_time_ms={cut_time_ms}).\n" + f"Last observed math_main state for {env!r}: {last_state}\n" + f"--- last {_RUNNER_LOG_TAIL_LINES} lines of " + f"{log_path if log_path is not None else '(no runner log)'} ---\n" + f"{_tail_lines(log_path)}" + ) + + write_snapshot(out_dir, env, i, "math_main", row) + snap_ok = {"math_main": True} + for table in ("math_bidtopid", "math_ptptstats"): + trow = fetch_math_row(conn, table, zid, env) + if trow is not None: + write_snapshot(out_dir, env, i, table, trow) + snap_ok[table] = trow is not None + batch_record["envs"][env] = {"ready": True, "snapshots": snap_ok} + + manifest["batches"].append(batch_record) + + if seam_after is not None and i == seam_after: + for env in (restart_envs_at_seam or ()): + runner = runners.get(env) + if runner is not None: + runner.kill() + if restart_builders and env in restart_builders: + new_runner = restart_builders[env]() + new_runner.start() + runners[env] = new_runner + + write_manifest(out_dir, manifest) + return manifest + + +def run_equiv_stream( + admin_url: str, + dataset_slug: str, + cuts: Sequence[int], + *, + out_dir: str | Path, + seam_after: int | None = None, + math_envs: tuple[str, str] = ("clj-ref", "py-shadow"), + restart_clj_at_seam: bool = False, + dbname: str = DEFAULT_DBNAME, + zid: int = DEFAULT_ZID, + poll_from_days_ago: float = 10000, + engine_mode: str = ENGINE_MODE_LEGACY, + wait_timeout: float = 120.0, + poll_interval: float = 0.5, + engine_factory: Callable[[str], Any] | None = None, + runner_builders: dict[str, Callable[[], Any]] | None = None, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, + wait_for_clj_poll_cycle: bool = True, + poll_cycle_gate_timeout: float = 60.0, +) -> dict[str, Any]: + """Stage C top-level orchestration SKELETON (spec §1 "feed"/"seam" + bullets): seed the throwaway DB, start both runners, then delegate every + per-batch decision to :func:`run_batch_loop`. + + ``wait_for_clj_poll_cycle`` (session 2, 2026-07-24, default True) wires + :func:`run_batch_loop`'s ``startup_gate_envs`` to ``[clj_env]`` — quirk + Q19's harness-level mitigation, see that function's docstring. + + ``math_envs`` is ``(clj_env_name, py_env_name)`` — the CLJ env is always + first, matching the spec's own default ``("clj-ref", "py-shadow")``; this + is a documented positional convention, not inferred from the strings. + ``restart_clj_at_seam`` mirrors the spec's "optionally the clj runner — + parameter" bullet: the PY runner is ALWAYS restarted at the seam, the clj + one only when this is set. + + ``runner_builders`` lets a test (or an alternate invocation) replace + ``{"clj": ..., "py": ...}`` runner factories wholesale — the DEFAULT + builders construct the real :class:`CljContainerRunner` / + :class:`PyPollerRunner` subprocess wrappers. ``engine_factory`` defaults + to ``sqlalchemy.create_engine`` — override with a fake in tests that + never touch a real Postgres. + + NOT exercised by the default test suite (needs a real Postgres AND real + ``clojure``/``uv run python`` subprocesses running for the duration of the + stream). Every decision this function makes is either delegated to + :func:`run_batch_loop` (itself fully unit-tested with fakes) or is thin + setup/teardown glue around it. + """ + clj_env, py_env = math_envs + ds = real_data.load_export_votes(dataset_slug) + + target_url = create_equiv_db(admin_url, dbname=dbname) + engine = (engine_factory or (lambda url: sa.create_engine(url, isolation_level="AUTOCOMMIT")))( + target_url + ) + + # REQUIRED FIX #3 (2026-07-24 live-debug task) "RUNNER EVIDENCE": each + # runner's stdout+stderr land in /.runner.log — append mode + # (see :class:`_SubprocessRunner`) means a seam restart's post-restart + # output lands in the SAME file as its pre-restart run, so the whole + # process lifetime is one file. compare_snapshots()'s consumers / + # PollerEquivStreamError's message both read this back. + out_dir_path = Path(out_dir) + default_builders: dict[str, Callable[[], Any]] = { + "clj": lambda: CljContainerRunner( + database_url=target_url, math_env=clj_env, poll_from_days_ago=poll_from_days_ago, + log_path=out_dir_path / f"{clj_env}.runner.log", + ), + "py": lambda: PyPollerRunner( + database_url=target_url, math_env=py_env, poll_from_days_ago=poll_from_days_ago, + engine_mode=engine_mode, log_path=out_dir_path / f"{py_env}.runner.log", + ), + } + builders = dict(default_builders) + if runner_builders: + builders.update(runner_builders) + + runners: dict[str, Any] = {} + try: + with engine.connect() as seed_conn: + seed_conversation(seed_conn, ds, zid=zid) + + runners = {clj_env: builders["clj"](), py_env: builders["py"]()} + for r in runners.values(): + r.start() + + restart_envs = [py_env] + ([clj_env] if restart_clj_at_seam else []) + restart_builders = {py_env: builders["py"]} + if restart_clj_at_seam: + restart_builders[clj_env] = builders["clj"] + + with engine.connect() as conn: + manifest = run_batch_loop( + conn, ds, cuts, [clj_env, py_env], runners, out_dir=out_dir, zid=zid, + seam_after=seam_after, restart_envs_at_seam=restart_envs, + restart_builders=restart_builders, wait_timeout=wait_timeout, + poll_interval=poll_interval, sleep=sleep, now=now, + startup_gate_envs=[clj_env] if wait_for_clj_poll_cycle else None, + startup_gate_timeout=poll_cycle_gate_timeout, + ) + finally: + for r in runners.values(): + r.kill() + engine.dispose() + + return manifest + + +# --------------------------------------------------------------------------- +# Comparer (spec §1 "compare" bullet). +# --------------------------------------------------------------------------- +def strictly_increasing(values: Sequence[Any]) -> dict[str, Any]: + """Pure monotonicity check for a per-batch ``caching_tick`` / ``math_tick`` + sequence (spec §1 compare bullets: "caching_tick strictly increasing per + env" / "math_ticks == number of completed recomputes per env"). + + The feeder gates each batch's insert on the PREVIOUS batch's readiness + (:func:`run_batch_loop`), so every observed tick is credited to exactly + one batch by construction — a strictly-increasing per-batch sequence is + the observable signature of "every batch got exactly one recompute; none + were skipped, none were silently merged away". ``None`` entries (a + missing snapshot) always count as a violation, never silently skipped. + """ + violations: list[dict[str, Any]] = [] + for i in range(1, len(values)): + a, b = values[i - 1], values[i] + if a is None or b is None or not (b > a): + violations.append({"index": i, "prev": a, "next": b}) + return { + "values": list(values), + "strictly_increasing": len(violations) == 0, + "violations": violations, + } + + +def _normalize_bidtopid(data: dict[str, Any]) -> dict[str, Any]: + """Normalize the ONE documented cross-engine representational difference + before an EXACT equality check: Python pids are strings, Clojure pids are + ints (``polismath/poller/__init__.py``'s "bidToPid shape" docstring / + ``math_writer.py:49-53`` — the TS server ``parseInt()``s them either way, + so this is harmless). Casts every member pid to ``str`` and SORTS each + group (membership is a set, not an ordered list). The outer bid ORDER + (i.e. base-cluster id order) is left untouched — that ordering IS + meaningful, positionally aligned to ``math_main.base-clusters.id`` + (``derive_bidtopid``'s docstring).""" + d = dict(data) + bid = d.get("bidToPid") + if isinstance(bid, list): + d["bidToPid"] = [ + sorted(str(pid) for pid in group) if isinstance(group, list) else group + for group in bid + ] + return d + + +def compare_bidtopid(a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any]: + """EXACT equality (spec §1 "compare" bullet: "math_bidtopid.data → EXACT + equality") modulo the pid int/str normalization above.""" + na, nb = _normalize_bidtopid(a), _normalize_bidtopid(b) + return {"match": na == nb, "a": na, "b": nb} + + +def _ptptstats_comparer(**kwargs: Any) -> StepComparer: + """``math_ptptstats.data`` = ``{"zid", "ptptstats", "lastVoteTimestamp"}`` + (``math_writer.py`` ``derive_ptptstats``) — a flat ENVELOPE both engines + emit directly, with no kebab/snake key mismatch to project away at the + envelope level (unlike math_main) — so a plain :class:`StepComparer` + compares the envelope correctly as-is, no acceptance-projection needed. + + CORRECTION (2026-07-24, live evidence — poller-equivalence harness full + vw run, real_data/.local/replays/poller_equiv/vw/main/): this docstring + used to ALSO claim ``ptptstats``' inner VALUE was "a flat envelope both + engines emit directly" with no shape difference — that was FALSE and is + the OPPOSITE of "no kebab/snake key mismatch to project away": the live + clj-ref row's ``ptptstats`` was a COLUMNAR dict (``{pid: [...], gid: + [...], n-votes: [...], centricness: [...], coreness: [...], extremeness: + [...]}``, math/repness.clj:383-413's shape via conv_man.clj's + ``columnize``) while the pre-fix py-shadow row was an entirely different, + ROW-wise, vote-correlation-based structure (``Conversation. + participant_info`` — n_agree/n_disagree/n_pass/group_correlations, a + Python-only statistic, not a shape variant of Clojure's). Fixed at the + SOURCE (``derive_ptptstats``/``math_writer.py`` now computes the same + geometric centricness/coreness/extremeness Clojure does, columnized + identically) rather than here — a comparer-side shape reconciliation + would have papered over a real py-poller correctness bug (production + consumers read the clj shape). This comparer needs no shape-projection + logic itself; it's a plain structural+tolerant compare same as any other + table, now that both sides actually agree on what they're emitting. + + Widens the tolerant-stat-key set with ``"ptptstats"`` purely for + REPORTING (so its nested per-pid float divergences classify as + 'tolerant' rather than 'exact' in the verdict) — the underlying + ``ConversationComparer`` already applies numeric tolerance to every float + leaf regardless of this label (stepcompare.py:19-23), so this does not + change pass/fail, only how a failure is described. Structural + tolerant + per spec §1's math_ptptstats bullet. + + ``**kwargs`` forwards to :class:`StepComparer` (e.g. Stage D's + :func:`_zero_tolerance_comparer` overrides ``abs_tolerance``/ + ``rel_tolerance``/``outlier_fraction`` to measure self-jitter) — every + existing no-arg call site is unaffected (same default as before).""" + return StepComparer(tolerant_stat_keys=DEFAULT_TOLERANT_STAT_KEYS | {"ptptstats"}, **kwargs) + + +def compare_batch( + out_dir: str | Path, + batch_index: int, + math_envs: tuple[str, str], + *, + math_main_comparer: StepComparer | None = None, + ptptstats_comparer: StepComparer | None = None, + envelope: dict[str, float] | None = None, +) -> dict[str, Any]: + """Per-batch, per-table verdict (spec §1 "compare" bullet). Reads ONLY + from disk (the snapshots + manifest :func:`run_batch_loop` wrote) — no DB, + no subprocess; fully exercisable against canned snapshot fixtures. + + ``math_main`` uses the SAME acceptance surface as certify: subgroup-* + excluded per Q7, structural identity + declared float tolerances + (``certify.py:100-113`` ``project_acceptance``, ``:115-126`` + ``_acceptance_projecting_comparer`` — the very function imported and used + here). We deliberately skip certify's hash-first cache + (``certify.py:609-664`` ``compare_recording_pair``) — that's a + performance optimization for batteries with many (dataset, schedule) + pairs; irrelevant at this scale (one comparer call per batch, run once). + + ``math_bidtopid`` is EXACT (:func:`compare_bidtopid`). ``math_ptptstats`` + is structural + tolerant (:func:`_ptptstats_comparer`). The math_main + table additionally carries a ``watermark`` verdict: both envs' + :func:`blob_total_votes` must equal this batch's manifest-recorded + ``expected_vote_count`` EXACTLY (spec: "no double-processing") — read + from :func:`load_manifest` rather than re-deriving from the dataset, so + the comparer never needs the dataset/CSV at hand, only the store. + + ``envelope`` (Stage D item 2, spec §2) is an OPTIONAL ``{path_pattern: + max_delta}`` mapping (:func:`compute_self_jitter_envelope`'s output). When + given, float ("tolerant"-family) divergences on ``math_main``/ + ``math_ptptstats`` within :func:`envelope_threshold` are reclassified into + a THIRD family, ``within_envelope`` — accepted for ``match`` but NEVER + silently dropped (``n_within_envelope`` is always reported alongside + ``n_divergences``, see :func:`_apply_envelope_to_step`). Structural + ("exact"-family) divergences are NEVER excused, envelope or not. + ``envelope=None`` (the default) leaves ``math_main``/``math_ptptstats`` + verdicts BYTE-IDENTICAL to the pre-Stage-D shape — no ``within_envelope`` + family, no ``n_within_envelope`` key. ``math_bidtopid`` is pure-int EXACT + equality (no float leaves) and is never envelope-adjusted. + """ + env_a, env_b = math_envs + tables: dict[str, Any] = {} + watermark: dict[str, Any] | None = None + + main_a = load_snapshot(out_dir, env_a, batch_index, "math_main") + main_b = load_snapshot(out_dir, env_b, batch_index, "math_main") + if main_a is None or main_b is None: + tables["math_main"] = { + "match": False, + "reason": "missing-snapshot", + "missing": [e for e, r in ((env_a, main_a), (env_b, main_b)) if r is None], + } + else: + cmp = math_main_comparer or _acceptance_projecting_comparer() + step = _apply_envelope_to_step( + cmp.compare_step(main_a["data"], main_b["data"], batch_index), envelope, "math_main", + ) + tables["math_main"] = { + "match": step["match"], + "n_divergences": step["n_divergences"], + "families": step["families"], + } + if envelope is not None: + tables["math_main"]["n_within_envelope"] = step.get("n_within_envelope", 0) + manifest = load_manifest(out_dir) + expected = None + if manifest is not None and batch_index < len(manifest.get("batches", [])): + expected = manifest["batches"][batch_index].get("expected_vote_count") + obs_a = blob_total_votes(main_a["data"]) + obs_b = blob_total_votes(main_b["data"]) + watermark = { + "expected": expected, + env_a: obs_a, + env_b: obs_b, + "ok": expected is not None and obs_a == expected and obs_b == expected, + } + + bid_a = load_snapshot(out_dir, env_a, batch_index, "math_bidtopid") + bid_b = load_snapshot(out_dir, env_b, batch_index, "math_bidtopid") + if bid_a is None or bid_b is None: + tables["math_bidtopid"] = { + "match": False, + "reason": "missing-snapshot", + "missing": [e for e, r in ((env_a, bid_a), (env_b, bid_b)) if r is None], + } + else: + tables["math_bidtopid"] = compare_bidtopid(bid_a["data"], bid_b["data"]) + + pt_a = load_snapshot(out_dir, env_a, batch_index, "math_ptptstats") + pt_b = load_snapshot(out_dir, env_b, batch_index, "math_ptptstats") + if pt_a is None or pt_b is None: + tables["math_ptptstats"] = { + "match": False, + "reason": "missing-snapshot", + "missing": [e for e, r in ((env_a, pt_a), (env_b, pt_b)) if r is None], + } + else: + cmp2 = ptptstats_comparer or _ptptstats_comparer() + step2 = _apply_envelope_to_step( + cmp2.compare_step(pt_a["data"], pt_b["data"], batch_index), envelope, "math_ptptstats", + ) + tables["math_ptptstats"] = { + "match": step2["match"], + "n_divergences": step2["n_divergences"], + "families": step2["families"], + } + if envelope is not None: + tables["math_ptptstats"]["n_within_envelope"] = step2.get("n_within_envelope", 0) + + result: dict[str, Any] = {"batch": batch_index, "tables": tables} + if watermark is not None: + result["watermark"] = watermark + return result + + +def check_batch_coverage(out_dir: str | Path, math_envs: Sequence[str]) -> dict[str, Any]: + """NO-COVERAGE GUARD (REQUIRED FIX #1, 2026-07-24 live-debug task): "a + vacuous pass must be structurally impossible". Reads the feeder's + ``manifest.json`` (if present, via :func:`load_manifest`) for any batch + EXPLICITLY marked ``ready: false`` for one of ``math_envs`` — the shape + :func:`run_batch_loop` writes both on a clean batch AND (since the + fail-fast fix) into the partial manifest it persists right before + raising :class:`PollerEquivStreamError`. Independently checks each env's + on-disk snapshot store isn't completely empty + (:func:`discover_batches`). + + A batch record that simply has NO ``envs`` entry at all for a given env + (older/canned-fixture manifests that never tracked per-env readiness) is + NOT penalized here — only an EXPLICIT ``ready: False`` counts as a + coverage failure. This keeps the guard additive: it catches the real, + observed failure mode (a manifest that HONESTLY records "this batch + never became ready") without requiring every historical/fixture + manifest to carry readiness bookkeeping it was never asked for. + + Reads ONLY from disk — no DB, no subprocess. + """ + out_dir = Path(out_dir) + manifest = load_manifest(out_dir) + not_ready: list[dict[str, Any]] = [] + n_manifest_batches = 0 + if manifest is not None: + batches = manifest.get("batches", []) + n_manifest_batches = len(batches) + for b in batches: + envs_info = b.get("envs") or {} + for env in math_envs: + info = envs_info.get(env) + if info is not None and info.get("ready") is False: + not_ready.append({"batch": b.get("index"), "env": env}) + empty_stores = [env for env in math_envs if not discover_batches(out_dir, env)] + return { + "manifest_present": manifest is not None, + "n_manifest_batches": n_manifest_batches, + "not_ready": not_ready, + "empty_stores": empty_stores, + "ok": not not_ready and not empty_stores, + } + + +def compare_snapshots( + out_dir: str | Path, + *, + math_envs: tuple[str, str] = ("clj-ref", "py-shadow"), + math_main_comparer: StepComparer | None = None, + ptptstats_comparer: StepComparer | None = None, + envelope: dict[str, float] | None = None, + expected_batches: int | None = None, +) -> dict[str, Any]: + """Top-level comparer (spec §1 "compare" bullet + Stage C item 2): + per-batch table verdicts (:func:`compare_batch`) plus the CROSS-batch + tick-monotonicity checks (:func:`strictly_increasing` over each env's + ``caching_tick`` / ``math_tick`` sequence, read from its math_main + snapshots). + + Batches present in only ONE env's store are reported (``batches_only_in``) + but never silently dropped from that visibility — only the ALIGNED + (present-in-both) batches are compared table-by-table, since a solo batch + has no partner to diff against. + + ``envelope`` (Stage D item 2) is forwarded unchanged to every + :func:`compare_batch` call. ``envelope=None`` (the default) returns a + report BYTE-IDENTICAL to the pre-Stage-D shape (no extra keys); passing an + envelope adds ``envelope_applied``/``n_within_envelope_total`` so the + within-envelope acceptance is always visible in the verdict JSON, never + silent. + """ + out_dir = Path(out_dir) + env_a, env_b = math_envs + batches_a = discover_batches(out_dir, env_a) + batches_b = discover_batches(out_dir, env_b) + aligned = sorted(set(batches_a) & set(batches_b)) + only_a = sorted(set(batches_a) - set(batches_b)) + only_b = sorted(set(batches_b) - set(batches_a)) + + per_batch = [ + compare_batch( + out_dir, i, math_envs, + math_main_comparer=math_main_comparer, ptptstats_comparer=ptptstats_comparer, + envelope=envelope, + ) + for i in aligned + ] + + tick_series: dict[str, dict[str, list[Any]]] = { + env: {"caching_tick": [], "math_tick": []} for env in math_envs + } + for i in aligned: + for env in math_envs: + row = load_snapshot(out_dir, env, i, "math_main") + tick_series[env]["caching_tick"].append(row.get("caching_tick") if row else None) + tick_series[env]["math_tick"].append(row.get("math_tick") if row else None) + + ticks = { + env: { + "caching_tick": strictly_increasing(tick_series[env]["caching_tick"]), + "math_tick": strictly_increasing(tick_series[env]["math_tick"]), + } + for env in math_envs + } + + tables_ok = all( + all(t.get("match", False) for t in b["tables"].values()) for b in per_batch + ) + watermarks_ok = all(b.get("watermark", {}).get("ok", True) for b in per_batch) + ticks_ok = all( + tr["caching_tick"]["strictly_increasing"] and tr["math_tick"]["strictly_increasing"] + for tr in ticks.values() + ) + coverage = check_batch_coverage(out_dir, math_envs) + # NO-COVERAGE GUARD (spec: "a vacuous pass must be structurally + # impossible") — ``len(aligned) > 0`` is checked EXPLICITLY, not merely + # inferred from ``coverage["ok"]``: a manifest that never tracked + # per-env readiness at all (coverage-neutral by design, see + # :func:`check_batch_coverage`) must still not let a zero-aligned-batch + # comparison report MATCH via the vacuous ``all([])`` behavior above. + # COMPLETENESS GUARD (#2657 review finding 2, 2026-07-24): non-zero is + # not enough — a feeder killed cleanly BETWEEN batches (outside the + # fail-fast paths that write ready:false) leaves later batches simply + # ABSENT, which the readiness coverage cannot see. When the caller + # knows the PLANNED batch count, aligned must equal it exactly. + complete = expected_batches is None or len(aligned) == expected_batches + overall_match = ( + len(aligned) > 0 and not only_a and not only_b and complete + and tables_ok and watermarks_ok and ticks_ok and coverage["ok"] + ) + + report: dict[str, Any] = { + "out_dir": str(out_dir), + "math_envs": list(math_envs), + "n_batches_aligned": len(aligned), + "batches_only_in": {env_a: only_a, env_b: only_b}, + "per_batch": per_batch, + "ticks": ticks, + "coverage": coverage, + "overall_match": overall_match, + } + if envelope is not None: + report["envelope_applied"] = True + report["n_within_envelope_total"] = sum( + b["tables"].get(t, {}).get("n_within_envelope", 0) + for b in per_batch + for t in ("math_main", "math_ptptstats") + ) + if expected_batches is not None: + # Same only-when-provided convention as ``envelope``: default calls + # keep the pre-existing report shape byte-identical. + report["expected_batches"] = expected_batches + return report + + +def write_compare_verdict(report: dict[str, Any], out_dir: str | Path) -> Path: + """Persist :func:`compare_snapshots`'s report to + ``/compare_verdict.json`` (mirrors certify's + ``/certify_report.json`` convention, certify.py:824).""" + path = Path(out_dir) / "compare_verdict.json" + with open(path, "w") as fh: + json.dump(report, fh, indent=2, sort_keys=True, default=str) + return path + + +def compare_exit_code(report: dict[str, Any]) -> int: + return 0 if report["overall_match"] else 1 + + +def render_compare_lines(report: dict[str, Any], *, max_lines: int = 40) -> list[str]: + """Render :func:`compare_snapshots`'s report to ≤``max_lines`` stdout + lines — certify's terse-output convention (certify.py:872-886 + ``render_run_lines``): a header, one line per aligned batch (truncated + with a '+N more' line if the run is too large to fit), and a footer + with the overall verdict + per-env tick-monotonicity status. + + When ``report`` carries ``envelope_applied`` (Stage D item 2 — + :func:`compare_snapshots` called WITH an envelope), one extra trailing + line reports the total within-envelope-accepted count — REPORTED, never + silent, per spec §2 item 2. Reports without that key (the default, + envelope-less path) get no such line, keeping the ≤40-line footer + unchanged from before Stage D. + """ + header = [ + f"poller-equiv compare: {report['n_batches_aligned']} aligned batches " + f"envs={report['math_envs']}", + ] + only_in = report["batches_only_in"] + if any(only_in.values()): + header.append(f" ! batches only in one env: {only_in}") + + coverage = report.get("coverage") + if coverage is not None and not coverage.get("ok", True): + header.append( + f" ! COVERAGE GUARD FAILED: not_ready={coverage.get('not_ready')} " + f"empty_stores={coverage.get('empty_stores')} " + f"manifest_present={coverage.get('manifest_present')}" + ) + + tick_bits = [] + for env, tr in report["ticks"].items(): + ok = tr["caching_tick"]["strictly_increasing"] and tr["math_tick"]["strictly_increasing"] + tick_bits.append(f"{env}:ticks_ok={ok}") + footer = [ + f"verdict: {'MATCH' if report['overall_match'] else 'DIVERGENCE'} " + + " ".join(tick_bits) + ] + if report.get("envelope_applied"): + footer.append( + f" envelope: {report.get('n_within_envelope_total', 0)} divergence(s) " + "accepted within self-jitter envelope" + ) + + budget = max(max_lines - len(header) - len(footer), 0) + body = [] + for b in report["per_batch"]: + bad = [t for t, v in b["tables"].items() if not v.get("match", False)] + wm = b.get("watermark", {}) + wm_flag = "" if wm.get("ok", True) else " WATERMARK-MISMATCH" + if bad or wm_flag: + body.append(f" batch {b['batch']}: FAIL tables={bad}{wm_flag}") + else: + body.append(f" batch {b['batch']}: MATCH") + + if len(body) > budget: + shown = body[: max(budget - 1, 0)] + body = shown + [f" … +{len(body) - len(shown)} more batches — see compare_verdict.json"] + + return header + body + footer + + +# ============================================================================= +# Stage D — self-jitter envelope + full-run orchestration. +# ============================================================================= +# The clj container's cold-tick PCA start is unseeded-random in production +# (MATH_POLLER_EQUIV_SPEC.md §2 — no Clojure source edits allowed, so this +# harness can never Q10/Q12-pin it away). Two independent clj runs on the +# SAME vote stream therefore differ in their float tails even with zero code +# changes; the "envelope" is the measured size of that self-jitter, per +# structural location, so Stage C's comparer can tell "py disagrees with clj" +# apart from "clj disagrees with itself". +_ENVELOPE_FLOOR = 1e-9 +_ENVELOPE_SAFETY_FACTOR = 2.0 +# math_bidtopid is pure-int bid->pid membership (compare_bidtopid, EXACT +# equality) — no float leaf exists to jitter, so it is never walked here. +_ENVELOPE_TABLES: tuple[str, ...] = ("math_main", "math_ptptstats") + + +def _safe_abs_delta(a: Any, b: Any) -> float | None: + """``abs(float(a) - float(b))``, or ``None`` when either side isn't + coercible to float (defensive — a "Numeric mismatch"-reasoned diff's + ``a``/``b`` are always floats in practice, but this never raises into a + caller's loop over many diffs).""" + try: + return abs(float(a) - float(b)) + except (TypeError, ValueError): + return None + + +def _zero_tolerance_comparer(table: str) -> StepComparer: + """The SAME comparer factory Stage C's :func:`compare_batch` uses for + ``table`` (:func:`_acceptance_projecting_comparer` for math_main — + acceptance-projected, prep-main-keyed; :func:`_ptptstats_comparer` for + math_ptptstats — the flat envelope, no projection needed), but with + EVERY tolerance floor forced to zero. ``ConversationComparer`` only + records a "Numeric mismatch" divergence when ``np.allclose(..., rtol=0, + atol=0)`` is False (comparer.py:856) — i.e. when the two floats are NOT + bit-identical — so this configuration surfaces every non-zero cross-run + delta, however small, as a measurable divergence entry. This is for + self-jitter MEASUREMENT only (:func:`compute_self_jitter_envelope`); + never used for pass/fail acceptance.""" + kwargs = {"abs_tolerance": 0.0, "rel_tolerance": 0.0, "outlier_fraction": 0.0} + if table == "math_main": + return _acceptance_projecting_comparer(**kwargs) + if table == "math_ptptstats": + return _ptptstats_comparer(**kwargs) + raise ValueError(f"no zero-tolerance comparer for table {table!r}; expected one of {_ENVELOPE_TABLES}") + + +def envelope_path_key(table: str, raw_path: str) -> str: + """``{table}.{normalized-path}`` — the envelope's lookup key. + + ``raw_path`` is normalized with certify's OWN fingerprint normalizer + (:func:`polismath.replay.certify.normalize_path`, imported — never + reimplemented, per the task's explicit instruction): strips the + ``step_N.`` prefix, collapses ``[idx]`` -> ``[]``, and collapses + purely-numeric dotted segments -> ``N``, so the SAME structural location + at a different batch/list-index/dict-key collapses to one key — exactly + the alignment :func:`compare_batch`'s per-batch divergence paths need to + match against when doing envelope-aware acceptance. Table-prefixed so + math_main and math_ptptstats can never collide even if a leaf name is + ever shared between them. + """ + return f"{table}.{normalize_path(raw_path)}" + + +def compute_self_jitter_envelope( + out_dir_run1: str | Path, + out_dir_run2: str | Path, + *, + math_env: str = "clj-ref", + tables: Sequence[str] = _ENVELOPE_TABLES, +) -> dict[str, Any]: + """Per-``path_pattern`` max absolute float-leaf delta between TWO + independent clj-ref snapshot stores of the SAME stream (spec §2 item 1 / + Stage D item 1: "clj self-jitter envelope first"). + + Walks the IDENTICAL acceptance-projected surface Stage C's + :func:`compare_batch` compares — :func:`_zero_tolerance_comparer` reuses + the exact same comparer factories (:func:`_acceptance_projecting_comparer` + / :func:`_ptptstats_comparer`), only with every tolerance forced to zero + so EVERY cross-run delta (not just ones beyond the default 1e-6/1% + tolerance) is captured, however small. Only ``"Numeric mismatch"`` + -reasoned leaves feed the envelope; any OTHER divergence reason (Integer/ + String/Value/Length/Key/Type mismatch — the two clj runs disagreeing + STRUCTURALLY, which would be a harness bug, not jitter) is reported + separately in ``structural_divergences`` — never silently folded into a + numeric envelope. + + Batches present in only one store (``batches_only_in``) or a missing + per-(batch, table) snapshot pair (``missing_snapshots``) are reported but + simply skipped for that slice — a partial self-jitter run still yields a + usable (partial) envelope for whichever (batch, table) pairs DID land on + both sides. + + Reads ONLY from disk — no DB, no subprocess; identical inputs (two stores + with byte-identical data) yield an EMPTY ``envelope`` mapping (no + "Numeric mismatch" is ever recorded for equal floats — ``np.allclose`` + with ``rtol=atol=0`` is true iff the floats are bit-identical), the + "all-zero" case — callers must read a missing key via + :func:`envelope_threshold` (which defaults to the floor), never assume a + present-but-zero entry. + """ + out_dir_run1, out_dir_run2 = Path(out_dir_run1), Path(out_dir_run2) + batches1 = discover_batches(out_dir_run1, math_env) + batches2 = discover_batches(out_dir_run2, math_env) + aligned = sorted(set(batches1) & set(batches2)) + only1 = sorted(set(batches1) - set(batches2)) + only2 = sorted(set(batches2) - set(batches1)) + + comparers = {t: _zero_tolerance_comparer(t) for t in tables} + envelope: dict[str, float] = {} + n_leaf_diffs = 0 + structural_divergences: list[dict[str, Any]] = [] + per_batch_table_diff_counts: dict[str, int] = {} + missing_snapshots: list[dict[str, Any]] = [] + + for i in aligned: + for table in tables: + row1 = load_snapshot(out_dir_run1, math_env, i, table) + row2 = load_snapshot(out_dir_run2, math_env, i, table) + if row1 is None or row2 is None: + missing_snapshots.append({ + "batch": i, "table": table, + "missing_in": [name for name, row in (("run1", row1), ("run2", row2)) if row is None], + }) + continue + + step = comparers[table].compare_step(row1["data"], row2["data"], i) + count_this = 0 + for family in ("exact", "tolerant"): + for d in step["families"][family]: + reason = d.get("reason") or "" + if not reason.startswith("Numeric mismatch"): + # 'exact'-family here means a NON-numeric structural + # disagreement between the two clj runs (e.g. an + # Integer/Value/Length/Key mismatch) — real, but not + # jitter; surfaced separately, never silently dropped. + structural_divergences.append({ + "batch": i, "table": table, "path": d.get("path"), "reason": reason, + }) + continue + delta = _safe_abs_delta(d.get("a"), d.get("b")) + if delta is None: + continue + key = envelope_path_key(table, d.get("path") or "") + envelope[key] = max(envelope.get(key, 0.0), delta) + count_this += 1 + n_leaf_diffs += 1 + per_batch_table_diff_counts[f"batch-{i:03d}.{table}"] = count_this + + return { + "math_env": math_env, + "out_dir_run1": str(out_dir_run1), + "out_dir_run2": str(out_dir_run2), + "n_batches_aligned": len(aligned), + "batches_only_in": {"run1": only1, "run2": only2}, + "missing_snapshots": missing_snapshots, + "n_leaf_diffs": n_leaf_diffs, + "per_batch_table_diff_counts": per_batch_table_diff_counts, + "structural_divergences": structural_divergences, + "envelope": dict(sorted(envelope.items())), + } + + +def write_envelope(envelope_report: dict[str, Any], out_dir: str | Path) -> Path: + """Persist :func:`compute_self_jitter_envelope`'s report to + ``/self_jitter_envelope.json`` (mirrors :func:`write_compare_verdict`'s + convention).""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / "self_jitter_envelope.json" + with open(path, "w") as fh: + json.dump(envelope_report, fh, indent=2, sort_keys=True, default=str) + return path + + +def load_envelope(out_dir: str | Path) -> dict[str, Any] | None: + path = Path(out_dir) / "self_jitter_envelope.json" + if not path.exists(): + return None + return json.loads(path.read_text()) + + +def envelope_threshold(envelope: dict[str, float] | None, key: str) -> float: + """``max(envelope[key] * 2, 1e-9)`` — spec §2 item 2's acceptance formula + (safety factor 2, floor 1e-9). ``envelope=None`` or a missing ``key`` + means "no observed self-jitter at this path" -> the threshold is just the + floor, never zero (a genuinely bit-identical clj-vs-clj path still allows + a hair of float noise before a py divergence counts as real).""" + base = (envelope or {}).get(key, 0.0) + return max(base * _ENVELOPE_SAFETY_FACTOR, _ENVELOPE_FLOOR) + + +def _apply_envelope_to_step( + step: dict[str, Any], envelope: dict[str, float] | None, table: str, +) -> dict[str, Any]: + """Reclassify ``step``'s (a :meth:`StepComparer.compare_step` result) + 'tolerant'-family divergences that fall within the self-jitter envelope + into a THIRD family, ``within_envelope`` (spec §2 item 2 / Stage D item + 2). Accepted for ``match`` purposes but ALWAYS counted + (``n_within_envelope``) — never silently dropped from the verdict. + + 'exact'-family divergences (structural: memberships/cluster ids/ + selections/priority ordering — see ``stepcompare.py``'s family + docstring) are NEVER touched: the envelope can only excuse a FLOAT + jitter, never a structural mismatch (spec's explicit "structural + divergences are NEVER excused by the envelope"). + + ``envelope=None`` returns ``step`` COMPLETELY UNCHANGED (same dict, + same keys: ``step``/``match``/``n_divergences``/``families``/ + ``sign_flips``) — :func:`compare_batch`'s "byte-identical when no + envelope" contract relies on this exact pass-through. + """ + if envelope is None: + return step + + still_tolerant: list[dict[str, Any]] = [] + within_envelope: list[dict[str, Any]] = [] + for d in step["families"]["tolerant"]: + reason = d.get("reason") or "" + delta = _safe_abs_delta(d.get("a"), d.get("b")) if reason.startswith("Numeric mismatch") else None + if delta is None: + still_tolerant.append(d) + continue + key = envelope_path_key(table, d.get("path") or "") + threshold = envelope_threshold(envelope, key) + if delta <= threshold: + within_envelope.append({**d, "envelope_key": key, "delta": delta, "threshold": threshold}) + else: + still_tolerant.append(d) + + n_exact = len(step["families"]["exact"]) + match = n_exact == 0 and len(still_tolerant) == 0 + return { + "step": step["step"], + "match": match, + "n_divergences": n_exact + len(still_tolerant), + "n_within_envelope": len(within_envelope), + "families": { + "exact": step["families"]["exact"], + "tolerant": still_tolerant, + "within_envelope": within_envelope, + }, + "sign_flips": step.get("sign_flips", []), + } + + +# --------------------------------------------------------------------------- +# clj-only self-jitter stream — spec §2 item 1's "run the clj side TWICE". +# --------------------------------------------------------------------------- +def run_clj_only_stream( + admin_url: str, + dataset_slug: str, + cuts: Sequence[int], + *, + out_dir: str | Path, + dbname: str, + math_env: str = "clj-ref", + zid: int = DEFAULT_ZID, + poll_from_days_ago: float = 10000, + wait_timeout: float = 120.0, + poll_interval: float = 0.5, + engine_factory: Callable[[str], Any] | None = None, + runner_builder: Callable[[], Any] | None = None, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, + wait_for_clj_poll_cycle: bool = True, + poll_cycle_gate_timeout: float = 60.0, +) -> dict[str, Any]: + """Seed a FRESH throwaway DB and run ONLY the clj container against it — + NO py runner at all (Stage D item 1 / spec §2 item 1: "run the clj side + TWICE on the same stream, fresh DB each"). A thin single-env wrapper + around the SAME :func:`run_batch_loop` Stage C already uses — no new + feeder logic, ``math_envs``/``runners`` is just a one-entry list/dict. + + ``wait_for_clj_poll_cycle`` (session 2, 2026-07-24, default True) — + quirk Q19's harness-level mitigation, same as :func:`run_equiv_stream`'s + own parameter of the same name; see :func:`wait_for_first_poll_cycle`'s + docstring for the full rationale. + + NOT exercised by the default test suite (needs a real Postgres AND a real + ``clojure`` subprocess running for the duration of the stream) — mirrors + :func:`run_equiv_stream`'s own "not exercised" note; every decision this + function makes is delegated to :func:`run_batch_loop` (fully unit-tested + with fakes) or is thin setup/teardown glue around it. + """ + ds = real_data.load_export_votes(dataset_slug) + target_url = create_equiv_db(admin_url, dbname=dbname) + engine = (engine_factory or (lambda url: sa.create_engine(url, isolation_level="AUTOCOMMIT")))( + target_url + ) + + build = runner_builder or (lambda: CljContainerRunner( + database_url=target_url, math_env=math_env, poll_from_days_ago=poll_from_days_ago, + log_path=Path(out_dir) / f"{math_env}.runner.log", + )) + + runners: dict[str, Any] = {} + try: + with engine.connect() as seed_conn: + seed_conversation(seed_conn, ds, zid=zid) + + runners = {math_env: build()} + runners[math_env].start() + + with engine.connect() as conn: + manifest = run_batch_loop( + conn, ds, cuts, [math_env], runners, out_dir=out_dir, zid=zid, + wait_timeout=wait_timeout, poll_interval=poll_interval, sleep=sleep, now=now, + startup_gate_envs=[math_env] if wait_for_clj_poll_cycle else None, + startup_gate_timeout=poll_cycle_gate_timeout, + ) + finally: + for r in runners.values(): + r.kill() + engine.dispose() + + return manifest + + +# --------------------------------------------------------------------------- +# Full-run orchestration (spec §2/§3 stage D item 3) — pure verdict assembly +# split out from the live I/O glue, per the task's explicit testability ask. +# --------------------------------------------------------------------------- +def assemble_full_run_verdict( + out_dir_run1: str | Path, + out_dir_run2: str | Path, + out_dir_main: str | Path, + *, + dataset: str = "", + cuts: Sequence[int] = (), + seam_after: int | None = None, + clj_env: str = "clj-ref", + py_env: str = "py-shadow", +) -> dict[str, Any]: + """PURE decision logic (Stage D item 3: "its DECISION LOGIC ... must be a + pure function unit-testable with canned snapshot dirs") for the full + protocol's envelope-wiring + verdict assembly: given THREE already- + populated snapshot stores — two independent clj-only self-jitter runs + (:func:`run_clj_only_stream`'s output dirs) plus one paired clj+py + restart-seam run (:func:`run_equiv_stream`'s output dir) — compute the + self-jitter envelope, feed it into an envelope-aware + :func:`compare_snapshots`, and assemble one verdict dict. + + Reads ONLY from disk (:func:`compute_self_jitter_envelope` and + :func:`compare_snapshots` are themselves disk-only) — never touches a DB + or a subprocess, so this is exercisable against canned/fixture snapshot + directories with no live services, exactly like every other Stage C/D + pure function in this module. + """ + expected_batches = len(cuts) if cuts else None + envelope_report = compute_self_jitter_envelope(out_dir_run1, out_dir_run2, math_env=clj_env) + compare_report = compare_snapshots( + out_dir_main, math_envs=(clj_env, py_env), envelope=envelope_report["envelope"], + expected_batches=expected_batches, + ) + # NO-COVERAGE GUARD (REQUIRED FIX #1, 2026-07-24 live-debug task) applied + # to the self-jitter streams too — this is the SAME class of bug as the + # "main" store's zero-aligned-batches guard, but on the envelope + # measurement itself: an envelope computed from 0 aligned batches (both + # clj-only runs empty) reports an all-zero "0 path(s) jittered" envelope + # that reads exactly like a clean, IDENTICAL pair — reproduced verbatim + # in the 2026-07-24 live run's terse summary. ``compare_snapshots`` + # already guards the "main" store; the self-jitter streams need the same + # ``check_batch_coverage`` guard directly, since :func:`compute_self_jitter_envelope` + # itself has no pass/fail concept (it's a pure measurement, by design). + jitter1_coverage = check_batch_coverage(out_dir_run1, [clj_env]) + jitter2_coverage = check_batch_coverage(out_dir_run2, [clj_env]) + self_jitter_ok = ( + envelope_report["n_batches_aligned"] > 0 + and jitter1_coverage["ok"] and jitter2_coverage["ok"] + # COMPLETENESS (#2657 review finding 2): a partial self-jitter + # stream under-measures the envelope the same way a partial main + # store under-compares — planned count must be met here too. + and (expected_batches is None + or envelope_report["n_batches_aligned"] == expected_batches) + ) + return { + "dataset": dataset, + "cuts": list(cuts), + "seam_after": seam_after, + "clj_env": clj_env, + "py_env": py_env, + "out_dir_run1": str(out_dir_run1), + "out_dir_run2": str(out_dir_run2), + "out_dir_main": str(out_dir_main), + "self_jitter_envelope": envelope_report, + "self_jitter_coverage": {"run1": jitter1_coverage, "run2": jitter2_coverage}, + "compare": compare_report, + "overall_pass": compare_report["overall_match"] and self_jitter_ok, + } + + +def write_full_run_verdict(verdict: dict[str, Any], out_root: str | Path) -> Path: + """Persist :func:`assemble_full_run_verdict`'s report to + ``/full_run_verdict.json`` (spec §2 item 3 / Stage D item + 3(e)).""" + out_root = Path(out_root) + out_root.mkdir(parents=True, exist_ok=True) + path = out_root / "full_run_verdict.json" + with open(path, "w") as fh: + json.dump(verdict, fh, indent=2, sort_keys=True, default=str) + return path + + +def render_full_run_lines(verdict: dict[str, Any], *, max_lines: int = 40) -> list[str]: + """≤``max_lines`` stdout summary (Stage D item 3(e): "overall PASS/FAIL + + per-batch/table counts + the envelope's worst paths"). Delegates the + per-batch/table portion to :func:`render_compare_lines` (already its own + ≤N-line renderer) after reserving room for a PASS/FAIL header and the + envelope's worst (largest-max-delta) paths — so the WHOLE full-run + summary, not just the compare section, respects the line budget. + """ + compare_report = verdict["compare"] + envelope = verdict["self_jitter_envelope"]["envelope"] + overall = "PASS" if verdict["overall_pass"] else "FAIL" + + header = [ + f"poller-equiv full-run: dataset={verdict.get('dataset', '?')} " + f"seam_after={verdict.get('seam_after')} verdict={overall}", + ] + + worst = sorted(envelope.items(), key=lambda kv: kv[1], reverse=True)[:5] + envelope_lines = [f" self-jitter envelope: {len(envelope)} path(s) jittered; worst:"] + if worst: + envelope_lines += [f" {path}: {delta:.3e}" for path, delta in worst] + else: + envelope_lines.append(" (none observed — identical self-jitter runs)") + + # NO-COVERAGE GUARD visibility (REQUIRED FIX #1): an empty/all-zero + # envelope is AMBIGUOUS on its own — "identical self-jitter runs" reads + # identically whether that's genuinely zero jitter OR zero batches ever + # measured (the exact misleading text the 2026-07-24 live run printed). + # This line disambiguates whenever self_jitter_coverage is present and + # failed — never silent. + jitter_coverage = verdict.get("self_jitter_coverage") + if jitter_coverage is not None: + bad_runs = [name for name, cov in jitter_coverage.items() if not cov.get("ok", True)] + if bad_runs: + envelope_lines.append(f" ! SELF-JITTER COVERAGE FAILED: {bad_runs}") + + reserved = len(header) + len(envelope_lines) + remaining = max(max_lines - reserved, 4) + body = render_compare_lines(compare_report, max_lines=remaining) + + lines = header + envelope_lines + body + if len(lines) > max_lines: + lines = lines[: max_lines - 1] + [" … output truncated — see full_run_verdict.json"] + return lines + + +# --------------------------------------------------------------------------- +# Default schedule resolution (Stage D item 3: "cuts (default: vw at its +# uniform8 slots — read the actual slots from scripts/schedules or the +# certify battery)"). +# --------------------------------------------------------------------------- +_VW_UNIFORM8_RESTART4_SCHEDULE = _DELPHI_ROOT / "scripts" / "schedules" / "vw-uniform8-restart4.json" + + +def default_full_run_schedule(dataset: str) -> tuple[list[int], int]: + """Default ``(cuts, seam_after)`` for the ``full-run`` CLI when + ``--cuts``/``--seam-after`` are not given explicitly. + + For ``vw`` this is read VERBATIM from the committed + ``scripts/schedules/vw-uniform8-restart4.json`` — the SAME schedule file + ``certify_battery.json``'s own restart-seam entry uses (its + ``"dataset": "vw", "schedule": "schedules/vw-uniform8-restart4.json"`` + row): vw's uniform-8 cuts with the restart seam at step 4 (mid-schedule + for 8 cuts, 0-based). + + For any OTHER dataset, uniform-8 cuts are derived from the dataset's OWN + vote count via :func:`polismath.replay.schedule.preset_uniform` — the + SAME preset certify's own ``"preset": "uniform", "n_cuts": 8`` battery + entries resolve through — with the seam fixed at the middle cut index. + """ + if dataset == "vw": + spec = sched.ScheduleSpec.from_json_file(_VW_UNIFORM8_RESTART4_SCHEDULE) + cuts = [int(c) for c in spec.cuts["at"]] + seam_after = spec.restart_after if spec.restart_after is not None else len(cuts) // 2 + return cuts, seam_after + ds = real_data.load_export_votes(dataset) + spec = sched.preset_uniform(dataset, ds.n, n_cuts=8) + cuts = [int(c) for c in spec.cuts["at"]] + return cuts, len(cuts) // 2 + + +# --------------------------------------------------------------------------- +# Live full-run orchestration — REQUIRES Postgres + the clojure CLI. +# --------------------------------------------------------------------------- +def preflight_check(admin_url: str, *, connect_timeout: float = 5.0) -> None: + """Fail FAST with a clear, actionable message when a prerequisite live + service is unreachable (Stage D item 3: "must fail fast ... when + Postgres/clojure are unreachable") — never a bare driver traceback deep + inside a multi-minute run.""" + if shutil.which("clojure") is None: + raise RuntimeError( + "poller-equiv full-run requires the 'clojure' CLI on PATH (it invokes " + "`clojure -M:run full` in math/, see CljContainerRunner) — not found. " + "Install/activate it before retrying." + ) + # psycopg2 rejects a float connect_timeout ("invalid integer value") — + # ceil to at least 1 whole second (found live, 2026-07-24 preflight). + probe = sa.create_engine( + admin_url, + connect_args={"connect_timeout": max(1, int(round(connect_timeout)))}, + ) + try: + with probe.connect(): + pass + except Exception as exc: + raise RuntimeError( + f"poller-equiv full-run cannot reach Postgres via --pg-admin-url " + f"(connect_timeout={connect_timeout}s): {exc}" + ) from exc + finally: + probe.dispose() + + +@dataclass(frozen=True) +class FullRunConfig: + """Full-run parameters (Stage D item 3). By the time + :func:`run_full_equiv_protocol` sees one of these, ``cuts``/``seam_after`` + are already FULLY RESOLVED — the CLI (or a test) resolves defaults via + :func:`default_full_run_schedule` before constructing this.""" + + dataset: str + admin_url: str + out_root: str + cuts: tuple[int, ...] + seam_after: int + dbname: str = "polis_equiv_full" + zid: int = DEFAULT_ZID + clj_env: str = "clj-ref" + py_env: str = "py-shadow" + engine_mode: str = ENGINE_MODE_LEGACY + poll_from_days_ago: float = 10000 + wait_timeout: float = 120.0 + poll_interval: float = 0.5 + # Spec §1's "seam" bullet: "Also restart the clj container at the same + # seam for symmetry (its load-or-init)" — full-run implements the + # COMPLETE protocol end-to-end, so this defaults True (stricter than the + # lower-level `feed` CLI subcommand's conservative default False). + restart_clj_at_seam: bool = True + # Quirk Q19 harness-level mitigation (session 2, 2026-07-24), approved + # under the goal's standing autonomy — see wait_for_first_poll_cycle's + # docstring. Defaults ON for the same reason restart_clj_at_seam does: + # full-run implements the complete, strictest protocol end-to-end. + wait_for_clj_poll_cycle: bool = True + poll_cycle_gate_timeout: float = 60.0 + + +def run_full_equiv_protocol( + config: FullRunConfig, + *, + engine_factory: Callable[[str], Any] | None = None, + clj_runner_builder: Callable[[], Any] | None = None, + py_runner_builder: Callable[[], Any] | None = None, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, + skip_preflight: bool = False, +) -> dict[str, Any]: + """LIVE orchestration of the spec's complete protocol (Stage D item 3): + (a) clj-ref run 1, (b) fresh DB + clj-ref run 2 -> self-jitter envelope, + (c) fresh DB + clj-ref/py-shadow paired run with a restart seam, (d)/(e) + envelope-aware compare + verdict JSON + terse summary. + + REQUIRES live Postgres + the ``clojure`` CLI — :func:`preflight_check` + fails fast (clear message, before touching anything) unless + ``skip_preflight`` is set (tests only). Every decision beyond I/O + sequencing is delegated to :func:`assemble_full_run_verdict` (pure, + canned-dir testable) — this function is thin glue around three + ``run_*_stream`` calls plus that assembly, mirroring + :func:`run_equiv_stream`'s own "not exercised by the default test suite" + status for the same reason (needs a real Postgres + real subprocesses). + """ + if not skip_preflight: + preflight_check(config.admin_url) + + out_root = Path(config.out_root) + out_dir_run1 = out_root / "self-jitter-1" + out_dir_run2 = out_root / "self-jitter-2" + out_dir_main = out_root / "main" + + run_clj_only_stream( + config.admin_url, config.dataset, config.cuts, out_dir=out_dir_run1, + dbname=f"{config.dbname}_jitter1", math_env=config.clj_env, zid=config.zid, + poll_from_days_ago=config.poll_from_days_ago, wait_timeout=config.wait_timeout, + poll_interval=config.poll_interval, engine_factory=engine_factory, + runner_builder=clj_runner_builder, sleep=sleep, now=now, + wait_for_clj_poll_cycle=config.wait_for_clj_poll_cycle, + poll_cycle_gate_timeout=config.poll_cycle_gate_timeout, + ) + run_clj_only_stream( + config.admin_url, config.dataset, config.cuts, out_dir=out_dir_run2, + dbname=f"{config.dbname}_jitter2", math_env=config.clj_env, zid=config.zid, + poll_from_days_ago=config.poll_from_days_ago, wait_timeout=config.wait_timeout, + poll_interval=config.poll_interval, engine_factory=engine_factory, + runner_builder=clj_runner_builder, sleep=sleep, now=now, + wait_for_clj_poll_cycle=config.wait_for_clj_poll_cycle, + poll_cycle_gate_timeout=config.poll_cycle_gate_timeout, + ) + + runner_builders: dict[str, Callable[[], Any]] = {} + if clj_runner_builder is not None: + runner_builders["clj"] = clj_runner_builder + if py_runner_builder is not None: + runner_builders["py"] = py_runner_builder + + run_equiv_stream( + config.admin_url, config.dataset, config.cuts, out_dir=out_dir_main, + seam_after=config.seam_after, math_envs=(config.clj_env, config.py_env), + restart_clj_at_seam=config.restart_clj_at_seam, dbname=f"{config.dbname}_main", + zid=config.zid, poll_from_days_ago=config.poll_from_days_ago, + engine_mode=config.engine_mode, wait_timeout=config.wait_timeout, + poll_interval=config.poll_interval, engine_factory=engine_factory, + runner_builders=runner_builders or None, sleep=sleep, now=now, + wait_for_clj_poll_cycle=config.wait_for_clj_poll_cycle, + poll_cycle_gate_timeout=config.poll_cycle_gate_timeout, + ) + + verdict = assemble_full_run_verdict( + out_dir_run1, out_dir_run2, out_dir_main, + dataset=config.dataset, cuts=config.cuts, seam_after=config.seam_after, + clj_env=config.clj_env, py_env=config.py_env, + ) + write_full_run_verdict(verdict, out_root) + return verdict diff --git a/delphi/polismath/utils/clj_hash.py b/delphi/polismath/utils/clj_hash.py index 60c013a355..83736378c3 100644 --- a/delphi/polismath/utils/clj_hash.py +++ b/delphi/polismath/utils/clj_hash.py @@ -81,10 +81,19 @@ def _hamt_path(h: int) -> tuple: def _as_long(k: Any) -> Any: - """Numeric-string keys hash as their Long value: production pids are - strings python-side (poll_votes / run_math_pipeline cast ``str(pid)``) - while Clojure holds the DB's integer pid — parity requires ordering by - the integer's hash. Mirrors the ``int(tid) if tid.isdigit()`` idiom used + """Numeric-string keys hash as their Long value: pids can arrive + Python-side as either numeric strings OR native ints, depending on the + pipeline — the legacy DynamoDB job pipeline (run_math_pipeline.py) still + produces string pids in places, while the LIVE poller + (``PostgresClient.poll_votes``/``poll_votes_since``) emits native ints as + of 2026-07-24 (previously it also cast ``str(pid)``; fixed as part of the + poller-equivalence harness's live-debugging session — see + ``polismath/poller/__init__.py``'s bidToPid-shape note for the full + rationale). Either way Clojure holds the DB's integer pid, so parity + requires ordering by the integer's hash regardless of which Python + pipeline produced the key — this function normalizes BOTH forms + uniformly (an int key already IS its own Long value; a numeric-string + key gets converted). Mirrors the ``int(tid) if tid.isdigit()`` idiom used for tids in conversation.py. Non-numeric keys pass through unchanged.""" if isinstance(k, str) and k.lstrip('-').isdigit(): return int(k) diff --git a/delphi/scripts/poller_equiv.py b/delphi/scripts/poller_equiv.py new file mode 100644 index 0000000000..622651ac83 --- /dev/null +++ b/delphi/scripts/poller_equiv.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Poller-equivalence harness CLI — Stages A (schema+seed), B (runners), +C (feeder + comparer), and D (self-jitter envelope + full-run orchestration). + +See ``delphi/docs/MATH_POLLER_EQUIV_SPEC.md`` and +``polismath.replay.poller_equiv`` (the library this is a thin click wrapper +over — same split as ``scripts/certify.py``: the library stays pure / +side-effect-scoped, this script owns printing and process exit codes). + +Stages A/B/C are wired up here: create the throwaway DB + seed a conversation; +start/stop the clj container and the python poller against it; feed timed vote +batches while snapshotting math_main/math_bidtopid/math_ptptstats per env, and +compare the resulting snapshot store. Stage D adds the ``full-run`` subcommand: +two independent clj-only runs -> self-jitter envelope, one paired clj+py +restart-seam run, envelope-aware compare, verdict JSON + terse summary. + +Usage (from delphi/):: + + # Create polis_equiv (dropping it first) and seed one dataset's full + # conversation (comments + ALL votes) under zid=1: + uv run python scripts/poller_equiv.py seed --dataset vw \\ + --admin-url postgresql://postgres:postgres@localhost:15432/postgres + + # Start the clj container against an already-seeded DB, block until Ctrl-C: + uv run python scripts/poller_equiv.py run-clj \\ + --database-url postgresql://postgres:postgres@localhost:15432/polis_equiv \\ + --math-env clj-ref + + # Start the python poller the same way: + uv run python scripts/poller_equiv.py run-py \\ + --database-url postgresql://postgres:postgres@localhost:15432/polis_equiv \\ + --math-env py-shadow + + # Feed a vw uniform-8 batch stream through BOTH runners at once, snapshotting + # each batch's three tables under --out, with a restart seam after batch 3: + uv run python scripts/poller_equiv.py feed --dataset vw \\ + --admin-url postgresql://postgres:postgres@localhost:15432/postgres \\ + --cuts 100,200,300,400,500,585 --seam-after 3 \\ + --out real_data/.local/replays/_poller_equiv/vw + + # Compare an existing --out snapshot store: + uv run python scripts/poller_equiv.py compare \\ + --out real_data/.local/replays/_poller_equiv/vw + + # Full protocol (spec §2/§3 stage D) — two clj-only self-jitter runs, + # one paired clj+py restart-seam run, envelope-aware compare, verdict: + uv run python scripts/poller_equiv.py full-run --dataset vw \\ + --admin-url postgresql://postgres:postgres@localhost:15432/postgres \\ + --out real_data/.local/replays/_poller_equiv_full/vw +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import click +import sqlalchemy as sa + +from polismath.replay import poller_equiv as pe +from polismath.replay.real_data import load_export_votes + + +@click.group() +def cli() -> None: + """Poller-equivalence harness (Stages A/B/C/D — see module docstring).""" + + +@cli.command() +@click.option("--dataset", required=True, help="Dataset slug (e.g. vw).") +@click.option("--admin-url", required=True, + help="Connection URL to an EXISTING db (e.g. .../postgres) on " + "the target server — NOT the equiv db itself.") +@click.option("--dbname", default=pe.DEFAULT_DBNAME, show_default=True, + help="Throwaway database name to (re)create.") +@click.option("--zid", type=int, default=pe.DEFAULT_ZID, show_default=True) +@click.option("--to-slot", type=int, default=None, + help="Insert votes[:to_slot] (default: the whole dataset).") +def seed(dataset, admin_url, dbname, zid, to_slot): + """Create the throwaway equiv DB and seed one dataset's conversation.""" + ds = load_export_votes(dataset) + to_slot = ds.n if to_slot is None else to_slot + click.echo(f"dataset={dataset} n_votes={ds.n} n_comments={len(ds.comments)}", err=True) + + target_url = pe.create_equiv_db(admin_url, dbname=dbname) + engine = sa.create_engine(target_url) + try: + with engine.begin() as conn: + pe.seed_conversation(conn, ds, zid=zid) + n = pe.insert_votes(conn, ds, 0, to_slot, zid=zid) + finally: + engine.dispose() + + click.echo(f"seeded zid={zid} into {dbname!r}: {n} votes, " + f"{len(ds.comments)} comments -> {target_url}") + + +@cli.command("run-clj") +@click.option("--database-url", required=True) +@click.option("--math-env", required=True) +@click.option("--poll-from-days-ago", type=float, default=10000, show_default=True) +def run_clj(database_url, math_env, poll_from_days_ago): + """Start the REAL clj math container loop (blocks; Ctrl-C stops it).""" + runner = pe.CljContainerRunner( + database_url=database_url, math_env=math_env, + poll_from_days_ago=poll_from_days_ago, + ) + _run_and_stream(runner, label="clj") + + +@cli.command("run-py") +@click.option("--database-url", required=True) +@click.option("--math-env", required=True) +@click.option("--poll-from-days-ago", type=float, default=10000, show_default=True) +@click.option("--engine-mode", default="clojure-legacy", show_default=True) +def run_py(database_url, math_env, poll_from_days_ago, engine_mode): + """Start the python math_poller (blocks; Ctrl-C stops it).""" + runner = pe.PyPollerRunner( + database_url=database_url, math_env=math_env, + poll_from_days_ago=poll_from_days_ago, engine_mode=engine_mode, + ) + _run_and_stream(runner, label="py") + + +@cli.command() +@click.option("--dataset", required=True, help="Dataset slug (e.g. vw).") +@click.option("--admin-url", required=True, + help="Connection URL to an EXISTING db (e.g. .../postgres) on " + "the target server — NOT the equiv db itself.") +@click.option("--cuts", required=True, + help="Comma-separated, strictly-increasing 1-based vote-count " + "cut slots (e.g. 100,200,300,400,500,585). Include the " + "dataset's total vote count as the last value to recompute " + "the tail (batch_slices' 'final batch to n' convention).") +@click.option("--out", "out_dir", required=True, type=click.Path(path_type=Path), + help="Snapshot store root — one subdir per math_env.") +@click.option("--seam-after", type=int, default=None, + help="Batch index (0-based) after which to kill+restart the " + "py runner (and, with --restart-clj-at-seam, the clj one).") +@click.option("--restart-clj-at-seam", is_flag=True, default=False) +@click.option("--dbname", default=pe.DEFAULT_DBNAME, show_default=True) +@click.option("--zid", type=int, default=pe.DEFAULT_ZID, show_default=True) +@click.option("--clj-env", default="clj-ref", show_default=True) +@click.option("--py-env", default="py-shadow", show_default=True) +@click.option("--engine-mode", default="clojure-legacy", show_default=True) +@click.option("--poll-from-days-ago", type=float, default=10000, show_default=True) +@click.option("--wait-timeout", type=float, default=120.0, show_default=True, + help="Seconds to wait for EACH math_env to reflect a batch " + "before giving up on it.") +def feed(dataset, admin_url, cuts, out_dir, seam_after, restart_clj_at_seam, dbname, zid, + clj_env, py_env, engine_mode, poll_from_days_ago, wait_timeout): + """Stage C feeder: seed the equiv DB, start both runners, then insert + vote batches one at a time — waiting for each math_env to reflect a + batch before snapshotting math_main/math_bidtopid/math_ptptstats and + moving on (spec §1 'feed'/'seam' bullets).""" + cut_slots = [int(c) for c in cuts.split(",") if c.strip()] + try: + manifest = pe.run_equiv_stream( + admin_url, dataset, cut_slots, + out_dir=out_dir, seam_after=seam_after, math_envs=(clj_env, py_env), + restart_clj_at_seam=restart_clj_at_seam, dbname=dbname, zid=zid, + poll_from_days_ago=poll_from_days_ago, engine_mode=engine_mode, + wait_timeout=wait_timeout, + ) + except pe.PollerEquivStreamError as exc: + click.echo(f"poller-equiv feed: ABORTED\n{exc}", err=True) + sys.exit(1) + n_ready = sum( + 1 for b in manifest["batches"] if all(e["ready"] for e in b["envs"].values()) + ) + click.echo( + f"fed {len(manifest['batches'])} batches ({n_ready} fully ready) -> {out_dir}" + ) + + +@cli.command() +@click.option("--out", "out_dir", required=True, + type=click.Path(exists=True, file_okay=False, path_type=Path)) +@click.option("--clj-env", default="clj-ref", show_default=True) +@click.option("--py-env", default="py-shadow", show_default=True) +def compare(out_dir, clj_env, py_env): + """Stage C comparer: verdict over an existing --out snapshot store (spec + §1 'compare' bullet — same acceptance as certify for math_main, exact + equality for math_bidtopid, structural+tolerant for math_ptptstats, plus + tick-monotonicity and watermark-exactly-once checks).""" + report = pe.compare_snapshots(out_dir, math_envs=(clj_env, py_env)) + pe.write_compare_verdict(report, out_dir) + for line in pe.render_compare_lines(report): + click.echo(line) + sys.exit(pe.compare_exit_code(report)) + + +@cli.command("full-run") +@click.option("--dataset", required=True, help="Dataset slug (e.g. vw).") +@click.option("--admin-url", required=True, + help="Connection URL to an EXISTING db (e.g. .../postgres) on " + "the target server — NOT any of the equiv dbs themselves.") +@click.option("--cuts", default=None, + help="Comma-separated, strictly-increasing 1-based vote-count " + "cut slots. Default: --dataset's uniform-8 slots (for vw, " + "read VERBATIM from scripts/schedules/vw-uniform8-restart4.json; " + "for any other dataset, derived the same way certify's own " + "uniform/n_cuts=8 battery entries are).") +@click.option("--seam-after", type=int, default=None, + help="Batch index (0-based) after which to restart the runners. " + "Default: the default schedule's own restart point (vw: " + "step 4) — mid-schedule for a derived schedule.") +@click.option("--out", "out_root", required=True, type=click.Path(path_type=Path), + help="Output root — holds self-jitter-1/, self-jitter-2/, main/ " + "snapshot stores plus full_run_verdict.json.") +@click.option("--dbname", default="polis_equiv_full", show_default=True, + help="Throwaway database name STEM — suffixed _jitter1/_jitter2/_main.") +@click.option("--zid", type=int, default=pe.DEFAULT_ZID, show_default=True) +@click.option("--clj-env", default="clj-ref", show_default=True) +@click.option("--py-env", default="py-shadow", show_default=True) +@click.option("--engine-mode", default="clojure-legacy", show_default=True) +@click.option("--poll-from-days-ago", type=float, default=10000, show_default=True) +@click.option("--wait-timeout", type=float, default=120.0, show_default=True, + help="Seconds to wait for EACH math_env to reflect a batch " + "before giving up on it.") +@click.option("--restart-clj-at-seam/--no-restart-clj-at-seam", default=True, show_default=True, + help="Also restart the clj container at the seam, for symmetry " + "with the py restart (spec §1's 'seam' bullet).") +@click.option("--wait-for-clj-poll-cycle/--no-wait-for-clj-poll-cycle", default=True, show_default=True, + help="Quirk Q19 harness-level mitigation: block feeding batch 0 " + "until the clj runner's log shows evidence of a completed " + "poll cycle (see wait_for_first_poll_cycle's docstring). " + "Avoids a conv_man.clj actor-creation race that can " + "silently drop an early batch's votes.") +@click.option("--poll-cycle-gate-timeout", type=float, default=60.0, show_default=True, + help="Seconds to wait for the poll-cycle gate signal before " + "aborting (only used when --wait-for-clj-poll-cycle).") +def full_run(dataset, admin_url, cuts, seam_after, out_root, dbname, zid, clj_env, py_env, + engine_mode, poll_from_days_ago, wait_timeout, restart_clj_at_seam, + wait_for_clj_poll_cycle, poll_cycle_gate_timeout): + """Stage D full protocol orchestration (spec §2/§3): (a) clj-ref run 1, + (b) fresh DB + clj-ref run 2 -> self-jitter envelope, (c) fresh DB + + clj-ref/py-shadow paired run with a restart seam, (d)/(e) envelope-aware + compare + verdict JSON + a terse (<=40-line) stdout summary. + + REQUIRES live Postgres + the ``clojure`` CLI — fails fast with a clear + message when either is unreachable (see ``preflight_check``).""" + default_cuts, default_seam = pe.default_full_run_schedule(dataset) + cut_slots = [int(c) for c in cuts.split(",") if c.strip()] if cuts else default_cuts + resolved_seam = seam_after if seam_after is not None else default_seam + + config = pe.FullRunConfig( + dataset=dataset, admin_url=admin_url, out_root=str(out_root), + cuts=tuple(cut_slots), seam_after=resolved_seam, dbname=dbname, zid=zid, + clj_env=clj_env, py_env=py_env, engine_mode=engine_mode, + poll_from_days_ago=poll_from_days_ago, wait_timeout=wait_timeout, + restart_clj_at_seam=restart_clj_at_seam, + wait_for_clj_poll_cycle=wait_for_clj_poll_cycle, + poll_cycle_gate_timeout=poll_cycle_gate_timeout, + ) + try: + verdict = pe.run_full_equiv_protocol(config) + except pe.PollerEquivStreamError as exc: + click.echo(f"poller-equiv full-run: ABORTED\n{exc}", err=True) + sys.exit(1) + for line in pe.render_full_run_lines(verdict): + click.echo(line) + sys.exit(0 if verdict["overall_pass"] else 1) + + +def _run_and_stream(runner: Any, *, label: str) -> None: + proc = runner.start() + click.echo(f"[{label}] started pid={proc.pid} cmd={' '.join(runner.cmd)}", err=True) + try: + if proc.stdout is not None: + for line in proc.stdout: + click.echo(f"[{label}] {line}", nl=False) + proc.wait() + except KeyboardInterrupt: + click.echo(f"\n[{label}] stopping…", err=True) + runner.kill() + sys.exit(proc.returncode or 0) + + +if __name__ == "__main__": + cli() diff --git a/delphi/tests/poller/test_load_or_init.py b/delphi/tests/poller/test_load_or_init.py index 6f19eaf083..7ab5560361 100644 --- a/delphi/tests/poller/test_load_or_init.py +++ b/delphi/tests/poller/test_load_or_init.py @@ -111,6 +111,42 @@ def test_warm_restore_then_full_rebuild(self): assert conv.raw_rating_mat.size > 0 assert conv.pca is not None + def test_cold_start_conversation_id_is_the_native_int_zid(self): + """2026-07-24 live finding (session 3): service.py used to construct + ``Conversation(str(zid), last_updated=1)`` — a Type mismatch against + Clojure's int zid showed up live as ``step_0.zid`` (and every + ``tids[i]``/``repness.*.tid``, fixed separately in postgres.py) in a + real vw poller-equivalence full-run. ``Conversation.__init__`` just + does a bare ``self.conversation_id = conversation_id`` (no + string-specific logic; grepped every ``.conversation_id`` use site — + the only ``str()`` casts are at the DynamoDB boundary, + database/dynamodb.py, which already handles either type + defensively), so passing the int through is a one-point fix.""" + pg = MagicMock() + pg.load_math_main.return_value = None + pg.poll_votes.return_value = _build_votes() + pg.poll_moderation.return_value = _empty_mods() + svc = MathPollerService(pg, PollerConfig()) + + conv = svc._load_or_init(42) + + assert conv.conversation_id == 42 + assert isinstance(conv.conversation_id, int) + + def test_cold_start_to_dict_zid_is_int(self): + """The observable, live-evidence-matching field: to_dict()['zid'] + (conversation.py:2379 renames conversation_id -> zid at emission).""" + pg = MagicMock() + pg.load_math_main.return_value = None + pg.poll_votes.return_value = _build_votes() + pg.poll_moderation.return_value = _empty_mods() + svc = MathPollerService(pg, PollerConfig()) + + conv = svc._load_or_init(42) + + assert conv.to_dict()["zid"] == 42 + assert isinstance(conv.to_dict()["zid"], int) + def test_from_dict_failure_falls_back_to_cold(self, monkeypatch): pg = MagicMock() pg.load_math_main.return_value = {"zid": 42, "data": {"garbage": object()}} diff --git a/delphi/tests/poller/test_math_writer.py b/delphi/tests/poller/test_math_writer.py index 2b66078a37..27846f8672 100644 --- a/delphi/tests/poller/test_math_writer.py +++ b/delphi/tests/poller/test_math_writer.py @@ -12,23 +12,29 @@ ONE shared math_tick (conv_man.clj:158-169). """ +import json +from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from polismath.poller.math_writer import derive_bidtopid, MathWriter +from polismath.poller.math_writer import derive_bidtopid, derive_ptptstats, MathWriter -def _fake_conv(zid=42, base_clusters=None, last_updated=1234567): +def _fake_conv(zid=42, base_clusters=None, last_updated=1234567, + group_clusters=None, proj=None): """Minimal stand-in exposing the public attributes the writer consumes.""" conv = SimpleNamespace() conv.conversation_id = str(zid) conv.last_updated = last_updated conv.base_clusters = base_clusters if base_clusters is not None else [] + conv.group_clusters = group_clusters if group_clusters is not None else [] + conv.proj = proj if proj is not None else {} conv.participant_info = {} conv.to_dict = lambda: {"base-clusters": {"id": [], "members": []}, - "lastVoteTimestamp": last_updated} + "lastVoteTimestamp": last_updated, + "user-vote-counts": {}} return conv @@ -61,6 +67,206 @@ def test_empty_base_clusters_gives_empty_bidToPid(self): assert derive_bidtopid(conv, 7)["bidToPid"] == [] +# --------------------------------------------------------------------------- # +# derive_ptptstats — REAL bug fix, 2026-07-24 (poller-equivalence harness live +# debugging session 2). Before this fix, derive_ptptstats just wrapped +# conv.participant_info (a Python-only, ROW-wise dict of n_agree/n_disagree/ +# n_pass/group_correlations — a COMPLETELY DIFFERENT statistic from Clojure's, +# not just a different shape). Clojure's prep-ptpt-stats (conv_man.clj:90-94) +# wraps a COLUMNAR {pid, gid, n-votes, centricness, coreness, extremeness} +# dict built by `columnize` (conv_man.clj:79-88) over +# repness/participant-stats (math/repness.clj:383-413) — a GEOMETRIC +# per-participant stat (distance-to-center in the PCA-projected plane), not a +# vote-correlation stat. Confirmed live: a real clj-ref math_ptptstats row +# (real_data/.local/replays/poller_equiv/vw/main/clj-ref/batch-000/ +# math_ptptstats.json) has EXACTLY these 6 keys, each a same-length array, +# with plain int pid/gid values — see TestDerivePtptstatsMatchesLiveClj below. +# --------------------------------------------------------------------------- # +class TestDerivePtptstatsMatchesLiveClj: + """Structural fidelity against the ACTUAL clj-ref row captured in the live + poller-equivalence store (2026-07-24 vw full-run — see the class + docstring above for the exact path).""" + + _LIVE_CLJ_PTPTSTATS_PATH = ( + Path(__file__).resolve().parents[2] + / "real_data" / ".local" / "replays" / "poller_equiv" / "vw" / "main" + / "clj-ref" / "batch-000" / "math_ptptstats.json" + ) + + def test_live_clj_row_has_the_expected_columnar_key_set(self): + """Sanity-checks the fixture itself is what this whole fix is based + on — if this ever fails, the live evidence path/shape changed and + the derive_ptptstats rewrite below needs re-deriving, not just this + assertion patched.""" + if not self._LIVE_CLJ_PTPTSTATS_PATH.exists(): + pytest.skip("live poller-equiv store not present in this checkout") + row = json.loads(self._LIVE_CLJ_PTPTSTATS_PATH.read_text()) + pt = row["data"]["ptptstats"] + assert set(pt.keys()) == {"pid", "gid", "n-votes", "centricness", "coreness", "extremeness"} + lengths = {len(v) for v in pt.values()} + assert len(lengths) == 1 # every column is the SAME length (positionally aligned) + assert all(isinstance(p, int) for p in pt["pid"]) + assert all(isinstance(g, int) for g in pt["gid"]) + + def test_derive_ptptstats_output_has_the_same_key_set(self): + conv = _fake_conv( + base_clusters=[{"id": 0, "members": [1]}, {"id": 1, "members": [2]}], + group_clusters=[{"id": 0, "members": [0, 1]}], + proj={1: [0.0, 0.0], 2: [1.0, 0.0]}, + ) + result = derive_ptptstats(conv, 7, user_vote_counts={1: 3, 2: 4}) + assert set(result["ptptstats"].keys()) == { + "pid", "gid", "n-votes", "centricness", "coreness", "extremeness", + } + + +class TestDerivePtptstatsMath: + """Hand-computed 3-participant / 2-group scenario — verifies the actual + geometry (repness/participant-stats, math/repness.clj:383-413): + + global_center = mean(proj) = mean([0,0], [2,0], [10,0]) = [4, 0] + group 0 = {pid 1, pid 2} (base clusters 0, 1) + center0 = mean([0,0], [2,0]) = [1, 0] + extreme_direction0 = normalise([1,0] - [4,0]) = normalise([-3,0]) = [-1, 0] + pid 1 @ [0,0]: centricness = 1 - |[0,0]-[4,0]| = 1-4 = -3 + coreness = 1 - |[0,0]-[1,0]| = 1-1 = 0 + extremeness = dot([0,0]-[1,0], [-1,0]) = dot([-1,0],[-1,0]) = 1 + pid 2 @ [2,0]: centricness = 1 - |[2,0]-[4,0]| = 1-2 = -1 + coreness = 1 - |[2,0]-[1,0]| = 1-1 = 0 + extremeness = dot([2,0]-[1,0], [-1,0]) = dot([1,0],[-1,0]) = -1 + group 1 = {pid 3} (base cluster 2) + center1 = mean([10,0]) = [10, 0] + extreme_direction1 = normalise([10,0]-[4,0]) = normalise([6,0]) = [1, 0] + pid 3 @ [10,0]: centricness = 1 - |[10,0]-[4,0]| = 1-6 = -5 + coreness = 1 - |[10,0]-[10,0]| = 1-0 = 1 + extremeness = dot([10,0]-[10,0], [1,0]) = dot([0,0],[1,0]) = 0 + """ + + def _conv(self): + return _fake_conv( + base_clusters=[ + {"id": 0, "members": [1]}, + {"id": 1, "members": [2]}, + {"id": 2, "members": [3]}, + ], + group_clusters=[ + {"id": 0, "members": [0, 1]}, # base clusters 0+1 -> pids 1,2 + {"id": 1, "members": [2]}, # base cluster 2 -> pid 3 + ], + proj={1: [0.0, 0.0], 2: [2.0, 0.0], 3: [10.0, 0.0]}, + ) + + def test_hand_computed_geometry_matches_exactly(self): + result = derive_ptptstats(self._conv(), 7, user_vote_counts={1: 5, 2: 7, 3: 9}) + pt = result["ptptstats"] + assert pt["pid"] == [1, 2, 3] + assert pt["gid"] == [0, 0, 1] + assert pt["n-votes"] == [5, 7, 9] + assert pt["centricness"] == pytest.approx([-3.0, -1.0, -5.0]) + assert pt["coreness"] == pytest.approx([0.0, 0.0, 1.0]) + assert pt["extremeness"] == pytest.approx([1.0, -1.0, 0.0]) + + def test_envelope_keys_are_zid_ptptstats_lastvotetimestamp(self): + result = derive_ptptstats(self._conv(), 7, user_vote_counts={1: 5, 2: 7, 3: 9}) + assert set(result.keys()) == {"zid", "ptptstats", "lastVoteTimestamp"} + assert result["zid"] == 7 + + def test_missing_vote_count_yields_none_matching_clojures_nil(self): + """Clojure's `(get ptpt-vote-counts pid)` returns nil for a pid not + in the map — mirror that as None, never a guessed 0.""" + result = derive_ptptstats(self._conv(), 7, user_vote_counts={1: 5}) # 2, 3 missing + assert result["ptptstats"]["n-votes"] == [5, None, None] + + def test_no_groups_gives_an_empty_ptptstats_dict_not_empty_arrays(self): + """Clojure's columnize on an empty stats seq returns `{}` (keys is + nil on an empty seq), NOT a dict of empty-array columns.""" + conv = _fake_conv(group_clusters=[], base_clusters=[], proj={}) + result = derive_ptptstats(conv, 7, user_vote_counts={}) + assert result["ptptstats"] == {} + + def test_no_proj_gives_an_empty_ptptstats_dict(self): + conv = _fake_conv( + group_clusters=[{"id": 0, "members": []}], base_clusters=[], proj={}, + ) + result = derive_ptptstats(conv, 7, user_vote_counts={}) + assert result["ptptstats"] == {} + + def test_group_member_not_in_proj_is_skipped_not_a_crash(self): + """A participant in a base-cluster's members but absent from `proj` + (e.g. transient state) must not raise — just excluded from stats.""" + conv = _fake_conv( + base_clusters=[{"id": 0, "members": [1, 99]}], # 99 has no proj entry + group_clusters=[{"id": 0, "members": [0]}], + proj={1: [0.0, 0.0]}, + ) + result = derive_ptptstats(conv, 7, user_vote_counts={1: 1}) + assert result["ptptstats"]["pid"] == [1] + + def test_single_group_zero_direction_matches_clj_zero_extremeness(self): + """The Q4-degenerate single-group case: ONE group covering every + participant makes group center == global center BY CONSTRUCTION, so + extreme-direction normalises a ZERO vector. #2657 review deduced from + vectorz `AVector.toNormal()` bytecode that Clojure would get nil and + crash — REFUTED empirically (2026-07-24, clojure -M on the pinned + stack, journal s5): `(mat/normalise (mat/matrix [0.0 0.0]))` returns + the ZERO VECTOR (not nil) and the extremeness dot is a clean 0.0. + Python's `direction/norm if norm > 0 else direction` therefore + MATCHES Clojure exactly here: extremeness 0.0 for every member, no + divergence, nothing to ledger. This test pins that agreement.""" + conv = _fake_conv( + base_clusters=[ + {"id": 0, "members": [1]}, + {"id": 1, "members": [2]}, + ], + group_clusters=[{"id": 0, "members": [0, 1]}], # ONE group = everyone + proj={1: [-1.0, 0.0], 2: [1.0, 0.0]}, # global center == group center == [0,0] + ) + result = derive_ptptstats(conv, 7, user_vote_counts={1: 2, 2: 2}) + stats = result["ptptstats"] + assert stats["pid"] == [1, 2] + assert stats["extremeness"] == [0.0, 0.0] + # centricness == coreness here (same center), sanity-pinning the geometry + assert stats["centricness"] == stats["coreness"] == [0.0, 0.0] + + +class TestDerivePtptstatsGroupOrderClojureHashMap: + """Clojure's `group-data` map (conv_man.clj's `(into {} ...)` over + group-clusters) is an ARRAY-map (insertion/group_clusters order) for <=8 + groups but a PersistentHashMap (HAMT id-hash order) for >8 — the EXACT + same threshold legacy_kmeans.py's cleared-clusters scan order already + documents and relies on (same clojure_hash_map_key_order utility).""" + + def test_at_most_8_groups_visited_in_group_clusters_order(self): + from polismath.utils.clj_hash import clojure_hash_map_key_order + + # 8 groups, deliberately NOT in id-ascending order. + ids = [7, 3, 5, 1, 8, 2, 6, 4] + assert clojure_hash_map_key_order(ids) != ids # sanity: hash order WOULD differ + base_clusters = [{"id": i, "members": [i]} for i in ids] + group_clusters = [{"id": gid, "members": [gid]} for gid in ids] + proj = {i: [float(i), 0.0] for i in ids} + conv = _fake_conv(base_clusters=base_clusters, group_clusters=group_clusters, proj=proj) + + result = derive_ptptstats(conv, 7, user_vote_counts={}) + # <=8 -> array-map -> INSERTION (group_clusters) order, not hash order. + assert result["ptptstats"]["gid"] == ids + + def test_more_than_8_groups_visited_in_clojure_hash_map_order(self): + from polismath.utils.clj_hash import clojure_hash_map_key_order + + ids = list(range(1, 10)) # 9 groups -> PersistentHashMap territory + expected_order = clojure_hash_map_key_order(ids) + assert expected_order != ids # sanity: this scenario actually exercises hash order + + base_clusters = [{"id": i, "members": [i]} for i in ids] + group_clusters = [{"id": gid, "members": [gid]} for gid in ids] + proj = {i: [float(i), 0.0] for i in ids} + conv = _fake_conv(base_clusters=base_clusters, group_clusters=group_clusters, proj=proj) + + result = derive_ptptstats(conv, 7, user_vote_counts={}) + assert result["ptptstats"]["gid"] == expected_order + + class TestMathWriterSharedTick: def test_all_writes_share_one_math_tick(self): client = MagicMock() @@ -95,6 +301,31 @@ def test_bidtopid_data_written_has_correct_shape(self): data = call.args[1] assert data["bidToPid"] == [["1", "2"]] + def test_ptptstats_written_uses_user_vote_counts_from_to_dict(self): + """write_conv_updates must thread data["user-vote-counts"] (already + computed once for math_main) into derive_ptptstats — not recompute + it, and not silently drop it (n-votes would be all-None otherwise).""" + client = MagicMock() + client.increment_math_tick.return_value = 1 + conv = _fake_conv( + zid=42, + base_clusters=[{"id": 0, "members": [1]}], + group_clusters=[{"id": 0, "members": [0]}], + proj={1: [0.0, 0.0]}, + ) + conv.to_dict = lambda: { + "base-clusters": {"id": [0], "members": [[1]]}, + "lastVoteTimestamp": 1234567, + "user-vote-counts": {1: 42}, + } + MathWriter(client).write_conv_updates(42, conv) + + call = client.write_participant_stats.call_args + data = call.kwargs.get("data") + if data is None: + data = call.args[1] + assert data["ptptstats"]["n-votes"] == [42] + class TestWriterSQLFidelity: """The Clojure-exact SQL lives in PostgresClient; verify text + params via a diff --git a/delphi/tests/poller/test_postgres_client_pid_types.py b/delphi/tests/poller/test_postgres_client_pid_types.py new file mode 100644 index 0000000000..d5a458404d --- /dev/null +++ b/delphi/tests/poller/test_postgres_client_pid_types.py @@ -0,0 +1,183 @@ +"""``PostgresClient.poll_votes`` / ``poll_votes_since`` / ``poll_moderation`` — +pid/tid TYPE parity with Clojure. + +ROOT CAUSE (found live, 2026-07-24 poller-equivalence harness debugging, +sessions 2-3): several live-poller ingress points cast ``str(...)`` on pid +and/or tid, while Clojure's poller holds the DB's native INTEGER ids +throughout. Session 2 fixed pid in ``poll_votes``/``poll_votes_since`` +(``math_main.base-clusters.members`` divergence). Session 3 (this file's +extension) fixes: + + * tid in ``poll_votes``/``poll_votes_since`` — was masked by the pid + divergence's sheer volume in the live diff; once pid was fixed, a live + vw full-run showed the SAME "Type mismatch: golden=int, current=str" + pattern on ``zid``, every ``tids[i]``, and every + ``repness.[i].tid``. + * tid AND pid in ``poll_moderation`` (the single-zid, full-moderation-state + variant used by ``update_moderation`` — NOT ``poll_moderation_since``, + which already returned ``int(m["tid"])``/``int(m["zid"])`` and was never + broken). Left unfixed, this would have been a LATENT regression + introduced BY the pid/tid ingress fixes above: + ``_apply_moderation`` (conversation.py) zeroes moderated-out comment + COLUMNS via ``[c for c in self.mod_out_tids if c in + self.rating_mat.columns]`` — with tid now int on the votes side but + still str from ``poll_moderation``, that intersection would ALWAYS be + empty, silently disabling comment moderation in the live poller. The + equivalent participant-ban check (``mod_out_ptpts``) is only exercised + in 'improved' engine mode (clojure-legacy intentionally leaks bans, + conversation.py's ``_apply_moderation`` docstring) but was fixed for the + same consistency reason. + +``Conversation.update_votes`` is deliberately type-agnostic at its ingress +(``ptpt_id = vote.get('pid')``/``comment_id = vote.get('tid') # Preserve +original type``, conversation.py) and ``raw_rating_mat``/``rating_mat`` are +ALWAYS rebuilt fresh from ``poll_votes``/``poll_votes_since`` on every +load-or-init (never restored via ``from_dict`` — see +``polismath/poller/__init__.py``'s "load-or-init finding" docstring) — so +removing these ``str()`` casts is a one-point (per site) fix with no other +code changes needed. The CSV/certify replay driver never cast pid OR tid at +all, and its blobs already matched clj int-for-int across 20 cross-validated +entries (the certified-battery evidence cited when this fix was authorized). + +NO live Postgres required — ``PostgresClient.query`` is monkeypatched to +return canned rows (mirrors tests/test_math_writer_numpy_serialization.py's +``_client_capturing()`` pattern), so this exercises the REAL row-mapping code +in postgres.py without a live DB. +""" + +from __future__ import annotations + +from polismath.database.postgres import PostgresClient, PostgresConfig + + +def _client_with_canned_rows(rows: list[dict]) -> PostgresClient: + client = PostgresClient(PostgresConfig(url="postgresql://ignored/db", math_env="t3")) + client.query = lambda sql, params=None: rows + return client + + +class TestPollVotesPidType: + def test_pid_and_tid_are_native_ints_matching_the_db_column_type(self): + """``votes.pid``/``votes.tid`` are INTEGER columns (migrations.sql) — + SQLAlchemy/psycopg2 already return native Python ints for them; this + just asserts poll_votes does NOT wrap EITHER in str() anymore.""" + client = _client_with_canned_rows( + [{"zid": 1, "tid": 10, "pid": 5, "vote": -1, "created": 1000}] + ) + votes = client.poll_votes(zid=1) + assert len(votes) == 1 + assert votes[0]["pid"] == 5 + assert isinstance(votes[0]["pid"], int) + assert votes[0]["tid"] == 10 + assert isinstance(votes[0]["tid"], int) + + def test_vote_sign_is_still_flipped_to_delphi_convention(self): + """The pid/tid-type fix must not disturb the (unrelated) sign flip + at the same ingress boundary.""" + client = _client_with_canned_rows( + [{"zid": 1, "tid": 10, "pid": 5, "vote": -1, "created": 1000}] # raw DB AGREE + ) + votes = client.poll_votes(zid=1) + assert votes[0]["vote"] == 1 # Delphi AGREE + + +class TestPollVotesSincePidType: + def test_pid_and_tid_are_native_ints(self): + client = _client_with_canned_rows( + [{"zid": 1, "tid": 10, "pid": 7, "vote": 1, "created": 2000}] + ) + votes = client.poll_votes_since(since=0) + assert len(votes) == 1 + assert votes[0]["pid"] == 7 + assert isinstance(votes[0]["pid"], int) + assert votes[0]["tid"] == 10 + assert isinstance(votes[0]["tid"], int) + + def test_zid_type_is_unchanged_already_int(self): + client = _client_with_canned_rows( + [{"zid": "1", "tid": 10, "pid": 7, "vote": 1, "created": 2000}] + ) + votes = client.poll_votes_since(since=0) + assert votes[0]["zid"] == 1 + assert isinstance(votes[0]["zid"], int) + + def test_multiple_rows_preserve_order_and_all_get_int_pids_and_tids(self): + client = _client_with_canned_rows([ + {"zid": 1, "tid": 10, "pid": 3, "vote": 1, "created": 1000}, + {"zid": 1, "tid": 11, "pid": 9, "vote": -1, "created": 1001}, + ]) + votes = client.poll_votes_since(since=0) + assert [v["pid"] for v in votes] == [3, 9] + assert [v["tid"] for v in votes] == [10, 11] + assert all(isinstance(v["pid"], int) and isinstance(v["tid"], int) for v in votes) + + +class TestPollModerationPidAndTidType: + """``poll_moderation`` (the single-zid, full-current-moderation-state + variant ``update_moderation`` consumes — NOT ``poll_moderation_since``, + the global-watermark variant, which already used int).""" + + def test_mod_out_tids_are_native_ints(self): + client = _client_with_canned_rows_for_moderation( + comments=[{"tid": 5, "modified": 100, "mod": -1, "is_meta": False}], + participants=[], + ) + result = client.poll_moderation(zid=1) + assert result["mod_out_tids"] == [5] + assert all(isinstance(t, int) for t in result["mod_out_tids"]) + + def test_mod_in_tids_are_native_ints(self): + client = _client_with_canned_rows_for_moderation( + comments=[{"tid": 7, "modified": 100, "mod": 1, "is_meta": False}], + participants=[], + ) + result = client.poll_moderation(zid=1) + assert result["mod_in_tids"] == [7] + assert all(isinstance(t, int) for t in result["mod_in_tids"]) + + def test_meta_tids_are_native_ints(self): + client = _client_with_canned_rows_for_moderation( + comments=[{"tid": 9, "modified": 100, "mod": 0, "is_meta": True}], + participants=[], + ) + result = client.poll_moderation(zid=1) + assert result["meta_tids"] == [9] + assert all(isinstance(t, int) for t in result["meta_tids"]) + + def test_mod_out_ptpts_are_native_ints(self): + client = _client_with_canned_rows_for_moderation( + comments=[], participants=[{"pid": 3}], + ) + result = client.poll_moderation(zid=1) + assert result["mod_out_ptpts"] == [3] + assert all(isinstance(p, int) for p in result["mod_out_ptpts"]) + + def test_string_valued_mod_column_still_recognized(self): + """The existing 'support for string values' branch (mod == "1" / + mod == "-1") must keep working — this fix only changes the id + TYPES, not the mod-value comparison logic.""" + client = _client_with_canned_rows_for_moderation( + comments=[{"tid": 4, "modified": 100, "mod": "-1", "is_meta": False}], + participants=[], + ) + result = client.poll_moderation(zid=1) + assert result["mod_out_tids"] == [4] + + +def _client_with_canned_rows_for_moderation(comments: list[dict], participants: list[dict]) -> PostgresClient: + """poll_moderation issues TWO queries (comments, then participants) — + this double-dispatches the monkeypatched ``query`` by SQL text, mirroring + the FROM-table matching convention already used elsewhere in this test + suite (e.g. poller_equiv.py's ``_FakeLoopConn``).""" + client = PostgresClient(PostgresConfig(url="postgresql://ignored/db", math_env="t3")) + + def fake_query(sql, params=None): + norm = " ".join(sql.lower().split()) + if "from comments" in norm: + return comments + if "from participants" in norm: + return participants + raise AssertionError(f"unexpected SQL in poll_moderation test: {sql!r}") + + client.query = fake_query + return client diff --git a/delphi/tests/replay_harness/test_poller_equiv_compare.py b/delphi/tests/replay_harness/test_poller_equiv_compare.py new file mode 100644 index 0000000000..74c4d91e4b --- /dev/null +++ b/delphi/tests/replay_harness/test_poller_equiv_compare.py @@ -0,0 +1,1155 @@ +"""Unit tests for the poller-equivalence harness — Stage C (feeder + +comparer), MATH_POLLER_EQUIV_SPEC.md. + +NO live Postgres/containers/subprocesses required: every test here is either +pure-Python, drives the on-disk snapshot store against ``tmp_path``, or drives +:func:`polismath.replay.poller_equiv.run_batch_loop` against fake connection / +runner doubles (mirrors the ``_FakeConn``/``_SequenceConn`` pattern already +used in ``test_poller_equiv_seed.py`` for :func:`wait_for_tick`). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from polismath.replay import poller_equiv as pe +from polismath.replay.types import CommentMeta, ReplayDataset, VoteEvent + + +# --------------------------------------------------------------------------- # +# Shared fixtures / doubles. +# --------------------------------------------------------------------------- # +def _dataset_no_revotes(n: int = 7) -> ReplayDataset: + """``n`` votes, each a DISTINCT (pid, tid) pair — no revotes — spread over + 3 participants and ``n`` distinct comments, one vote per millisecond.""" + raw = [(1000 + i, (i % 3) + 1, i, 1) for i in range(n)] + comments = {i: CommentMeta(tid=i, created_ms=900) for i in range(n)} + return ReplayDataset.build(raw, comments=comments) + + +def _dataset_with_revote() -> ReplayDataset: + """5 votes where vote index 3 is a REVOTE of vote index 0's (pid, tid).""" + raw = [ + (1000, 1, 10, 1), + (1001, 2, 11, 1), + (1002, 3, 12, 1), + (1003, 1, 10, -1), # revote: same (pid=1, tid=10) as the first vote + (1004, 2, 13, 1), + ] + comments = {tid: CommentMeta(tid=tid, created_ms=900) for tid in (10, 11, 12, 13)} + return ReplayDataset.build(raw, comments=comments) + + +def _math_main_blob( + *, pca_comp0: float = 1.0, user_vote_counts: dict[str, int] | None = None, + last_vote_ts: int = 1000, in_conv: list[int] | None = None, +) -> dict[str, Any]: + """A minimal math_main-shaped blob (prep-main key spelling — mirrors + ``test_certify.py``'s ``_acceptance_blob`` helper) with the two extra + keys Stage C's pure helpers read: ``lastVoteTimestamp``, + ``user-vote-counts``.""" + return { + "zid": 1, + "n": 2, + "n-cmts": 2, + "in-conv": in_conv if in_conv is not None else [1, 2], + "tids": [0, 1], + "pca": {"center": [0.1, 0.2], "comps": [[pca_comp0, 0.0], [0.0, 1.0]]}, + "base-clusters": { + "id": [0, 1], "x": [0.1, -0.1], "y": [0.2, -0.2], + "count": [1, 2], "members": [[1], [2, 3]], + }, + "repness": {}, + "lastVoteTimestamp": last_vote_ts, + "user-vote-counts": user_vote_counts if user_vote_counts is not None else {"1": 3}, + } + + +def _row(data: dict[str, Any], *, last_vote_timestamp: int = 1000, caching_tick: int = 1, + math_tick: int = 1) -> dict[str, Any]: + """A math_main-table row shape (the dict :func:`wait_for_tick` / + :func:`fetch_math_row` return).""" + return { + "zid": 1, "math_env": "e", "data": data, + "last_vote_timestamp": last_vote_timestamp, + "caching_tick": caching_tick, "math_tick": math_tick, "modified": 123, + } + + +# --------------------------------------------------------------------------- # +# blob_total_votes — pure, against the DOCUMENTED shapes (see the function's +# own docstring for the real recording numbers this was verified against). +# --------------------------------------------------------------------------- # +class TestBlobTotalVotes: + def test_sums_user_vote_counts(self): + blob = {"user-vote-counts": {"1": 5, "2": 3, "3": 0}} + assert pe.blob_total_votes(blob) == 8 + + def test_falls_back_to_vote_stats_n_votes_when_user_vote_counts_absent(self): + blob = {"vote_stats": {"n_votes": 42}} + assert pe.blob_total_votes(blob) == 42 + + def test_prefers_user_vote_counts_over_vote_stats(self): + blob = {"user-vote-counts": {"1": 8}, "vote_stats": {"n_votes": 999}} + assert pe.blob_total_votes(blob) == 8 + + def test_votes_base_bucket_form_is_never_used_as_the_primary_signal(self): + """Regression guard for the clojure-legacy undercount bug (module + docstring, FP-81fda13ef6): even if 'votes-base' is present, it must + NOT be summed — only 'user-vote-counts' (or the vote_stats fallback) + may drive the result.""" + blob = { + "votes-base": {"0": {"A": [1], "D": [0], "S": [1]}}, # would sum to 1 + "user-vote-counts": {"1": 5, "2": 3}, # true total: 8 + } + assert pe.blob_total_votes(blob) == 8 + + def test_returns_none_when_neither_key_present(self): + assert pe.blob_total_votes({"some": "other-blob"}) is None + + def test_returns_none_for_non_dict_blob(self): + assert pe.blob_total_votes(None) is None + assert pe.blob_total_votes("not-a-blob") is None + + def test_returns_none_on_malformed_user_vote_counts(self): + assert pe.blob_total_votes({"user-vote-counts": {"1": "not-a-number"}}) is None + + +# --------------------------------------------------------------------------- # +# expected_cumulative_vote_count — pure, revote-aware. +# --------------------------------------------------------------------------- # +class TestExpectedCumulativeVoteCount: + def test_no_revotes_matches_slot_count(self): + ds = _dataset_no_revotes(7) + assert pe.expected_cumulative_vote_count(ds, 3) == 3 + assert pe.expected_cumulative_vote_count(ds, 7) == 7 + + def test_revote_does_not_inflate_the_count(self): + ds = _dataset_with_revote() + # Prefix of 4 votes includes the revote at index 3 (0-based) -> only + # 3 DISTINCT (pid, tid) pairs so far ((1,10), (2,11), (3,12)); the + # revote re-touches (1,10), not a new pair. + assert pe.expected_cumulative_vote_count(ds, 4) == 3 + # Full 5-vote prefix adds one more distinct pair, (2,13). + assert pe.expected_cumulative_vote_count(ds, 5) == 4 + + def test_zero_slot_is_zero(self): + ds = _dataset_no_revotes(7) + assert pe.expected_cumulative_vote_count(ds, 0) == 0 + + def test_matches_committed_vw_recording(self): + """Verified against the committed vw recording (see blob_total_votes' + docstring): step-000/001/002 cut_slots 585/1171/1756 have + blob_total_votes 585/1169/1754 — i.e. 2 revotes land in the + (585, 1171] range.""" + path = Path("real_data/.local/replays/vw/uniform8-clojure-legacy/py") + if not path.is_dir(): + pytest.skip("vw certified recording not present in this checkout") + from polismath.replay.real_data import load_export_votes + + ds = load_export_votes("vw") + for step_file, expected in ( + ("step-000.json", 585), ("step-001.json", 1169), ("step-002.json", 1754), + ): + payload = json.loads((path / step_file).read_text()) + cut_slot = payload["cut_slot"] + assert pe.expected_cumulative_vote_count(ds, cut_slot) == expected + + +# --------------------------------------------------------------------------- # +# make_batch_ready_predicate — pure. +# --------------------------------------------------------------------------- # +class TestMakeBatchReadyPredicate: + def _predicate(self, **kwargs): + return pe.make_batch_ready_predicate(min_last_vote_ts=1000, min_vote_count=5, **kwargs) + + def test_none_row_never_ready(self): + pred = self._predicate() + assert pred(None) is False + + def test_row_missing_data_never_ready(self): + pred = self._predicate() + assert pred({"last_vote_timestamp": 2000}) is False + + def test_timestamp_not_yet_reached(self): + pred = pe.make_batch_ready_predicate(min_last_vote_ts=2000, min_vote_count=1) + row = _row(_math_main_blob(last_vote_ts=1999, user_vote_counts={"1": 100})) + assert pred(row) is False + + def test_vote_count_not_yet_advanced(self): + pred = pe.make_batch_ready_predicate(min_last_vote_ts=1000, min_vote_count=10) + row = _row(_math_main_blob(last_vote_ts=2000, user_vote_counts={"1": 3})) + assert pred(row) is False + + def test_both_conditions_satisfied(self): + pred = pe.make_batch_ready_predicate(min_last_vote_ts=1000, min_vote_count=5) + row = _row(_math_main_blob(last_vote_ts=1000, user_vote_counts={"1": 5})) + assert pred(row) is True + + def test_falls_back_to_persisted_last_vote_timestamp_column(self): + """When the blob itself carries no lastVoteTimestamp, the persisted + column is used instead (defensive fallback).""" + blob = {"user-vote-counts": {"1": 5}} + row = _row(blob, last_vote_timestamp=1000) + pred = pe.make_batch_ready_predicate(min_last_vote_ts=1000, min_vote_count=5) + assert pred(row) is True + + def test_exact_boundary_values_are_ready(self): + pred = pe.make_batch_ready_predicate(min_last_vote_ts=1000, min_vote_count=5) + row = _row(_math_main_blob(last_vote_ts=1000, user_vote_counts={"1": 5})) + assert pred(row) is True + + +# --------------------------------------------------------------------------- # +# batch_slices — pure, edge cases per the task's explicit ask. +# --------------------------------------------------------------------------- # +class TestBatchSlices: + def test_empty_cuts_yields_no_batches(self): + assert pe.batch_slices([]) == [] + + def test_first_batch_starts_at_slot_zero(self): + slices = pe.batch_slices([3, 7, 10]) + assert slices[0] == (0, 3) + + def test_final_batch_runs_to_n_when_n_is_the_last_cut(self): + n = 10 + slices = pe.batch_slices([3, 7, n]) + assert slices[-1] == (7, n) + + def test_single_cut_is_one_batch_from_zero(self): + assert pe.batch_slices([5]) == [(0, 5)] + + def test_multiple_cuts_chain_correctly(self): + assert pe.batch_slices([2, 5, 9]) == [(0, 2), (2, 5), (5, 9)] + + def test_non_increasing_cuts_raise(self): + with pytest.raises(ValueError, match="strictly increasing"): + pe.batch_slices([5, 5]) + + def test_decreasing_cuts_raise(self): + with pytest.raises(ValueError, match="strictly increasing"): + pe.batch_slices([5, 3]) + + def test_zero_cut_raises(self): + with pytest.raises(ValueError, match="strictly increasing"): + pe.batch_slices([0]) + + +# --------------------------------------------------------------------------- # +# snap_cuts_past_timestamp_ties — ROOT CAUSE #4 (2026-07-24 live-debug task): +# both pollers watermark on STRICT `created > ts` (postgres.clj's global vote +# poll / postgres.py's poll_votes_since, byte-identical per the module +# docstring). A batch cut that falls INSIDE a run of votes sharing the exact +# same `created` millisecond makes the tail of that run PERMANENTLY +# unreachable for BOTH engines — verified live: clj-ref undercounted the +# SAME 3 (pid, tid) pairs (pid=2/tid=43, pid=17/tid=11, pid=22/tid=22, all +# sharing t_ms=1732028794000 with the vw dataset's cut=585 boundary vote) +# its own watermark made unreachable, stalling the feeder's readiness +# predicate forever (its target vote count assumed every vote up to the cut +# was reachable). NOT a clj-vs-py divergence — both engines drop the exact +# same votes, identically, by construction (same SQL, same watermark) — but +# a structurally unreachable target for the harness's OWN readiness +# predicate, which this pure cut-adjustment function fixes at the source. +# --------------------------------------------------------------------------- # +def _ties_dataset(raw: list[tuple[int, int, int, int]]) -> ReplayDataset: + comments = {tid: CommentMeta(tid=tid, created_ms=900) for (_, _, tid, _) in raw} + return ReplayDataset.build(raw, comments=comments) + + +class TestSnapCutsPastTimestampTies: + def test_no_ties_leaves_cuts_unchanged(self): + raw = [(1000 + i, (i % 3) + 1, i, 1) for i in range(10)] + ds = _ties_dataset(raw) + assert pe.snap_cuts_past_timestamp_ties(ds, [3, 7, 10]) == [3, 7, 10] + + def test_cut_inside_a_tie_cluster_is_pushed_past_it(self): + # votes[2..4] all share t_ms=1002 -> cutting at 3 splits the tie. + raw = [ + (1000, 1, 0, 1), (1001, 2, 1, 1), + (1002, 3, 2, 1), (1002, 1, 3, 1), (1002, 2, 4, 1), + (1003, 3, 5, 1), + ] + ds = _ties_dataset(raw) + assert pe.snap_cuts_past_timestamp_ties(ds, [3]) == [5] + + def test_cut_already_at_a_tie_boundary_is_unchanged(self): + raw = [ + (1000, 1, 0, 1), (1002, 2, 1, 1), (1002, 3, 2, 1), + (1003, 1, 3, 1), + ] + ds = _ties_dataset(raw) + assert pe.snap_cuts_past_timestamp_ties(ds, [3]) == [3] + + def test_final_cut_at_dataset_length_is_never_adjusted(self): + """The dataset's total vote count as the last cut (batch_slices' + 'final batch to n' convention) has no 'next' vote to tie against — + must stay exactly n, never grow past the dataset.""" + raw = [(1000, 1, 0, 1), (1000, 2, 1, 1), (1000, 3, 2, 1)] + ds = _ties_dataset(raw) + assert pe.snap_cuts_past_timestamp_ties(ds, [3]) == [3] + + def test_multiple_cuts_each_independently_snapped(self): + raw = [ + (1000, 1, 0, 1), + (1001, 2, 1, 1), (1001, 3, 2, 1), + (1002, 1, 3, 1), + (1003, 2, 4, 1), (1003, 3, 5, 1), + (1004, 1, 6, 1), + ] + ds = _ties_dataset(raw) + assert pe.snap_cuts_past_timestamp_ties(ds, [2, 5]) == [3, 6] + + def test_never_moves_a_cut_backward(self): + raw = [(1000 + i, (i % 3) + 1, i, 1) for i in range(10)] + ds = _ties_dataset(raw) + adjusted = pe.snap_cuts_past_timestamp_ties(ds, [3, 7, 10]) + assert all(a >= c for a, c in zip(adjusted, [3, 7, 10])) + + +# --------------------------------------------------------------------------- # +# strictly_increasing — pure. +# --------------------------------------------------------------------------- # +class TestStrictlyIncreasing: + def test_strictly_increasing_sequence_passes(self): + result = pe.strictly_increasing([1, 2, 3, 4]) + assert result["strictly_increasing"] is True + assert result["violations"] == [] + + def test_single_value_trivially_passes(self): + assert pe.strictly_increasing([1])["strictly_increasing"] is True + + def test_empty_trivially_passes(self): + assert pe.strictly_increasing([])["strictly_increasing"] is True + + def test_flat_sequence_fails(self): + result = pe.strictly_increasing([1, 1, 2]) + assert result["strictly_increasing"] is False + assert result["violations"] == [{"index": 1, "prev": 1, "next": 1}] + + def test_decreasing_pair_fails(self): + result = pe.strictly_increasing([3, 2]) + assert result["strictly_increasing"] is False + + def test_none_entry_is_a_violation(self): + result = pe.strictly_increasing([1, None, 3]) + assert result["strictly_increasing"] is False + indices = [v["index"] for v in result["violations"]] + assert 1 in indices and 2 in indices + + +# --------------------------------------------------------------------------- # +# Snapshot store — round trip against tmp_path (no DB). +# --------------------------------------------------------------------------- # +class TestSnapshotStore: + def test_write_then_load_round_trips(self, tmp_path): + row = _row(_math_main_blob()) + pe.write_snapshot(tmp_path, "clj-ref", 0, "math_main", row) + loaded = pe.load_snapshot(tmp_path, "clj-ref", 0, "math_main") + assert loaded == row + + def test_load_missing_snapshot_is_none(self, tmp_path): + assert pe.load_snapshot(tmp_path, "clj-ref", 0, "math_main") is None + + def test_snapshot_path_rejects_unknown_table(self, tmp_path): + with pytest.raises(ValueError, match="unknown equiv table"): + pe.snapshot_path(tmp_path, "clj-ref", 0, "not_a_table") + + def test_snapshot_dir_layout(self, tmp_path): + d = pe.snapshot_dir(tmp_path, "py-shadow", 5) + assert d == tmp_path / "py-shadow" / "batch-005" + + def test_snapshot_path_rejects_unsafe_math_env(self, tmp_path): + with pytest.raises(ValueError): + pe.snapshot_path(tmp_path, "../escape", 0, "math_main") + + def test_discover_batches_sorted_and_filtered(self, tmp_path): + for i in (2, 0, 1): + pe.write_snapshot(tmp_path, "clj-ref", i, "math_main", _row(_math_main_blob())) + assert pe.discover_batches(tmp_path, "clj-ref") == [0, 1, 2] + + def test_discover_batches_empty_when_env_dir_absent(self, tmp_path): + assert pe.discover_batches(tmp_path, "nonexistent-env") == [] + + def test_write_then_load_manifest_round_trips(self, tmp_path): + manifest = {"zid": 1, "math_envs": ["clj-ref", "py-shadow"], "batches": [{"index": 0}]} + pe.write_manifest(tmp_path, manifest) + assert pe.load_manifest(tmp_path) == manifest + + def test_load_manifest_missing_is_none(self, tmp_path): + assert pe.load_manifest(tmp_path) is None + + +# --------------------------------------------------------------------------- # +# fetch_math_row — fake conn double (no real DB). +# --------------------------------------------------------------------------- # +class _FakeResult: + def __init__(self, row): + self._row = row + + def mappings(self): + return self + + def first(self): + return self._row + + +class _CapturingConn: + def __init__(self, row=None): + self.row = row + self.calls: list[tuple[str, dict]] = [] + + def execute(self, stmt, params=None): + self.calls.append((str(stmt), dict(params or {}))) + return _FakeResult(self.row) + + +class TestFetchMathRow: + def test_rejects_unknown_table(self): + with pytest.raises(ValueError, match="unknown equiv table"): + pe.fetch_math_row(_CapturingConn(), "worker_tasks", zid=1, math_env="e") + + def test_builds_query_for_known_table(self): + conn = _CapturingConn(row={"zid": 1, "data": {}}) + row = pe.fetch_math_row(conn, "math_bidtopid", zid=1, math_env="py-shadow") + assert row == {"zid": 1, "data": {}} + sql, params = conn.calls[0] + assert "FROM math_bidtopid" in sql + assert params == {"zid": 1, "math_env": "py-shadow"} + + def test_returns_none_when_no_row(self): + conn = _CapturingConn(row=None) + assert pe.fetch_math_row(conn, "math_ptptstats", zid=1, math_env="e") is None + + +# --------------------------------------------------------------------------- # +# compare_bidtopid — EXACT equality modulo the documented pid int/str +# representational difference. +# --------------------------------------------------------------------------- # +class TestCompareBidtopid: + def test_match_after_normalizing_pid_types_and_member_order(self): + a = {"zid": 1, "bidToPid": [[1, 2], [3]], "lastVoteTimestamp": 1000} + b = {"zid": 1, "bidToPid": [["2", "1"], ["3"]], "lastVoteTimestamp": 1000} + result = pe.compare_bidtopid(a, b) + assert result["match"] is True + + def test_mismatch_on_different_membership(self): + a = {"zid": 1, "bidToPid": [[1, 2]], "lastVoteTimestamp": 1000} + b = {"zid": 1, "bidToPid": [[1, 2, 3]], "lastVoteTimestamp": 1000} + assert pe.compare_bidtopid(a, b)["match"] is False + + def test_mismatch_on_different_last_vote_timestamp(self): + a = {"zid": 1, "bidToPid": [[1]], "lastVoteTimestamp": 1000} + b = {"zid": 1, "bidToPid": [[1]], "lastVoteTimestamp": 2000} + assert pe.compare_bidtopid(a, b)["match"] is False + + def test_bid_group_order_is_preserved_not_sorted(self): + """Outer bid ORDER (base-cluster id order) is meaningful and must NOT + be silently reordered — only within-group membership is set-like.""" + a = {"zid": 1, "bidToPid": [[1], [2]], "lastVoteTimestamp": 1000} + b = {"zid": 1, "bidToPid": [[2], [1]], "lastVoteTimestamp": 1000} + assert pe.compare_bidtopid(a, b)["match"] is False + + +# --------------------------------------------------------------------------- # +# compare_batch / compare_snapshots — canned row fixtures on disk (spec Stage +# C item 3's explicit scenario list): match, float-within-tolerance, +# structural mismatch, caching_tick regression, double-processed votes. +# --------------------------------------------------------------------------- # +class TestCompareBatch: + ENVS = ("clj-ref", "py-shadow") + + def _seed_batch(self, tmp_path, index, *, main_a, main_b, bid_a=None, bid_b=None, + pt_a=None, pt_b=None): + env_a, env_b = self.ENVS + pe.write_snapshot(tmp_path, env_a, index, "math_main", main_a) + pe.write_snapshot(tmp_path, env_b, index, "math_main", main_b) + pe.write_snapshot(tmp_path, env_a, index, "math_bidtopid", + bid_a or _row({"zid": 1, "bidToPid": [[1]], "lastVoteTimestamp": 1000})) + pe.write_snapshot(tmp_path, env_b, index, "math_bidtopid", + bid_b or _row({"zid": 1, "bidToPid": [[1]], "lastVoteTimestamp": 1000})) + pe.write_snapshot(tmp_path, env_a, index, "math_ptptstats", + pt_a or _row({"zid": 1, "ptptstats": {}, "lastVoteTimestamp": 1000})) + pe.write_snapshot(tmp_path, env_b, index, "math_ptptstats", + pt_b or _row({"zid": 1, "ptptstats": {}, "lastVoteTimestamp": 1000})) + + def test_match_scenario(self, tmp_path): + blob = _math_main_blob(user_vote_counts={"1": 5}) + self._seed_batch(tmp_path, 0, main_a=_row(blob), main_b=_row(blob)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 5}]}) + + result = pe.compare_batch(tmp_path, 0, self.ENVS) + assert result["tables"]["math_main"]["match"] is True + assert result["tables"]["math_bidtopid"]["match"] is True + assert result["tables"]["math_ptptstats"]["match"] is True + assert result["watermark"]["ok"] is True + + def test_float_within_tolerance_scenario(self, tmp_path): + blob_a = _math_main_blob(pca_comp0=1.0, user_vote_counts={"1": 5}) + blob_b = _math_main_blob(pca_comp0=1.0 + 1e-9, user_vote_counts={"1": 5}) + self._seed_batch(tmp_path, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 5}]}) + + result = pe.compare_batch(tmp_path, 0, self.ENVS) + assert result["tables"]["math_main"]["match"] is True + assert result["tables"]["math_main"]["n_divergences"] == 0 + + def test_structural_mismatch_scenario(self, tmp_path): + blob_a = _math_main_blob(in_conv=[1, 2], user_vote_counts={"1": 5}) + blob_b = _math_main_blob(in_conv=[1, 2, 3], user_vote_counts={"1": 5}) + self._seed_batch(tmp_path, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 5}]}) + + result = pe.compare_batch(tmp_path, 0, self.ENVS) + assert result["tables"]["math_main"]["match"] is False + assert result["tables"]["math_main"]["n_divergences"] >= 1 + assert any( + d["path"] == "step_0.in-conv" for d in result["tables"]["math_main"]["families"]["exact"] + ) + + def test_double_processed_votes_scenario_watermark_mismatch(self, tmp_path): + """Both envs report MORE votes than the manifest's expected count for + this batch — the observable signature of double-processing (or a + vote-counting bug).""" + blob_a = _math_main_blob(user_vote_counts={"1": 9}) + blob_b = _math_main_blob(user_vote_counts={"1": 9}) + self._seed_batch(tmp_path, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 5}]}) + + result = pe.compare_batch(tmp_path, 0, self.ENVS) + # The blobs are otherwise IDENTICAL -> math_main itself still "matches" + # structurally; the double-processing signature is caught SEPARATELY + # by the watermark check. + assert result["tables"]["math_main"]["match"] is True + assert result["watermark"]["ok"] is False + assert result["watermark"]["expected"] == 5 + assert result["watermark"][self.ENVS[0]] == 9 + + def test_missing_snapshot_is_reported_not_raised(self, tmp_path): + blob = _math_main_blob() + pe.write_snapshot(tmp_path, self.ENVS[0], 0, "math_main", _row(blob)) + # env_b's math_main snapshot was never written. + result = pe.compare_batch(tmp_path, 0, self.ENVS) + assert result["tables"]["math_main"]["match"] is False + assert result["tables"]["math_main"]["reason"] == "missing-snapshot" + assert self.ENVS[1] in result["tables"]["math_main"]["missing"] + + +class TestCompareSnapshots: + ENVS = ("clj-ref", "py-shadow") + + def _write_batch(self, tmp_path, index, *, caching_tick_a, caching_tick_b, + math_tick_a=None, math_tick_b=None, vote_count=5): + env_a, env_b = self.ENVS + blob = _math_main_blob(user_vote_counts={"1": vote_count}) + pe.write_snapshot(tmp_path, env_a, index, "math_main", + _row(blob, caching_tick=caching_tick_a, + math_tick=math_tick_a if math_tick_a is not None else caching_tick_a)) + pe.write_snapshot(tmp_path, env_b, index, "math_main", + _row(blob, caching_tick=caching_tick_b, + math_tick=math_tick_b if math_tick_b is not None else caching_tick_b)) + for table in ("math_bidtopid", "math_ptptstats"): + for env in (env_a, env_b): + pe.write_snapshot( + tmp_path, env, index, table, + _row({"zid": 1, "ptptstats": {}, "bidToPid": [[1]], + "lastVoteTimestamp": 1000}), + ) + + def test_overall_match_true_for_a_clean_two_batch_run(self, tmp_path): + self._write_batch(tmp_path, 0, caching_tick_a=1, caching_tick_b=1, vote_count=3) + self._write_batch(tmp_path, 1, caching_tick_a=2, caching_tick_b=2, vote_count=5) + pe.write_manifest(tmp_path, {"batches": [ + {"expected_vote_count": 3}, {"expected_vote_count": 5}, + ]}) + + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + assert report["overall_match"] is True + assert report["n_batches_aligned"] == 2 + assert pe.compare_exit_code(report) == 0 + + def test_caching_tick_regression_flips_overall_match_false(self, tmp_path): + """spec §1 'caching_tick strictly increasing per env' — a REGRESSION + (batch 1's caching_tick <= batch 0's, for one env) must be caught.""" + self._write_batch(tmp_path, 0, caching_tick_a=5, caching_tick_b=1, vote_count=3) + self._write_batch(tmp_path, 1, caching_tick_a=3, caching_tick_b=2, vote_count=5) # env_a regresses 5->3 + pe.write_manifest(tmp_path, {"batches": [ + {"expected_vote_count": 3}, {"expected_vote_count": 5}, + ]}) + + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + assert report["ticks"][self.ENVS[0]]["caching_tick"]["strictly_increasing"] is False + assert report["ticks"][self.ENVS[1]]["caching_tick"]["strictly_increasing"] is True + assert report["overall_match"] is False + assert pe.compare_exit_code(report) == 1 + + def test_batches_only_in_one_env_are_reported_and_excluded_from_overall_match(self, tmp_path): + env_a, env_b = self.ENVS + blob = _math_main_blob(user_vote_counts={"1": 3}) + pe.write_snapshot(tmp_path, env_a, 0, "math_main", _row(blob)) + # env_b never got batch 0 at all (e.g. it timed out). + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + assert report["batches_only_in"][env_a] == [0] + assert report["n_batches_aligned"] == 0 + assert report["overall_match"] is False + + # ----------------------------------------------------------------- # + # NO-COVERAGE GUARD — REQUIRED FIX #1 (2026-07-24 live-debug task): + # compare_snapshots must FAIL when aligned batches == 0, when ANY batch + # in the manifest is explicitly marked ready=False for either env, or + # when a snapshot store is completely empty. A vacuous pass — EXACTLY + # what the 2026-07-24 live full-run produced ("0 aligned batches" -> + # PASS) — must be structurally impossible. + # ----------------------------------------------------------------- # + def test_zero_aligned_batches_in_both_empty_stores_forces_overall_match_false(self, tmp_path): + """The EXACT bug reproduced: nothing was ever written to either + env's store (both runners crashed at startup) -> the pre-fix + all([])==True vacuous logic reported a spurious MATCH.""" + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + assert report["n_batches_aligned"] == 0 + assert report["overall_match"] is False + assert report["coverage"]["ok"] is False + assert pe.compare_exit_code(report) == 1 + + def test_manifest_ready_false_for_a_batch_missing_from_both_stores_fails_coverage(self, tmp_path): + """Batch 0 is a clean, fully-matching aligned batch (would ALONE + report overall_match=True under the pre-fix logic). Batch 1 timed + out for BOTH envs (per the manifest) and therefore has NO snapshot + in EITHER store — invisible to the old only_a/only_b logic, since a + batch missing from both stores never appears as 'only in one env'. + This is the residual vacuous-pass shape the aligned==0 guard alone + does not catch.""" + self._write_batch(tmp_path, 0, caching_tick_a=1, caching_tick_b=1, vote_count=3) + pe.write_manifest(tmp_path, {"batches": [ + {"index": 0, "expected_vote_count": 3, + "envs": {self.ENVS[0]: {"ready": True}, self.ENVS[1]: {"ready": True}}}, + {"index": 1, "expected_vote_count": 5, + "envs": {self.ENVS[0]: {"ready": False}, self.ENVS[1]: {"ready": False}}}, + ]}) + + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + assert report["n_batches_aligned"] == 1 # batch 0 only + assert report["coverage"]["ok"] is False + assert any(nr["batch"] == 1 for nr in report["coverage"]["not_ready"]) + assert report["overall_match"] is False + + def test_clean_manifest_with_explicit_ready_true_passes_coverage(self, tmp_path): + self._write_batch(tmp_path, 0, caching_tick_a=1, caching_tick_b=1, vote_count=3) + pe.write_manifest(tmp_path, {"batches": [ + {"index": 0, "expected_vote_count": 3, + "envs": {self.ENVS[0]: {"ready": True}, self.ENVS[1]: {"ready": True}}}, + ]}) + + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + assert report["coverage"]["ok"] is True + assert report["overall_match"] is True + + def test_write_compare_verdict_writes_json(self, tmp_path): + self._write_batch(tmp_path, 0, caching_tick_a=1, caching_tick_b=1, vote_count=3) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + path = pe.write_compare_verdict(report, tmp_path) + assert path == tmp_path / "compare_verdict.json" + assert json.loads(path.read_text())["overall_match"] == report["overall_match"] + + +# --------------------------------------------------------------------------- # +# render_compare_lines — pure rendering, ≤40-line contract. +# --------------------------------------------------------------------------- # +class TestRenderCompareLines: + def _report(self, n_batches: int, *, overall_match: bool = True) -> dict: + per_batch = [ + {"batch": i, "tables": {"math_main": {"match": True}, "math_bidtopid": {"match": True}, + "math_ptptstats": {"match": True}}, + "watermark": {"ok": True}} + for i in range(n_batches) + ] + return { + "n_batches_aligned": n_batches, "math_envs": ["clj-ref", "py-shadow"], + "batches_only_in": {"clj-ref": [], "py-shadow": []}, + "per_batch": per_batch, + "ticks": { + "clj-ref": {"caching_tick": {"strictly_increasing": True}, + "math_tick": {"strictly_increasing": True}}, + "py-shadow": {"caching_tick": {"strictly_increasing": True}, + "math_tick": {"strictly_increasing": True}}, + }, + "overall_match": overall_match, + } + + def test_small_report_fits_without_truncation(self): + lines = pe.render_compare_lines(self._report(3)) + assert len(lines) <= 40 + assert any("MATCH" in line for line in lines) + assert lines[-1].startswith("verdict: MATCH") + + def test_large_report_is_truncated_to_max_lines(self): + lines = pe.render_compare_lines(self._report(100), max_lines=40) + assert len(lines) <= 40 + assert any("more batches" in line for line in lines) + + def test_divergence_verdict_shown_in_footer(self): + lines = pe.render_compare_lines(self._report(2, overall_match=False)) + assert lines[-1].startswith("verdict: DIVERGENCE") + + def test_failing_batch_line_names_bad_tables(self): + report = self._report(1) + report["per_batch"][0]["tables"]["math_bidtopid"]["match"] = False + lines = pe.render_compare_lines(report) + assert any("FAIL" in line and "math_bidtopid" in line for line in lines) + + def test_watermark_mismatch_flagged_in_batch_line(self): + report = self._report(1) + report["per_batch"][0]["watermark"]["ok"] = False + lines = pe.render_compare_lines(report) + assert any("WATERMARK-MISMATCH" in line for line in lines) + + def test_missing_coverage_key_does_not_raise(self): + """Fixture-shaped reports without a 'coverage' key (pre-guard shape, + or any caller that never populated it) must still render — the + renderer must not assume the key exists.""" + report = self._report(1) + assert "coverage" not in report + lines = pe.render_compare_lines(report) + assert len(lines) <= 40 + + def test_coverage_failure_is_flagged_loudly(self): + report = self._report(1) + report["overall_match"] = False + report["coverage"] = { + "ok": False, "not_ready": [{"batch": 1, "env": "clj-ref"}], + "empty_stores": [], "manifest_present": True, "n_manifest_batches": 2, + } + lines = pe.render_compare_lines(report) + assert any("COVERAGE" in line for line in lines) + assert len(lines) <= 40 + + +# --------------------------------------------------------------------------- # +# run_batch_loop — the LIVE orchestration loop, exercised with fake +# conn/runners/insert_fn doubles (no DB, no subprocess). +# --------------------------------------------------------------------------- # +class _FakeLoopConn: + """Serves a pre-programmed row per (table, math_env) for every SELECT — + ``run_batch_loop`` never issues raw vote INSERTs itself (that's + ``insert_fn``'s job, faked separately below), so this only needs to + answer :func:`wait_for_tick`'s math_main poll and :func:`fetch_math_row`'s + bidtopid/ptptstats reads.""" + + def __init__(self): + self._rows: dict[tuple[str, str], dict] = {} + + def set_row(self, table: str, math_env: str, row: dict) -> None: + self._rows[(table, math_env)] = row + + def execute(self, stmt, params=None): + text = str(stmt) + params = params or {} + math_env = params.get("math_env") + for table in pe.EQUIV_TABLES: + if f"FROM {table} " in text: + return _FakeResult(self._rows.get((table, math_env))) + raise AssertionError(f"unexpected SQL issued to fake loop conn: {text!r}") + + +class _FakeRunner: + def __init__(self, name: str, *, log_path=None): + self.name = name + self.started = False + self.killed = False + self.log_path = log_path + + def start(self): + self.started = True + return self + + def kill(self, grace: float = 5.0): + self.killed = True + + +class TestRunBatchLoop: + ENVS = ("clj-ref", "py-shadow") + + def _ready_row(self, env: str, *, vote_count: int, ts: int, tick: int) -> dict: + blob = _math_main_blob(user_vote_counts={"1": vote_count}, last_vote_ts=ts) + return _row(blob, last_vote_timestamp=ts, caching_tick=tick, math_tick=tick) + + def _make_conn_ready_for_final_batch(self, dataset: ReplayDataset, cuts): + """Every env's math_main row already satisfies the LAST batch's + thresholds from the start — since thresholds only grow, this makes + wait_for_tick succeed on its FIRST poll for every batch, so the test + never actually needs to sleep/retry (:func:`wait_for_tick` itself is + already covered by ``test_poller_equiv_seed.py``).""" + final_cut = cuts[-1] + final_ts = dataset.votes[final_cut - 1].t_ms + final_votes = pe.expected_cumulative_vote_count(dataset, final_cut) + conn = _FakeLoopConn() + for env in self.ENVS: + row = self._ready_row(env, vote_count=final_votes, ts=final_ts, tick=9) + conn.set_row("math_main", env, row) + conn.set_row("math_bidtopid", env, + _row({"zid": 1, "bidToPid": [[1]], "lastVoteTimestamp": final_ts})) + conn.set_row("math_ptptstats", env, + _row({"zid": 1, "ptptstats": {}, "lastVoteTimestamp": final_ts})) + return conn + + def _refuse_to_sleep(self, seconds): + raise AssertionError("run_batch_loop should never need to retry in this test") + + def test_processes_batches_in_order_and_writes_manifest(self, tmp_path): + ds = _dataset_no_revotes(7) + cuts = [3, 7] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + runners = {env: _FakeRunner(env) for env in self.ENVS} + insert_calls = [] + + def fake_insert(conn, dataset, prev, cut, zid): + insert_calls.append((prev, cut)) + return cut - prev + + manifest = pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, sleep=self._refuse_to_sleep, insert_fn=fake_insert, + ) + + assert insert_calls == [(0, 3), (3, 7)] + assert len(manifest["batches"]) == 2 + assert manifest["batches"][0]["prev_slot"] == 0 + assert manifest["batches"][0]["cut_slot"] == 3 + assert manifest["batches"][1]["cut_slot"] == 7 + for b in manifest["batches"]: + for env in self.ENVS: + assert b["envs"][env]["ready"] is True + assert b["envs"][env]["snapshots"] == { + "math_main": True, "math_bidtopid": True, "math_ptptstats": True, + } + + # Snapshots actually landed on disk. + for i in range(2): + for env in self.ENVS: + for table in pe.EQUIV_TABLES: + assert pe.load_snapshot(tmp_path, env, i, table) is not None + + assert pe.load_manifest(tmp_path) == manifest + + def test_batch_not_ready_raises_fail_fast_error_naming_env_batch_and_state(self, tmp_path): + """REQUIRED FIX #2 (2026-07-24 live-debug task) — 'FAIL-FAST FEEDER': + a batch whose readiness predicate times out must abort the stream + with a loud error naming the env, batch, elapsed, and the last + observed math_main state (or 'no row ever appeared'). This REPLACES + the old 'recorded without raising' contract, which is exactly the + bug that produced a silent, vacuous PASS in the 2026-07-24 live run + (the feeder kept feeding after readiness timeouts and snapshotted + nothing, with nothing surfacing the failure).""" + ds = _dataset_no_revotes(3) + cuts = [3] + conn = _FakeLoopConn() # no rows programmed at all -> never ready + runners = {env: _FakeRunner(env) for env in self.ENVS} + + with pytest.raises(pe.PollerEquivStreamError) as excinfo: + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=0.05, poll_interval=0.01, + insert_fn=lambda *a: 0, + ) + msg = str(excinfo.value) + assert "clj-ref" in msg # the FIRST env in ENVS times out first + assert "batch=0" in msg + assert "0.05" in msg # elapsed/timeout + assert "no row ever appeared" in msg + + # Partial manifest is still persisted for post-mortem (spec: "keep + # the DB alive... for post-mortem" — same intent for the manifest). + manifest = pe.load_manifest(tmp_path) + assert manifest is not None + assert manifest["batches"][0]["envs"]["clj-ref"]["ready"] is False + assert pe.load_snapshot(tmp_path, "clj-ref", 0, "math_main") is None + + def test_fail_fast_error_includes_last_observed_math_main_state(self, tmp_path): + """When a row DOES exist but never satisfies the readiness predicate + (e.g. the env is polling but stuck on a stale tick), the error must + report that row's state — not just 'no row ever appeared'.""" + ds = _dataset_no_revotes(3) + cuts = [3] + conn = _FakeLoopConn() + stale_row = self._ready_row("clj-ref", vote_count=1, ts=1, tick=7) + conn.set_row("math_main", "clj-ref", stale_row) # never meets the batch's threshold + runners = {env: _FakeRunner(env) for env in self.ENVS} + + with pytest.raises(pe.PollerEquivStreamError) as excinfo: + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=0.05, poll_interval=0.01, + insert_fn=lambda *a: 0, + ) + msg = str(excinfo.value) + assert "caching_tick=7" in msg + assert "no row ever appeared" not in msg + + def test_fail_fast_error_tails_the_failing_runners_log(self, tmp_path): + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("boom: connection refused\nmore diagnostic output\n") + ds = _dataset_no_revotes(3) + cuts = [3] + conn = _FakeLoopConn() + runners = {"clj-ref": _FakeRunner("clj-ref", log_path=log_path), + "py-shadow": _FakeRunner("py-shadow")} + + with pytest.raises(pe.PollerEquivStreamError) as excinfo: + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=0.05, poll_interval=0.01, + insert_fn=lambda *a: 0, + ) + msg = str(excinfo.value) + assert "boom: connection refused" in msg + + def test_first_env_success_is_preserved_when_second_env_times_out(self, tmp_path): + """The failing env is NOT necessarily the first one processed — an + earlier env's success within the SAME batch must survive (snapshot on + disk + manifest entry) even though the batch as a whole aborts.""" + ds = _dataset_no_revotes(3) + cuts = [3] + conn = _FakeLoopConn() + ready_row = self._ready_row( + "clj-ref", vote_count=pe.expected_cumulative_vote_count(ds, 3), + ts=ds.votes[2].t_ms, tick=1, + ) + conn.set_row("math_main", "clj-ref", ready_row) + conn.set_row("math_bidtopid", "clj-ref", _row({"zid": 1, "bidToPid": [[1]]})) + conn.set_row("math_ptptstats", "clj-ref", _row({"zid": 1, "ptptstats": {}})) + # py-shadow: no row ever -> times out. + runners = {env: _FakeRunner(env) for env in self.ENVS} + + with pytest.raises(pe.PollerEquivStreamError) as excinfo: + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=0.05, poll_interval=0.01, + insert_fn=lambda *a: 0, + ) + assert "py-shadow" in str(excinfo.value) + assert pe.load_snapshot(tmp_path, "clj-ref", 0, "math_main") is not None + manifest = pe.load_manifest(tmp_path) + assert manifest["batches"][0]["envs"]["clj-ref"]["ready"] is True + assert manifest["batches"][0]["envs"]["py-shadow"]["ready"] is False + + def test_earlier_successful_batch_is_preserved_after_a_later_batch_aborts(self, tmp_path): + ds = _dataset_no_revotes(7) + cuts = [3, 7] + conn = _FakeLoopConn() + # Batch 0 (slots 0:3) is ready for BOTH envs from the start. + b0_votes = pe.expected_cumulative_vote_count(ds, 3) + b0_ts = ds.votes[2].t_ms + for env in self.ENVS: + row = self._ready_row(env, vote_count=b0_votes, ts=b0_ts, tick=1) + conn.set_row("math_main", env, row) + conn.set_row("math_bidtopid", env, _row({"zid": 1, "bidToPid": [[1]]})) + conn.set_row("math_ptptstats", env, _row({"zid": 1, "ptptstats": {}})) + runners = {env: _FakeRunner(env) for env in self.ENVS} + + with pytest.raises(pe.PollerEquivStreamError): + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=0.05, poll_interval=0.01, + insert_fn=lambda *a: 0, + ) + + # Batch 0's snapshots and manifest entry are untouched by batch 1's abort. + for env in self.ENVS: + assert pe.load_snapshot(tmp_path, env, 0, "math_main") is not None + manifest = pe.load_manifest(tmp_path) + assert len(manifest["batches"]) == 2 + assert manifest["batches"][0]["envs"]["clj-ref"]["ready"] is True + assert manifest["batches"][1]["envs"]["clj-ref"]["ready"] is False + + def test_seam_restarts_only_the_designated_env(self, tmp_path): + ds = _dataset_no_revotes(7) + cuts = [3, 7] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + clj_runner = _FakeRunner("clj-ref") + py_runner = _FakeRunner("py-shadow") + runners = {"clj-ref": clj_runner, "py-shadow": py_runner} + new_py_runner = _FakeRunner("py-shadow-restarted") + + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + seam_after=0, restart_envs_at_seam=["py-shadow"], + restart_builders={"py-shadow": lambda: new_py_runner}, + wait_timeout=5.0, sleep=self._refuse_to_sleep, insert_fn=lambda *a: 0, + ) + + assert py_runner.killed is True + assert clj_runner.killed is False # NOT restarted — not in restart_envs_at_seam + assert runners["py-shadow"] is new_py_runner + assert new_py_runner.started is True + assert runners["clj-ref"] is clj_runner # unchanged + + def test_no_seam_means_no_restart(self, tmp_path): + ds = _dataset_no_revotes(7) + cuts = [3, 7] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + runners = {env: _FakeRunner(env) for env in self.ENVS} + + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + seam_after=None, wait_timeout=5.0, sleep=self._refuse_to_sleep, + insert_fn=lambda *a: 0, + ) + for r in runners.values(): + assert r.killed is False + + def test_expected_vote_count_recorded_per_batch(self, tmp_path): + ds = _dataset_with_revote() # 5 votes, 1 revote -> distinct pairs: 3, 3, 4, 4? see below + cuts = [4, 5] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + runners = {env: _FakeRunner(env) for env in self.ENVS} + + manifest = pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, sleep=self._refuse_to_sleep, insert_fn=lambda *a: 0, + ) + assert manifest["batches"][0]["expected_vote_count"] == 3 # (1,10)/(2,11)/(3,12) + assert manifest["batches"][1]["expected_vote_count"] == 4 # + (2,13) + + # ----------------------------------------------------------------- # + # Quirk Q19 mitigation — startup_gate_envs wiring (session 2, + # 2026-07-24). Disabled by default (byte-identical manifest shape for + # every EXISTING caller/test above); opt-in via startup_gate_envs. + # ----------------------------------------------------------------- # + def test_startup_gate_disabled_by_default(self, tmp_path): + ds = _dataset_no_revotes(3) + cuts = [3] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + runners = {env: _FakeRunner(env) for env in self.ENVS} + + manifest = pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, sleep=self._refuse_to_sleep, insert_fn=lambda *a: 0, + ) + assert manifest["startup_gate"] == {"enabled": False, "envs": {}} + + def test_startup_gate_waits_for_the_signal_before_batch_0_and_records_it(self, tmp_path): + ds = _dataset_no_revotes(3) + cuts = [3] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("boot chatter, no poll line yet\n") + runners = {"clj-ref": _FakeRunner("clj-ref", log_path=log_path), + "py-shadow": _FakeRunner("py-shadow")} + insert_calls = [] + + def sleep_then_append(seconds): + with open(log_path, "a") as fh: + fh.write("Polling :votes > 0\n") + + manifest = pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, poll_interval=0.01, sleep=sleep_then_append, + insert_fn=lambda c, d, p, cu, z: (insert_calls.append((p, cu)), 0)[1], + startup_gate_envs=["clj-ref"], + ) + assert manifest["startup_gate"]["enabled"] is True + assert manifest["startup_gate"]["envs"]["clj-ref"]["observed"] is True + assert insert_calls == [(0, 3)] # batch 0 still got inserted, AFTER the gate + + def test_startup_gate_ignores_stale_signal_from_a_prior_attempts_log(self, tmp_path): + """ROOT CAUSE (found live, 2026-07-24 session 2): the runner log is + append-mode. A gate that reads the WHOLE file (no offset) is + satisfied instantly by a "Polling :votes >" line left over from a + PREVIOUS attempt in the SAME --out dir — silently defeating the + mitigation after the very first run. run_batch_loop must capture + the log's size BEFORE waiting and only accept NEW content.""" + ds = _dataset_no_revotes(3) + cuts = [3] + conn = _FakeLoopConn() # never ready -> would time out on wait_for_tick too + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("Polling :votes > 999\n") # STALE, from a prior attempt + runners = {"clj-ref": _FakeRunner("clj-ref", log_path=log_path), + "py-shadow": _FakeRunner("py-shadow")} + insert_calls = [] + + with pytest.raises(pe.PollerEquivStreamError, match="startup gate"): + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, poll_interval=0.01, + insert_fn=lambda c, d, p, cu, z: (insert_calls.append((p, cu)), 0)[1], + startup_gate_envs=["clj-ref"], startup_gate_timeout=0.05, + ) + assert insert_calls == [] # the stale line must NOT have satisfied the gate + + def test_startup_gate_timeout_aborts_before_any_insert(self, tmp_path): + ds = _dataset_no_revotes(3) + cuts = [3] + conn = _FakeLoopConn() # no rows at all -> would time out on wait_for_tick too + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("boot chatter, never a poll line\n") + runners = {"clj-ref": _FakeRunner("clj-ref", log_path=log_path), + "py-shadow": _FakeRunner("py-shadow")} + insert_calls = [] + + with pytest.raises(pe.PollerEquivStreamError, match="startup gate"): + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, poll_interval=0.01, + insert_fn=lambda c, d, p, cu, z: (insert_calls.append((p, cu)), 0)[1], + startup_gate_envs=["clj-ref"], startup_gate_timeout=0.05, + ) + assert insert_calls == [] # the gate blocks BEFORE batch 0 is ever touched + manifest = pe.load_manifest(tmp_path) + assert manifest["startup_gate"]["envs"]["clj-ref"]["observed"] is False + + # ----------------------------------------------------------------- # + # Mod-event feeding — interleave-by-timestamp (session 2, 2026-07-24). + # ----------------------------------------------------------------- # + def test_mod_insert_fn_receives_the_correct_time_windows_per_batch(self, tmp_path): + ds = _dataset_no_revotes(7) + cuts = [3, 7] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + runners = {env: _FakeRunner(env) for env in self.ENVS} + mod_calls = [] + + def fake_mod_insert(c, d, prev_t, cut_t, z): + mod_calls.append((prev_t, cut_t)) + return 0 + + pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, sleep=self._refuse_to_sleep, insert_fn=lambda *a: 0, + mod_insert_fn=fake_mod_insert, + ) + cut0_ms = ds.votes[2].t_ms + cut1_ms = ds.votes[6].t_ms + assert mod_calls == [(None, cut0_ms), (cut0_ms, cut1_ms)] + + def test_mod_events_applied_count_is_recorded_per_batch(self, tmp_path): + ds = _dataset_no_revotes(3) + cuts = [3] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + runners = {env: _FakeRunner(env) for env in self.ENVS} + + manifest = pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, sleep=self._refuse_to_sleep, insert_fn=lambda *a: 0, + mod_insert_fn=lambda *a: 2, + ) + assert manifest["batches"][0]["n_mod_events_applied"] == 2 + + def test_default_mod_insert_fn_is_the_real_one_and_is_a_noop_with_no_mod_events(self, tmp_path): + """The default wiring (no mod_insert_fn override) must be safe for + EVERY existing dataset with zero mod_events — insert_mod_events + finds nothing in range and never touches the connection, so the + strict _FakeLoopConn (which only knows the 3 EQUIV_TABLES SELECTs) + is untouched by it.""" + ds = _dataset_no_revotes(3) + cuts = [3] + conn = self._make_conn_ready_for_final_batch(ds, cuts) + runners = {env: _FakeRunner(env) for env in self.ENVS} + + manifest = pe.run_batch_loop( + conn, ds, cuts, self.ENVS, runners, out_dir=tmp_path, + wait_timeout=5.0, sleep=self._refuse_to_sleep, insert_fn=lambda *a: 0, + ) + assert manifest["batches"][0]["n_mod_events_applied"] == 0 diff --git a/delphi/tests/replay_harness/test_poller_equiv_envelope.py b/delphi/tests/replay_harness/test_poller_equiv_envelope.py new file mode 100644 index 0000000000..16199e3b1b --- /dev/null +++ b/delphi/tests/replay_harness/test_poller_equiv_envelope.py @@ -0,0 +1,724 @@ +"""Unit tests for the poller-equivalence harness — Stage D (self-jitter +envelope + full-run orchestration), MATH_POLLER_EQUIV_SPEC.md §2-3. + +NO live Postgres/containers/subprocesses required: every test here is either +pure-Python or drives the on-disk snapshot store against ``tmp_path`` (same +convention as ``test_poller_equiv_compare.py``'s Stage C tests). The one live +prerequisite this Stage introduces — ``run_full_equiv_protocol`` needing a +real Postgres + the ``clojure`` CLI — is NOT exercised here; only its +fail-fast :func:`~polismath.replay.poller_equiv.preflight_check` gate is +(itself designed to fail in well under a second, no live service needed to +observe the failure path). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from polismath.replay import poller_equiv as pe +from polismath.replay import real_data + + +# --------------------------------------------------------------------------- # +# Shared fixtures / doubles — mirrors test_poller_equiv_compare.py's helpers. +# --------------------------------------------------------------------------- # +def _blob( + *, repness_val: float = 0.0, + in_conv: list[int] | None = None, user_vote_counts: dict[str, int] | None = None, +) -> dict[str, Any]: + """A minimal math_main-shaped blob (prep-main key spelling). ``repness`` + carries a SINGLE controllable scalar float leaf at a NON-PCA-related path + (``repness..repness-test``) — deliberately NOT under ``.pca.comps``/ + ``.proj.``/``.center`` (``_is_pca_related_path``, comparer.py:917-947), + because those paths get an automatic LOOSE tolerance (1000x abs / 10x + rel) for any PCA list shorter than 10 elements whenever + ``outlier_fraction > 0`` (comparer.py:761-771) — real behavior, but it + would swallow the tiny (1e-5-scale) deltas these tests need to observe + surviving the DEFAULT tolerance so envelope acceptance has something to + act on.""" + return { + "zid": 1, "n": 2, "n-cmts": 2, + "in-conv": in_conv if in_conv is not None else [1, 2], + "tids": [0, 1], + "pca": {"center": [0.1, 0.2], "comps": [[1.0, 0.0], [0.0, 1.0]]}, + "base-clusters": { + "id": [0, 1], "x": [0.1, -0.1], "y": [0.2, -0.2], + "count": [1, 2], "members": [[1], [2, 3]], + }, + "repness": {"1": {"repness-test": repness_val}}, + "lastVoteTimestamp": 1000, + "user-vote-counts": user_vote_counts if user_vote_counts is not None else {"1": 3}, + } + + +def _ptptstats_blob(*, val: float = 0.0) -> dict[str, Any]: + return {"zid": 1, "ptptstats": {"1": {"n-votes": val}}, "lastVoteTimestamp": 1000} + + +def _row(data: dict[str, Any], *, math_env: str = "e", last_vote_timestamp: int = 1000, + caching_tick: int = 1, math_tick: int = 1) -> dict[str, Any]: + return { + "zid": 1, "math_env": math_env, "data": data, + "last_vote_timestamp": last_vote_timestamp, + "caching_tick": caching_tick, "math_tick": math_tick, "modified": 123, + } + + +def _write_paired_batch( + tmp_path: Path, envs: tuple[str, str], index: int, *, main_a, main_b, + bid_a=None, bid_b=None, pt_a=None, pt_b=None, +) -> None: + env_a, env_b = envs + pe.write_snapshot(tmp_path, env_a, index, "math_main", main_a) + pe.write_snapshot(tmp_path, env_b, index, "math_main", main_b) + pe.write_snapshot( + tmp_path, env_a, index, "math_bidtopid", + bid_a or _row({"zid": 1, "bidToPid": [[1]], "lastVoteTimestamp": 1000}), + ) + pe.write_snapshot( + tmp_path, env_b, index, "math_bidtopid", + bid_b or _row({"zid": 1, "bidToPid": [[1]], "lastVoteTimestamp": 1000}), + ) + pe.write_snapshot(tmp_path, env_a, index, "math_ptptstats", pt_a or _row(_ptptstats_blob())) + pe.write_snapshot(tmp_path, env_b, index, "math_ptptstats", pt_b or _row(_ptptstats_blob())) + + +# --------------------------------------------------------------------------- # +# compute_self_jitter_envelope — spec §2 item 1 / Stage D item 1. +# --------------------------------------------------------------------------- # +class TestComputeSelfJitterEnvelope: + ENV = "clj-ref" + + def test_identical_stores_yield_an_empty_all_zero_envelope(self, tmp_path): + run1, run2 = tmp_path / "run1", tmp_path / "run2" + blob = _blob(repness_val=1.2345) + for run in (run1, run2): + pe.write_snapshot(run, self.ENV, 0, "math_main", _row(blob)) + pe.write_snapshot(run, self.ENV, 0, "math_ptptstats", _row(_ptptstats_blob())) + + report = pe.compute_self_jitter_envelope(run1, run2, math_env=self.ENV) + + assert report["envelope"] == {} + assert report["n_leaf_diffs"] == 0 + assert report["structural_divergences"] == [] + assert report["n_batches_aligned"] == 1 + # "all-zero" is a CONVENTION (missing key => 0 via envelope_threshold), + # never a literal zero-valued entry — nothing should be recorded. + assert pe.envelope_threshold(report["envelope"], "math_main.repness.N.repness-test") == pytest.approx(1e-9) + + def test_jittered_floats_recorded_with_correct_max_delta(self, tmp_path): + run1, run2 = tmp_path / "run1", tmp_path / "run2" + pe.write_snapshot(run1, self.ENV, 0, "math_main", _row(_blob(repness_val=1.0))) + pe.write_snapshot(run2, self.ENV, 0, "math_main", _row(_blob(repness_val=1.0 + 3e-5))) + for run in (run1, run2): + pe.write_snapshot(run, self.ENV, 0, "math_ptptstats", _row(_ptptstats_blob())) + + report = pe.compute_self_jitter_envelope(run1, run2, math_env=self.ENV) + + key = "math_main.repness.N.repness-test" + assert report["envelope"][key] == pytest.approx(3e-5, abs=1e-12) + assert report["n_leaf_diffs"] == 1 + + def test_path_normalization_collapses_across_batches_and_takes_the_max(self, tmp_path): + """Two DIFFERENT batches jitter the SAME normalized path pattern by + different amounts — the envelope must record the MAX, not the last + or first observed (spec: "record the max cross-run delta").""" + run1, run2 = tmp_path / "run1", tmp_path / "run2" + # Batch 0: small jitter (1e-6). Batch 1: larger jitter (7e-5), same + # normalized path ("repness.N.repness-test" — the numeric dict key + # '1' collapses to 'N' regardless of batch index). + pe.write_snapshot(run1, self.ENV, 0, "math_main", _row(_blob(repness_val=1.0))) + pe.write_snapshot(run2, self.ENV, 0, "math_main", _row(_blob(repness_val=1.0 + 1e-6 + 1e-6))) + pe.write_snapshot(run1, self.ENV, 1, "math_main", _row(_blob(repness_val=2.0))) + pe.write_snapshot(run2, self.ENV, 1, "math_main", _row(_blob(repness_val=2.0 + 7e-5))) + for run in (run1, run2): + for i in (0, 1): + pe.write_snapshot(run, self.ENV, i, "math_ptptstats", _row(_ptptstats_blob())) + + report = pe.compute_self_jitter_envelope(run1, run2, math_env=self.ENV) + + key = "math_main.repness.N.repness-test" + assert report["envelope"][key] == pytest.approx(7e-5, abs=1e-9) + + def test_structural_divergence_is_reported_separately_never_in_envelope(self, tmp_path): + run1, run2 = tmp_path / "run1", tmp_path / "run2" + pe.write_snapshot(run1, self.ENV, 0, "math_main", _row(_blob(in_conv=[1, 2]))) + pe.write_snapshot(run2, self.ENV, 0, "math_main", _row(_blob(in_conv=[1, 2, 3]))) + for run in (run1, run2): + pe.write_snapshot(run, self.ENV, 0, "math_ptptstats", _row(_ptptstats_blob())) + + report = pe.compute_self_jitter_envelope(run1, run2, math_env=self.ENV) + + assert report["envelope"] == {} + assert len(report["structural_divergences"]) == 1 + assert report["structural_divergences"][0]["path"] == "step_0.in-conv" + assert report["n_leaf_diffs"] == 0 + + def test_ptptstats_jitter_is_keyed_under_its_own_table_prefix(self, tmp_path): + run1, run2 = tmp_path / "run1", tmp_path / "run2" + for run in (run1, run2): + pe.write_snapshot(run, self.ENV, 0, "math_main", _row(_blob())) + pe.write_snapshot(run1, self.ENV, 0, "math_ptptstats", _row(_ptptstats_blob(val=0.0))) + pe.write_snapshot(run2, self.ENV, 0, "math_ptptstats", _row(_ptptstats_blob(val=2e-5))) + + report = pe.compute_self_jitter_envelope(run1, run2, math_env=self.ENV) + + assert any(k.startswith("math_ptptstats.") for k in report["envelope"]) + assert not any(k.startswith("math_main.") for k in report["envelope"]) + + def test_batches_only_in_one_store_are_reported(self, tmp_path): + run1, run2 = tmp_path / "run1", tmp_path / "run2" + pe.write_snapshot(run1, self.ENV, 0, "math_main", _row(_blob())) + pe.write_snapshot(run1, self.ENV, 0, "math_ptptstats", _row(_ptptstats_blob())) + # run2 never got batch 0 at all. + report = pe.compute_self_jitter_envelope(run1, run2, math_env=self.ENV) + + assert report["batches_only_in"]["run1"] == [0] + assert report["batches_only_in"]["run2"] == [] + assert report["n_batches_aligned"] == 0 + + def test_missing_table_snapshot_on_one_side_is_reported_not_raised(self, tmp_path): + run1, run2 = tmp_path / "run1", tmp_path / "run2" + for run in (run1, run2): + pe.write_snapshot(run, self.ENV, 0, "math_main", _row(_blob())) + pe.write_snapshot(run1, self.ENV, 0, "math_ptptstats", _row(_ptptstats_blob())) + # run2's math_ptptstats snapshot for batch 0 was never written. + report = pe.compute_self_jitter_envelope(run1, run2, math_env=self.ENV) + + assert len(report["missing_snapshots"]) == 1 + entry = report["missing_snapshots"][0] + assert entry["table"] == "math_ptptstats" + assert entry["missing_in"] == ["run2"] + + +# --------------------------------------------------------------------------- # +# write_envelope / load_envelope — disk round-trip. +# --------------------------------------------------------------------------- # +class TestEnvelopeStore: + def test_write_then_load_round_trips(self, tmp_path): + report = {"envelope": {"math_main.pca.comps[][]": 1e-5}, "n_leaf_diffs": 1} + path = pe.write_envelope(report, tmp_path) + assert path == tmp_path / "self_jitter_envelope.json" + assert pe.load_envelope(tmp_path) == report + + def test_load_missing_is_none(self, tmp_path): + assert pe.load_envelope(tmp_path) is None + + +# --------------------------------------------------------------------------- # +# envelope_threshold — pure formula (spec §2 item 2: envelope x 2, floor 1e-9). +# --------------------------------------------------------------------------- # +class TestEnvelopeThreshold: + def test_none_envelope_is_the_floor(self): + assert pe.envelope_threshold(None, "x") == pytest.approx(1e-9) + + def test_missing_key_is_the_floor(self): + assert pe.envelope_threshold({"y": 1.0}, "x") == pytest.approx(1e-9) + + def test_safety_factor_is_exactly_two(self): + assert pe.envelope_threshold({"x": 1e-6}, "x") == pytest.approx(2e-6) + + def test_floor_wins_over_a_tiny_envelope_value(self): + # 1e-12 * 2 = 2e-12 < the 1e-9 floor -> floor wins. + assert pe.envelope_threshold({"x": 1e-12}, "x") == pytest.approx(1e-9) + + def test_floor_applies_even_with_an_explicit_zero_entry(self): + assert pe.envelope_threshold({"x": 0.0}, "x") == pytest.approx(1e-9) + + +# --------------------------------------------------------------------------- # +# envelope_path_key — pure, reuses certify.normalize_path (never reimplements). +# --------------------------------------------------------------------------- # +class TestEnvelopePathKey: + def test_strips_step_prefix_and_collapses_indices(self): + assert ( + pe.envelope_path_key("math_main", "step_3.pca.comps[0][1]") + == "math_main.pca.comps[][]" + ) + + def test_collapses_numeric_dict_keys(self): + assert pe.envelope_path_key("math_main", "step_0.repness.42.score") == "math_main.repness.N.score" + + def test_table_prefix_disambiguates_identical_leaf_names(self): + a = pe.envelope_path_key("math_main", "step_0.foo") + b = pe.envelope_path_key("math_ptptstats", "step_0.foo") + assert a != b + + +# --------------------------------------------------------------------------- # +# Envelope-aware acceptance — compare_batch/compare_snapshots (Stage D item 2). +# --------------------------------------------------------------------------- # +class TestEnvelopeAwareCompareBatch: + ENVS = ("clj-ref", "py-shadow") + + def test_default_no_envelope_is_byte_identical_to_pre_stage_d_shape(self, tmp_path): + """Regression guard for the task's explicit 'keep the default ... + byte-identical' requirement: no ``within_envelope`` family key, no + ``n_within_envelope`` key, at all — not even an empty one.""" + blob_a = _blob(repness_val=1.0) + blob_b = _blob(repness_val=1.0 + 1e-9) # well within the OWN default tolerance + _write_paired_batch(tmp_path, self.ENVS, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + + result = pe.compare_batch(tmp_path, 0, self.ENVS) + + main = result["tables"]["math_main"] + assert set(main.keys()) == {"match", "n_divergences", "families"} + assert set(main["families"].keys()) == {"exact", "tolerant"} + ptpt = result["tables"]["math_ptptstats"] + assert set(ptpt.keys()) == {"match", "n_divergences", "families"} + + def test_within_envelope_divergence_is_accepted_and_counted(self, tmp_path): + blob_a = _blob(repness_val=0.0) + blob_b = _blob(repness_val=5e-6) # exceeds the comparer's OWN 1e-6 atol + _write_paired_batch(tmp_path, self.ENVS, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + + # Sanity: WITHOUT an envelope this is a real (tolerant-family) divergence. + no_env = pe.compare_batch(tmp_path, 0, self.ENVS) + assert no_env["tables"]["math_main"]["match"] is False + + envelope = {"math_main.repness.N.repness-test": 3e-6} # threshold = 6e-6 >= delta 5e-6 + result = pe.compare_batch(tmp_path, 0, self.ENVS, envelope=envelope) + + main = result["tables"]["math_main"] + assert main["match"] is True + assert main["n_divergences"] == 0 + assert main["n_within_envelope"] == 1 + assert len(main["families"]["within_envelope"]) == 1 + assert main["families"]["within_envelope"][0]["path"] == "step_0.repness.1.repness-test" + assert main["families"]["tolerant"] == [] + + def test_beyond_envelope_divergence_is_rejected(self, tmp_path): + blob_a = _blob(repness_val=0.0) + blob_b = _blob(repness_val=5e-6) + _write_paired_batch(tmp_path, self.ENVS, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + + envelope = {"math_main.repness.N.repness-test": 1e-7} # threshold = 2e-7 < delta 5e-6 + result = pe.compare_batch(tmp_path, 0, self.ENVS, envelope=envelope) + + main = result["tables"]["math_main"] + assert main["match"] is False + assert main["n_divergences"] == 1 + assert main["n_within_envelope"] == 0 + assert len(main["families"]["tolerant"]) == 1 + + def test_structural_divergence_is_never_excused_by_a_huge_envelope(self, tmp_path): + blob_a = _blob(in_conv=[1, 2]) + blob_b = _blob(in_conv=[1, 2, 3]) + _write_paired_batch(tmp_path, self.ENVS, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + + huge_envelope = {"math_main.repness.N.repness-test": 1e6, "math_main.in-conv": 1e6} + result = pe.compare_batch(tmp_path, 0, self.ENVS, envelope=huge_envelope) + + main = result["tables"]["math_main"] + assert main["match"] is False + assert len(main["families"]["exact"]) == 1 + assert main["families"]["exact"][0]["path"] == "step_0.in-conv" + + def test_floor_rejects_a_delta_above_1e_minus_9_with_no_observed_jitter(self, tmp_path): + """Regression for the floor itself (not just the formula): an EMPTY + envelope (no observed self-jitter at all) must not silently accept + every tiny divergence — only ones at or below the 1e-9 floor.""" + blob_a = _blob(repness_val=0.0) + blob_b = _blob(repness_val=5e-6) + _write_paired_batch(tmp_path, self.ENVS, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + + result = pe.compare_batch(tmp_path, 0, self.ENVS, envelope={}) + assert result["tables"]["math_main"]["match"] is False + + +class TestEnvelopeAwareCompareSnapshots: + ENVS = ("clj-ref", "py-shadow") + + def test_default_no_envelope_report_has_no_extra_keys(self, tmp_path): + blob = _blob(user_vote_counts={"1": 3}) + _write_paired_batch(tmp_path, self.ENVS, 0, main_a=_row(blob), main_b=_row(blob)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS) + assert "envelope_applied" not in report + assert "n_within_envelope_total" not in report + + def test_envelope_applied_flag_and_total_count_are_reported(self, tmp_path): + blob_a = _blob(repness_val=0.0, user_vote_counts={"1": 3}) + blob_b = _blob(repness_val=5e-6, user_vote_counts={"1": 3}) + _write_paired_batch(tmp_path, self.ENVS, 0, main_a=_row(blob_a), main_b=_row(blob_b)) + pe.write_manifest(tmp_path, {"batches": [{"expected_vote_count": 3}]}) + + envelope = {"math_main.repness.N.repness-test": 3e-6} + report = pe.compare_snapshots(tmp_path, math_envs=self.ENVS, envelope=envelope) + + assert report["envelope_applied"] is True + assert report["n_within_envelope_total"] == 1 + assert report["overall_match"] is True + + +class TestRenderCompareLinesEnvelopeFooter: + def _report(self, **overrides) -> dict[str, Any]: + base = { + "n_batches_aligned": 1, "math_envs": ["clj-ref", "py-shadow"], + "batches_only_in": {"clj-ref": [], "py-shadow": []}, + "per_batch": [{"batch": 0, "tables": { + "math_main": {"match": True}, "math_bidtopid": {"match": True}, + "math_ptptstats": {"match": True}, + }, "watermark": {"ok": True}}], + "ticks": { + "clj-ref": {"caching_tick": {"strictly_increasing": True}, + "math_tick": {"strictly_increasing": True}}, + "py-shadow": {"caching_tick": {"strictly_increasing": True}, + "math_tick": {"strictly_increasing": True}}, + }, + "overall_match": True, + } + base.update(overrides) + return base + + def test_no_envelope_key_means_no_extra_footer_line(self): + lines = pe.render_compare_lines(self._report()) + assert not any("envelope" in line for line in lines) + assert lines[-1].startswith("verdict: MATCH") + + def test_envelope_applied_adds_a_reported_count_line(self): + report = self._report(envelope_applied=True, n_within_envelope_total=3) + lines = pe.render_compare_lines(report) + assert any("3 divergence(s)" in line and "envelope" in line for line in lines) + assert len(lines) <= 40 + + +# --------------------------------------------------------------------------- # +# assemble_full_run_verdict — the PURE decision logic (Stage D item 3), +# exercised entirely against canned snapshot dirs. +# --------------------------------------------------------------------------- # +class TestAssembleFullRunVerdict: + CLJ, PY = "clj-ref", "py-shadow" + + def _write_self_jitter_pair(self, tmp_path, *, jitter: float = 0.0): + run1, run2 = tmp_path / "self-jitter-1", tmp_path / "self-jitter-2" + pe.write_snapshot(run1, self.CLJ, 0, "math_main", _row(_blob(repness_val=1.0))) + pe.write_snapshot(run2, self.CLJ, 0, "math_main", _row(_blob(repness_val=1.0 + jitter))) + for run in (run1, run2): + pe.write_snapshot(run, self.CLJ, 0, "math_ptptstats", _row(_ptptstats_blob())) + pe.write_manifest(run, {"batches": [ + {"index": 0, "envs": {self.CLJ: {"ready": True}}}, + ]}) + return run1, run2 + + def _write_main(self, tmp_path, *, py_delta: float = 0.0, structural_break: bool = False): + main = tmp_path / "main" + in_conv_b = [1, 2, 3] if structural_break else [1, 2] + # Base value 0.0 (not e.g. 1.0): the underlying ConversationComparer's + # OWN default tolerance is |a-b| <= 1e-6 + 0.01*|b| — at base=1.0 a + # 5e-5 delta is already comfortably inside that 1% relative band and + # would never even reach the envelope-acceptance step. At base=0.0 + # only the absolute floor (1e-6) applies, so these deltas (1e-7..1e-4 + # scale) genuinely exercise envelope acceptance rather than being + # silently absorbed one layer down. + blob_a = _blob(repness_val=0.0, user_vote_counts={"1": 3}) + blob_b = _blob(repness_val=py_delta, user_vote_counts={"1": 3}, in_conv=in_conv_b) + _write_paired_batch( + main, (self.CLJ, self.PY), 0, + main_a=_row(blob_a, math_env=self.CLJ), main_b=_row(blob_b, math_env=self.PY), + ) + pe.write_manifest(main, {"batches": [{"expected_vote_count": 3}]}) + return main + + def test_clean_run_passes(self, tmp_path): + # cuts matches the ONE batch the fixture stores actually hold — the + # original version of this test passed cuts=[100, 200] against a + # 1-batch store and asserted True, which was exactly the + # completeness hole #2657's review flagged (finding 2). + run1, run2 = self._write_self_jitter_pair(tmp_path, jitter=0.0) + main = self._write_main(tmp_path, py_delta=0.0) + + verdict = pe.assemble_full_run_verdict( + run1, run2, main, dataset="vw", cuts=[100], seam_after=None, + clj_env=self.CLJ, py_env=self.PY, + ) + + assert verdict["overall_pass"] is True + assert verdict["self_jitter_envelope"]["envelope"] == {} + assert verdict["dataset"] == "vw" + assert verdict["cuts"] == [100] + + def test_partial_main_store_fails_completeness(self, tmp_path): + """#2657 review finding 2: a feeder killed cleanly BETWEEN batches + (SIGTERM/OOM outside the fail-fast paths) leaves a store whose later + batches simply never appear — no ready:false marker — so + n_batches_aligned > 0 alone reads as a pass. The verdict must + compare aligned batches against the PLANNED count (len(cuts)).""" + run1, run2 = self._write_self_jitter_pair(tmp_path, jitter=0.0) + main = self._write_main(tmp_path, py_delta=0.0) # ONE batch on disk + + verdict = pe.assemble_full_run_verdict( + run1, run2, main, dataset="vw", cuts=[100, 200], seam_after=1, + clj_env=self.CLJ, py_env=self.PY, + ) + + assert verdict["overall_pass"] is False + assert verdict["compare"]["expected_batches"] == 2 + assert verdict["compare"]["overall_match"] is False + + def test_compare_snapshots_expected_batches_guard(self, tmp_path): + # Direct compare_snapshots-level check of the same rule, both sides. + main = self._write_main(tmp_path, py_delta=0.0) + short = pe.compare_snapshots( + main, math_envs=(self.CLJ, self.PY), expected_batches=2) + assert short["overall_match"] is False + assert short["expected_batches"] == 2 + exact = pe.compare_snapshots( + main, math_envs=(self.CLJ, self.PY), expected_batches=1) + assert exact["overall_match"] is True + # Default (None) keeps the pre-existing report shape: no new key. + default = pe.compare_snapshots(main, math_envs=(self.CLJ, self.PY)) + assert "expected_batches" not in default + + def test_py_divergence_within_measured_self_jitter_is_accepted(self, tmp_path): + # Self-jitter runs show clj disagreeing with itself by up to 4e-5; + # the paired run's py value sits within 2x that envelope of clj. + run1, run2 = self._write_self_jitter_pair(tmp_path, jitter=4e-5) + main = self._write_main(tmp_path, py_delta=5e-5) # <= 2 * 4e-5 = 8e-5 + + verdict = pe.assemble_full_run_verdict(run1, run2, main, clj_env=self.CLJ, py_env=self.PY) + + assert verdict["self_jitter_envelope"]["envelope"] + assert verdict["compare"]["envelope_applied"] is True + assert verdict["overall_pass"] is True + + def test_py_divergence_beyond_the_envelope_still_fails(self, tmp_path): + run1, run2 = self._write_self_jitter_pair(tmp_path, jitter=1e-7) + main = self._write_main(tmp_path, py_delta=5e-5) # >> 2 * 1e-7 + + verdict = pe.assemble_full_run_verdict(run1, run2, main, clj_env=self.CLJ, py_env=self.PY) + + assert verdict["overall_pass"] is False + + def test_structural_divergence_fails_regardless_of_envelope(self, tmp_path): + run1, run2 = self._write_self_jitter_pair(tmp_path, jitter=1e6) # absurdly huge envelope + main = self._write_main(tmp_path, py_delta=0.0, structural_break=True) + + verdict = pe.assemble_full_run_verdict(run1, run2, main, clj_env=self.CLJ, py_env=self.PY) + + assert verdict["overall_pass"] is False + + # ------------------------------------------------------------- # + # NO-COVERAGE GUARD — REQUIRED FIX #1 (2026-07-24 live-debug task): + # a vacuous self-jitter measurement (0 batches from either/both clj-only + # runs) must fail the WHOLE full-run verdict, not just silently report + # an empty envelope. This is EXACTLY the shape of the real 2026-07-24 + # live-run bug: "self-jitter envelope: 0 path(s) jittered... (none + # observed — identical self-jitter runs)" printed as if it were a clean + # signal, when in fact NOTHING was ever measured. + # ------------------------------------------------------------- # + def test_empty_self_jitter_streams_fail_the_full_run_even_if_main_is_clean(self, tmp_path): + main = self._write_main(tmp_path, py_delta=0.0) + run1, run2 = tmp_path / "self-jitter-1", tmp_path / "self-jitter-2" + # NEITHER self-jitter store ever got a single snapshot (both clj-ref + # containers crashed at startup, say) — no write_snapshot call at all. + + verdict = pe.assemble_full_run_verdict(run1, run2, main, clj_env=self.CLJ, py_env=self.PY) + + assert verdict["self_jitter_envelope"]["n_batches_aligned"] == 0 + assert verdict["overall_pass"] is False + + def test_self_jitter_manifest_ready_false_fails_the_full_run(self, tmp_path): + main = self._write_main(tmp_path, py_delta=0.0) + run1, run2 = tmp_path / "self-jitter-1", tmp_path / "self-jitter-2" + pe.write_snapshot(run1, self.CLJ, 0, "math_main", _row(_blob(repness_val=1.0))) + pe.write_snapshot(run2, self.CLJ, 0, "math_main", _row(_blob(repness_val=1.0))) + for run in (run1, run2): + pe.write_snapshot(run, self.CLJ, 0, "math_ptptstats", _row(_ptptstats_blob())) + # run1's manifest HONESTLY records that batch 1 (a second, never-fed + # batch) never became ready — batch 0 (aligned, clean) would ALONE + # report a passing envelope under the pre-guard logic. + pe.write_manifest(run1, {"batches": [ + {"index": 0, "envs": {self.CLJ: {"ready": True}}}, + {"index": 1, "envs": {self.CLJ: {"ready": False}}}, + ]}) + pe.write_manifest(run2, {"batches": [ + {"index": 0, "envs": {self.CLJ: {"ready": True}}}, + ]}) + + verdict = pe.assemble_full_run_verdict(run1, run2, main, clj_env=self.CLJ, py_env=self.PY) + + assert verdict["overall_pass"] is False + assert verdict["self_jitter_coverage"]["run1"]["ok"] is False + + +class TestWriteFullRunVerdict: + def test_writes_json_at_the_expected_path(self, tmp_path): + verdict = {"overall_pass": True, "dataset": "vw"} + path = pe.write_full_run_verdict(verdict, tmp_path) + assert path == tmp_path / "full_run_verdict.json" + assert json.loads(path.read_text()) == verdict + + +class TestRenderFullRunLines: + def _verdict(self, *, overall_pass=True, envelope=None, n_batches=1) -> dict[str, Any]: + per_batch = [ + {"batch": i, "tables": {"math_main": {"match": True}, "math_bidtopid": {"match": True}, + "math_ptptstats": {"match": True}}, + "watermark": {"ok": True}} + for i in range(n_batches) + ] + compare = { + "n_batches_aligned": n_batches, "math_envs": ["clj-ref", "py-shadow"], + "batches_only_in": {"clj-ref": [], "py-shadow": []}, + "per_batch": per_batch, + "ticks": { + "clj-ref": {"caching_tick": {"strictly_increasing": True}, + "math_tick": {"strictly_increasing": True}}, + "py-shadow": {"caching_tick": {"strictly_increasing": True}, + "math_tick": {"strictly_increasing": True}}, + }, + "overall_match": overall_pass, + } + return { + "dataset": "vw", "seam_after": 4, "overall_pass": overall_pass, + "compare": compare, + "self_jitter_envelope": {"envelope": envelope or {}}, + } + + def test_pass_verdict_renders_within_line_budget(self): + lines = pe.render_full_run_lines(self._verdict(overall_pass=True)) + assert len(lines) <= 40 + assert any("PASS" in line for line in lines) + assert any("vw" in line for line in lines) + + def test_fail_verdict_is_labeled_fail(self): + lines = pe.render_full_run_lines(self._verdict(overall_pass=False)) + assert any("FAIL" in line for line in lines) + + def test_empty_envelope_says_none_observed(self): + lines = pe.render_full_run_lines(self._verdict(envelope={})) + assert any("none observed" in line for line in lines) + + def test_worst_envelope_paths_are_shown(self): + envelope = {f"math_main.k{i}": 10.0 ** (-i) for i in range(1, 8)} + lines = pe.render_full_run_lines(self._verdict(envelope=envelope)) + # The single largest delta (k1 -> 1e-1) must be visible. + assert any("k1" in line for line in lines) + assert len(lines) <= 40 + + def test_large_batch_count_still_respects_the_line_budget(self): + envelope = {f"math_main.k{i}": 10.0 ** (-i) for i in range(1, 8)} + lines = pe.render_full_run_lines(self._verdict(n_batches=200, envelope=envelope)) + assert len(lines) <= 40 + + +# --------------------------------------------------------------------------- # +# default_full_run_schedule — reads real schedule files / dataset CSVs, no +# live services (pure file I/O against the committed repo + real_data/). +# --------------------------------------------------------------------------- # +class TestDefaultFullRunSchedule: + def test_vw_reads_the_committed_uniform8_restart4_schedule_verbatim(self): + cuts, seam_after = pe.default_full_run_schedule("vw") + assert cuts == [585, 1171, 1756, 2342, 2927, 3512, 4098, 4683] + assert seam_after == 4 + + def test_other_dataset_derives_uniform8_from_its_own_vote_count(self): + if real_data.dataset_dir("biodiversity") is None: + pytest.skip("biodiversity dataset not present in this checkout") + cuts, seam_after = pe.default_full_run_schedule("biodiversity") + assert len(cuts) == 8 + assert all(b > a for a, b in zip(cuts, cuts[1:])) # strictly increasing + assert seam_after == 4 # mid-schedule for 8 cuts, 0-based + + +# --------------------------------------------------------------------------- # +# preflight_check — fail-fast gate (Stage D item 3). No live service reached: +# the clojure-missing path is a pure shutil.which stub, and the unreachable-DB +# path targets a definitely-closed local port (immediate ECONNREFUSED). +# --------------------------------------------------------------------------- # +class TestPreflightCheck: + def test_missing_clojure_cli_raises_a_clear_message(self, monkeypatch): + monkeypatch.setattr(pe.shutil, "which", lambda name: None) + with pytest.raises(RuntimeError, match="clojure"): + pe.preflight_check("postgresql://u:p@127.0.0.1:1/nonexistent") + + def test_unreachable_postgres_raises_a_clear_message_fast(self, monkeypatch): + monkeypatch.setattr(pe.shutil, "which", lambda name: "/usr/bin/clojure") + with pytest.raises(RuntimeError, match="cannot reach Postgres"): + pe.preflight_check("postgresql://u:p@127.0.0.1:1/nonexistent", connect_timeout=1.0) + + def test_reachable_prerequisites_do_not_raise(self, monkeypatch): + """No real DB is touched: a fake sqlalchemy engine/connection double + stands in so this stays a pure/offline test.""" + monkeypatch.setattr(pe.shutil, "which", lambda name: "/usr/bin/clojure") + + class _FakeConnCtx: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + class _FakeEngine: + def connect(self): + return _FakeConnCtx() + + def dispose(self): + pass + + monkeypatch.setattr(pe.sa, "create_engine", lambda *a, **k: _FakeEngine()) + pe.preflight_check("postgresql://u:p@127.0.0.1:1/nonexistent") # must not raise + + +# --------------------------------------------------------------------------- # +# FullRunConfig — defaults. +# --------------------------------------------------------------------------- # +class TestFullRunConfig: + def test_restart_clj_at_seam_defaults_true(self): + config = pe.FullRunConfig( + dataset="vw", admin_url="postgresql://x", out_root="/tmp/x", + cuts=(1, 2), seam_after=0, + ) + assert config.restart_clj_at_seam is True + + def test_is_frozen(self): + config = pe.FullRunConfig( + dataset="vw", admin_url="postgresql://x", out_root="/tmp/x", + cuts=(1, 2), seam_after=0, + ) + with pytest.raises(Exception): + config.dataset = "biodiversity" + + +# --------------------------------------------------------------------------- # +# CLI wiring — the full-run subcommand exists and accepts its flags. +# --------------------------------------------------------------------------- # +def _cli_module(): + import importlib.util + + spec = importlib.util.spec_from_file_location( + "poller_equiv_cli_stage_d", Path(__file__).resolve().parents[2] / "scripts" / "poller_equiv.py" + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class TestFullRunCli: + def test_full_run_subcommand_is_registered(self): + from click.testing import CliRunner + + mod = _cli_module() + result = CliRunner().invoke(mod.cli, ["--help"]) + assert result.exit_code == 0 + assert "full-run" in result.output + + def test_full_run_help_lists_its_flags(self): + from click.testing import CliRunner + + mod = _cli_module() + result = CliRunner().invoke(mod.cli, ["full-run", "--help"]) + assert result.exit_code == 0 + assert "--seam-after" in result.output + assert "--restart-clj-at-seam" in result.output diff --git a/delphi/tests/replay_harness/test_poller_equiv_seed.py b/delphi/tests/replay_harness/test_poller_equiv_seed.py new file mode 100644 index 0000000000..ba78b3e7c3 --- /dev/null +++ b/delphi/tests/replay_harness/test_poller_equiv_seed.py @@ -0,0 +1,1018 @@ +"""Unit tests for the poller-equivalence harness — Stages A (schema+seeder) +and B (runners), MATH_POLLER_EQUIV_SPEC.md. + +NO live Postgres/containers required by default: every test here is either +pure-Python or drives ``polismath.replay.poller_equiv`` against a fake +connection double that records ``execute(stmt, params)`` calls. A handful of +end-to-end tests are gated on ``POLLER_EQUIV_PG_URL`` and self-skip when it is +unset (mirrors ``tests/poller/test_integration_postgres.py``). +""" + +from __future__ import annotations + +import importlib.util +import os +import subprocess +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import sqlalchemy as sa +from click.testing import CliRunner + +from polismath.replay import poller_equiv as pe +from polismath.replay.types import CommentMeta, ModEvent, ReplayDataset +from polismath.utils.engine_mode import ENGINE_MODE_ENV_VAR, ENGINE_MODE_LEGACY +from polismath.utils.general import delphi_vote_to_postgres, postgres_vote_to_delphi + +PG_URL = os.environ.get("POLLER_EQUIV_PG_URL") +_needs_live_pg = pytest.mark.skipif( + not PG_URL, reason="POLLER_EQUIV_PG_URL not set — skipping live-postgres test" +) + + +# --------------------------------------------------------------------------- # +# Fixtures +# --------------------------------------------------------------------------- # +def _dataset() -> ReplayDataset: + """A tiny hand-built dataset: 2 comments, 4 votes across 2 participants.""" + raw = [ + (1000, 1, 10, 1), # AGREE (Delphi convention, +1) + (1001, 1, 11, -1), # DISAGREE + (1002, 2, 10, 0), # PASS + (1003, 2, 11, 1), # AGREE + ] + comments = { + 10: CommentMeta(tid=10, created_ms=900, is_meta=False), + 11: CommentMeta(tid=11, created_ms=901, is_meta=True), + } + return ReplayDataset.build(raw, comments=comments) + + +class _FakeResult: + def __init__(self, row): + self._row = row + + def mappings(self): + return self + + def first(self): + return self._row + + +class _FakeConn: + """Records every execute() call; never touches a real database.""" + + def __init__(self): + self.calls: list[tuple[str, dict]] = [] + + def execute(self, stmt, params=None): + self.calls.append((str(stmt), dict(params or {}))) + return _FakeResult(None) + + +# --------------------------------------------------------------------------- # +# Stage A.1 — schema DDL covers the exact columns both pollers' SQL uses. +# --------------------------------------------------------------------------- # +class TestSchemaColumns: + def test_parse_schema_columns_finds_every_table(self): + tables = pe.parse_schema_columns() + expected_tables = { + "conversations", "votes", "comments", "participants", + "math_ticks", "math_main", "math_ptptstats", "math_bidtopid", + "math_profile", + } + assert expected_tables <= set(tables) + + def test_votes_columns_cover_both_pollers_select(self): + # clj: postgres.clj:132-145 poll / :197-212 conv-poll (SELECT *, + # downstream only touches pid/tid/vote — conv_man.clj:202-203). + # py: postgres.py:474-532 poll_votes / :534-567 poll_votes_since + # (SELECT zid, tid, pid, vote, created). + cols = set(pe.parse_schema_columns()["votes"]) + assert {"zid", "pid", "tid", "vote", "created"} <= cols + + def test_comments_columns_cover_both_pollers_select(self): + # clj: postgres.clj:148-161 mod-poll / :214-225 conv-mod-poll, consumed + # by math/conversation.clj:846-884 mod-update (:tid :is_meta :mod + # :modified). py: postgres.py:569-604 poll_moderation_since (zid, tid, + # modified, mod, is_meta) / :645-723 poll_moderation (tid, modified, + # mod, is_meta). + cols = set(pe.parse_schema_columns()["comments"]) + assert {"zid", "tid", "modified", "mod", "is_meta"} <= cols + + def test_conversations_is_fk_target_only(self): + cols = set(pe.parse_schema_columns()["conversations"]) + assert cols == {"zid"} + + def test_math_main_columns(self): + # postgres.clj:323-338 upload-math-main / :419-434 load-conv; + # postgres.py:757-817 write_math_main / :725-755 load_math_main. + cols = set(pe.parse_schema_columns()["math_main"]) + assert { + "zid", "math_env", "data", "last_vote_timestamp", + "caching_tick", "math_tick", + } <= cols + + def test_math_bidtopid_and_ptptstats_columns(self): + for table in ("math_bidtopid", "math_ptptstats"): + cols = set(pe.parse_schema_columns()[table]) + assert {"zid", "math_env", "math_tick", "data"} <= cols, table + + def test_math_ticks_columns(self): + # postgres.clj:292-295 inc-math-tick; postgres.py:910-939 increment_math_tick. + cols = set(pe.parse_schema_columns()["math_ticks"]) + assert {"zid", "math_env", "math_tick", "caching_tick"} <= cols + + def test_participants_shim_columns(self): + # SHIM: only postgres.py:701-713 poll_moderation's mod_out_ptpts query + # needs this table to exist at all (see module docstring). + cols = set(pe.parse_schema_columns()["participants"]) + assert {"pid", "zid", "mod"} <= cols + + def test_math_profile_columns(self): + # clj-write-only: conv_man.clj:97-113 handle-profile-data -> + # postgres.clj:340-348 upload-math-profile. + cols = set(pe.parse_schema_columns()["math_profile"]) + assert {"zid", "math_env", "data"} <= cols + + def test_parser_ignores_constraint_lines(self): + # UNIQUE(...) / FOREIGN KEY / PRIMARY KEY lines must never be mistaken + # for column declarations. + cols = pe.parse_schema_columns()["comments"] + assert "UNIQUE" not in [c.upper() for c in cols] + assert "FOREIGN" not in [c.upper() for c in cols] + + def test_parser_handles_nested_parens_in_column_types(self): + # VARCHAR(999) / VARCHAR(1000) contain a comma-free nested paren pair; + # a naive "split on any comma" parser would still work here, but a + # naive "first ');' terminates the table" parser could be fooled by + # now_as_millis() calls inside DEFAULT clauses — regression-guard that + # every table in the real schema still parses to a plausible column + # count (no table silently truncated). + tables = pe.parse_schema_columns() + assert len(tables["comments"]) >= 8 + assert len(tables["math_main"]) >= 6 + + +# --------------------------------------------------------------------------- # +# Stage A.2 — vote sign convention. +# --------------------------------------------------------------------------- # +class TestVoteSignConvention: + @pytest.mark.parametrize( + "delphi_sign, raw_db_sign", + [(1, -1), (-1, 1), (0, 0)], # AGREE, DISAGREE, PASS + ) + def test_delphi_vote_to_postgres_matches_raw_db_convention(self, delphi_sign, raw_db_sign): + # migrations.sql:742-747 — RAW DB: -1=agree, +1=disagree, 0=pass. + assert delphi_vote_to_postgres(delphi_sign) == raw_db_sign + + @pytest.mark.parametrize("delphi_sign", [1, -1, 0]) + def test_full_round_trip_dataset_to_db_to_py_ingress(self, delphi_sign): + """dataset.sign (Delphi convention, driver.py:56) -> seeder flip -> + RAW DB value -> py poll_votes ingress flip (postgres_vote_to_delphi, + general.py:19-40) -> back to the original Delphi-convention sign.""" + raw_db_value = delphi_vote_to_postgres(delphi_sign) + recovered = postgres_vote_to_delphi(raw_db_value) + assert recovered == delphi_sign + + def test_insert_votes_flips_sign_to_raw_db_convention(self): + ds = _dataset() + conn = _FakeConn() + n = pe.insert_votes(conn, ds, 0, ds.n) + + assert n == ds.n + # ATOMIC per batch (see test_insert_votes_is_one_atomic_statement's + # docstring for the root-cause rationale): exactly ONE execute() + # call carries every row's VALUES tuple, not one call per row. + assert len(conn.calls) == 1 + sql, params = conn.calls[0] + assert "INSERT INTO votes" in sql + assert sql.count("VALUES") == 1 + # First vote: sign=+1 (AGREE, Delphi) -> raw DB must be -1. + assert params["vote0"] == -1 + assert params["pid0"] == ds.votes[0].pid + assert params["tid0"] == ds.votes[0].tid + assert params["created0"] == ds.votes[0].t_ms + assert params["zid"] == pe.DEFAULT_ZID + + # Third vote (index 2, sorted order) is the PASS (sign=0) -> raw 0. + pass_idx = next(i for i, v in enumerate(ds.votes) if v.sign == 0) + assert params[f"vote{pass_idx}"] == 0 + + def test_insert_votes_is_one_atomic_statement(self): + """ROOT CAUSE #5 (2026-07-24 live-debug task, discovered AFTER root + cause #4's cut-boundary fix): a per-row execute() loop under + AUTOCOMMIT (the harness's default isolation level, ``build_clj_env`` + et al.) lets a CONCURRENTLY-RUNNING poller (both engines poll every + ~1s regardless of harness batch boundaries — production behavior, + unrelated to :func:`snap_cuts_past_timestamp_ties`) observe a + PARTIAL batch mid-insert. If that partial snapshot's max ``created`` + value ties with a not-yet-committed row's ``created`` (extremely + common in the vw dataset — most timestamps are shared by 2-8 votes, + module docstring), the STRICT ``created > watermark`` comparison + (both pollers, verbatim SQL) permanently drops that row the instant + the watermark advances past it — reproduced live: pid=33's LAST vote + (index 2050, comfortably INSIDE batch 3's [1757, 2349) range, nowhere + near either cut edge) went missing from clj-ref's own + ``user-vote-counts``, undercounting by exactly 1, even AFTER the cut + boundaries themselves were tie-free. + + Building ONE multi-row INSERT statement (as opposed to N single-row + executes, whether looped directly or via DBAPI executemany — which + for psycopg2 is ITSELF just a client-side loop of single-row + executes, not one atomic statement) makes the whole batch atomic + under Postgres MVCC: any concurrent reader sees either NONE or ALL of + a batch's rows, never a subset — eliminating the intra-batch race + entirely (the snap-cuts fix separately handles the CROSS-batch edge + case, where the tie spans two batches rather than sitting inside + one).""" + ds = _dataset() + conn = _FakeConn() + pe.insert_votes(conn, ds, 0, ds.n) + assert len(conn.calls) == 1 + + def test_insert_votes_slot_slicing(self): + ds = _dataset() + conn = _FakeConn() + n = pe.insert_votes(conn, ds, 0, 2) + assert n == 2 + assert len(conn.calls) == 1 + + conn2 = _FakeConn() + n2 = pe.insert_votes(conn2, ds, 2, ds.n) + assert n2 == ds.n - 2 + assert len(conn2.calls) == 1 + + def test_insert_votes_empty_slice_issues_no_statement(self): + ds = _dataset() + conn = _FakeConn() + n = pe.insert_votes(conn, ds, ds.n, ds.n) + assert n == 0 + assert len(conn.calls) == 0 + + +# --------------------------------------------------------------------------- # +# insert_mod_events — the moderation-stream analogue of insert_votes. Needed +# to actually exercise pc-meta-02's "interleave-by-timestamp" moderation +# schedule (2026-07-24 session 2: previously ONLY the CSV-based driver +# (schedule.py's slice_schedule) applied mod_events; the live poller-equiv +# feeder never wired this in at all — seed_conversation's own docstring +# flagged it as "the feeder's job", but Stage C never built it). Mirrors +# slice_schedule's EXACT time-windowing semantics (schedule.py:206-225) so +# both drivers attach a mod_event to the same batch, and insert_votes' +# atomicity fix (root cause #5) — one multi-row UPDATE, never a per-row loop. +# --------------------------------------------------------------------------- # +def _dataset_with_mods() -> ReplayDataset: + raw = [ + (1000, 1, 10, 1), (1001, 1, 11, -1), (1002, 2, 10, 0), (1003, 2, 11, 1), + ] + comments = { + 10: CommentMeta(tid=10, created_ms=900, is_meta=False), + 11: CommentMeta(tid=11, created_ms=901, is_meta=False), + } + mod_events = [ + ModEvent(t_ms=1000, tid=10, mod=-1), # lands in the FIRST window (<=1001) + ModEvent(t_ms=1001, tid=11, mod=1), # ALSO the first window (boundary-inclusive) + ModEvent(t_ms=1002, tid=10, mod=1), # second window (>1001, <=1003) — revises tid=10 + ModEvent(t_ms=1500, tid=11, mod=-1), # AFTER the last cut (1003) — dropped, like tail votes + ] + return ReplayDataset.build(raw, comments=comments, mod_events=mod_events) + + +class TestInsertModEvents: + def test_time_window_matches_slice_schedule_semantics(self): + """Mirrors schedule.py's slice_schedule: an event lands in the batch + whose cut_time_ms is the FIRST to reach it — ``prev_time_ms < t_ms + <= cut_time_ms``, ``prev_time_ms=None`` meaning no floor.""" + ds = _dataset_with_mods() + conn = _FakeConn() + n = pe.insert_mod_events(conn, ds, None, 1001, zid=1) + assert n == 2 # tid=10@1000 and tid=11@1001 + assert len(conn.calls) == 1 + sql, params = conn.calls[0] + assert "UPDATE comments" in sql + assert sql.count("VALUES") == 1 + assert params["zid"] == 1 + + def test_second_window_excludes_first_windows_events(self): + ds = _dataset_with_mods() + conn = _FakeConn() + n = pe.insert_mod_events(conn, ds, 1001, 1003, zid=1) + assert n == 1 # only tid=10@1002 + + def test_events_after_the_final_cut_are_dropped(self): + ds = _dataset_with_mods() + conn = _FakeConn() + # Even a huge upper bound only reaches events with t_ms <= cut_time_ms + # given as the argument — the "after the last real cut" drop is the + # CALLER's job (never invoking this with a cut past the schedule), + # exactly like insert_votes' tail-votes convention. + n = pe.insert_mod_events(conn, ds, 1003, 1400, zid=1) + assert n == 0 + assert len(conn.calls) == 0 + + def test_empty_window_issues_no_statement(self): + ds = _dataset_with_mods() + conn = _FakeConn() + n = pe.insert_mod_events(conn, ds, 1600, 1700, zid=1) + assert n == 0 + assert len(conn.calls) == 0 + + def test_no_mod_events_at_all_issues_no_statement(self): + """The common case (vw has none) — must be a total no-op, never even + touching the connection.""" + ds = _dataset() # no mod_events passed -> defaults to () + conn = _FakeConn() + n = pe.insert_mod_events(conn, ds, None, 10_000, zid=1) + assert n == 0 + assert len(conn.calls) == 0 + + def test_multiple_events_for_the_same_tid_in_one_window_keep_the_latest(self): + """Two mod_events for the SAME tid landing in the SAME window (e.g. a + moderator flip-flopping within one batch) must produce exactly ONE + VALUES row for that tid — Postgres's UPDATE...FROM semantics are + UNSPECIFIED when the FROM subquery has multiple rows matching the + same target row, so de-duplication (latest t_ms wins, mirroring + votes' latest-vote-wins) must happen BEFORE the SQL is built, not be + left to the database.""" + raw = [(1000, 1, 10, 1)] + comments = {10: CommentMeta(tid=10, created_ms=900)} + mod_events = [ + ModEvent(t_ms=1000, tid=10, mod=-1), + ModEvent(t_ms=1001, tid=10, mod=1), # supersedes the -1 above + ] + ds = ReplayDataset.build(raw, comments=comments, mod_events=mod_events) + conn = _FakeConn() + n = pe.insert_mod_events(conn, ds, None, 2000, zid=1) + assert n == 2 # raw event count returned (mirrors insert_votes' row count) + sql, params = conn.calls[0] + assert sql.count("VALUES (") == 1 # exactly ONE tuple in the VALUES list + # The LATEST (t_ms=1001) mod value must be the one sent. + mod_values = [v for k, v in params.items() if k.startswith("mod") and not k.startswith("modified")] + assert mod_values == [1] + + +# --------------------------------------------------------------------------- # +# Stage A.2/A.3 — seeder structure + idempotency (fake-conn, no real DB). +# --------------------------------------------------------------------------- # +class TestSeedConversation: + def test_seed_conversation_inserts_conversation_and_all_comments(self): + ds = _dataset() + conn = _FakeConn() + pe.seed_conversation(conn, ds, zid=7) + + conv_calls = [c for c in conn.calls if "INSERT INTO conversations" in c[0]] + comment_calls = [c for c in conn.calls if "INSERT INTO comments" in c[0]] + assert len(conv_calls) == 1 + assert conv_calls[0][1]["zid"] == 7 + assert len(comment_calls) == len(ds.comments) + + by_tid = {c[1]["tid"]: c[1] for c in comment_calls} + assert by_tid[10]["is_meta"] is False + assert by_tid[11]["is_meta"] is True + # Placeholder text only — never real content. + assert by_tid[10]["txt"] == "comment 10" + + def test_seed_conversation_sql_is_idempotent_by_construction(self): + """Both inserts must be conflict-safe (ON CONFLICT ... DO NOTHING) so + re-seeding an already-seeded conversation never raises a duplicate-key + error — required by the spec's "idempotent-safe or fails loudly".""" + ds = _dataset() + conn = _FakeConn() + pe.seed_conversation(conn, ds, zid=1) + for sql, _params in conn.calls: + assert "ON CONFLICT" in sql + assert "DO NOTHING" in sql + + def test_seed_conversation_called_twice_produces_same_call_shape(self): + """Calling seed_conversation twice (e.g. a retried seed) must not + change the SET of statements issued — behavioral idempotency is + verified against a live DB in TestLiveEndToEnd; this checks the + fake-conn call shape is stable across repeats (no accumulation of + distinct non-conflict-safe statements).""" + ds = _dataset() + conn = _FakeConn() + pe.seed_conversation(conn, ds, zid=1) + first_pass = list(conn.calls) + conn.calls.clear() + pe.seed_conversation(conn, ds, zid=1) + second_pass = list(conn.calls) + assert len(first_pass) == len(second_pass) + assert [c[0] for c in first_pass] == [c[0] for c in second_pass] + + +# --------------------------------------------------------------------------- # +# Stage B.1/B.2 — runner env/cmd assembly (NO subprocess launches). +# --------------------------------------------------------------------------- # +class TestRunnerEnvAssembly: + def test_build_clj_env_sets_required_vars(self): + env = pe.build_clj_env( + database_url="postgresql://x/polis_equiv", math_env="clj-ref", + poll_from_days_ago=10000, base_env={"PATH": "/usr/bin", "KEEP": "1"}, + ) + # DATABASE_URL scheme is translated postgresql:// -> postgres:// — see + # test_build_clj_env_translates_postgresql_scheme_to_postgres below for + # the root-cause rationale (Hikari's postgres.clj regex only matches a + # literal "postgres://" prefix; "postgresql://" silently destructures + # to all-nil host/port/user/pass -> ConnectException). + assert env["DATABASE_URL"] == "postgres://x/polis_equiv" + assert env["MATH_ENV"] == "clj-ref" + assert env["POLL_FROM_DAYS_AGO"] == "10000" + # base_env is preserved, not clobbered wholesale. + assert env["PATH"] == "/usr/bin" + assert env["KEEP"] == "1" + + def test_build_clj_env_translates_postgresql_scheme_to_postgres(self): + """ROOT CAUSE #1 (2026-07-24 live debug): create_equiv_db/_url_with_dbname + always hand back a SQLAlchemy-style ``postgresql://`` URL (preserving + whatever scheme --admin-url used). Clojure's ``create-hikari-datasource`` + (postgres.clj:18) parses DATABASE_URL with + ``#"postgres://(?:(.+):(.*)@)?([^:]+)(?::(\\d+))?/(.+)"`` — a regex that + requires the LITERAL prefix "postgres://", not "postgresql://". Feeding + it "postgresql://..." makes ``re-matches`` return nil, so the + destructured user/password/host/port/db are ALL nil, producing + ``jdbc:postgresql://:5432/`` (empty host, wrong port) and a + ConnectException — reproduced live: `clojure -M:run full` crashed with + exactly this stack trace against a real Postgres until the scheme was + corrected to postgres://.""" + for given in ( + "postgresql://u:p@127.0.0.1:15432/polis_equiv", + "postgresql+psycopg2://u:p@127.0.0.1:15432/polis_equiv", + ): + env = pe.build_clj_env(database_url=given, math_env="clj-ref", base_env={}) + assert env["DATABASE_URL"] == "postgres://u:p@127.0.0.1:15432/polis_equiv" + + def test_build_clj_env_formats_a_click_float_default_as_a_bare_integer(self): + """ROOT CAUSE #2 (2026-07-24 live debug): the CLI's + ``--poll-from-days-ago`` option is ``type=float, default=10000`` — Click + resolves that default THROUGH the type, so the value the CLI actually + passes to build_clj_env is the FLOAT 10000.0, not the int 10000 (every + existing test in this class calls build_clj_env with a bare int literal, + which never exercised this path). ``str(10000.0)`` is "10000.0", and + Clojure's ``->long`` config parser (config.clj) is + ``Long/parseLong`` — which THROWS on "10000.0", caught + logged as a + warning, returning nil. ``deep-merge`` then REPLACES (not falls back to) + the default 10 with that nil, and ``polismath.poller/poll`` computes + ``(* nil 1000 60 60 24)`` — reproduced live as + ``Execution error (NullPointerException) at polismath.poller/poll + (poller.clj:15)``, silently swallowed inside the async go-loop's + result channel (never printed) until the DATABASE_URL fix above let the + Postgres component actually start.""" + env = pe.build_clj_env( + database_url="postgresql://x/polis_equiv", math_env="clj-ref", + poll_from_days_ago=10000.0, base_env={}, + ) + assert env["POLL_FROM_DAYS_AGO"] == "10000" + + def test_build_clj_env_rounds_a_fractional_days_ago_to_the_nearest_integer(self): + env = pe.build_clj_env( + database_url="postgresql://x/polis_equiv", math_env="clj-ref", + poll_from_days_ago=3.6, base_env={}, + ) + assert env["POLL_FROM_DAYS_AGO"] == "4" + + def test_build_clj_env_defaults_logging_level_to_info(self): + """RUNNER EVIDENCE (2026-07-24 live-debug task, REQUIRED FIX #3): + the clj container's default logging level is :warn (config.clj + defaults map) — at :warn, EVERY application-level trace (poll + cycles, conv-manager batch processing, recompute completion) is + silently suppressed, leaving a captured runner log with nothing but + HikariCP connection-pool heartbeats. This is not cosmetic: a stalled + or slow-to-converge container is INDISTINGUISHABLE from a crashed + one without this. ``LOGGING_LEVEL`` is the env var + ``polismath.components.logger`` honors (config.clj's + ``:logging-level`` rule -> ``get-in config [:logging :level]``).""" + env = pe.build_clj_env( + database_url="postgresql://x/polis_equiv", math_env="clj-ref", base_env={}, + ) + assert env["LOGGING_LEVEL"] == "info" + + def test_build_clj_env_logging_level_is_overridable(self): + env = pe.build_clj_env( + database_url="postgresql://x/polis_equiv", math_env="clj-ref", base_env={}, + logging_level="debug", + ) + assert env["LOGGING_LEVEL"] == "debug" + + def test_build_py_env_sets_required_vars_including_engine_mode(self): + env = pe.build_py_env( + database_url="postgresql://x/polis_equiv", math_env="py-shadow", + poll_from_days_ago=10000, engine_mode="clojure-legacy", + base_env={}, + ) + assert env["DATABASE_URL"] == "postgresql://x/polis_equiv" + assert env["MATH_ENV"] == "py-shadow" + assert env["POLL_FROM_DAYS_AGO"] == "10000" + assert env[ENGINE_MODE_ENV_VAR] == "clojure-legacy" + + def test_build_py_env_default_engine_mode_is_legacy(self): + env = pe.build_py_env(database_url="x", math_env="e", base_env={}) + assert env[ENGINE_MODE_ENV_VAR] == ENGINE_MODE_LEGACY + + def test_build_py_env_forces_postgresql_scheme(self): + """Symmetric guard to the clj-side scheme fix: SQLAlchemy/psycopg2 no + longer accept the bare "postgres://" scheme (dropped in SQLAlchemy + 1.4+, raises 'plain "postgres" dialect is no longer supported') — so + even if a caller's admin-url happened to use "postgres://" (the SAME + scheme the clj side needs), the py side must always get + "postgresql://".""" + env = pe.build_py_env( + database_url="postgres://u:p@127.0.0.1:15432/polis_equiv", + math_env="py-shadow", base_env={}, + ) + assert env["DATABASE_URL"] == "postgresql://u:p@127.0.0.1:15432/polis_equiv" + + def test_build_py_env_defaults_database_ssl_mode_to_disable(self): + """ROOT CAUSE #3 (2026-07-24 live debug): ``scripts/math_poller.py`` + never loads a .env file — ``PollerConfig.from_env``'s DATABASE_URL + comes straight from the process env this harness constructs. + ``polismath.database.postgres.PostgresClient`` defaults ``ssl_mode`` to + ``os.environ.get("DATABASE_SSL_MODE", "require")`` (postgres.py:92) + when the caller (math_poller.py's ``_build_service``) doesn't pass one + explicitly — which it doesn't. Reproduced live: the py poller looped + forever on ``psycopg2.OperationalError: ... server does not support + SSL, but SSL was required`` against the harness's local (non-SSL) + Postgres target, until DATABASE_SSL_MODE=disable was set explicitly.""" + env = pe.build_py_env(database_url="postgresql://x/polis_equiv", math_env="e", base_env={}) + assert env["DATABASE_SSL_MODE"] == "disable" + + def test_build_py_env_database_ssl_mode_is_overridable(self): + env = pe.build_py_env( + database_url="postgresql://x/polis_equiv", math_env="e", base_env={}, + database_ssl_mode="require", + ) + assert env["DATABASE_SSL_MODE"] == "require" + + def test_clj_container_runner_cmd_cwd_env(self): + runner = pe.CljContainerRunner( + database_url="postgresql://x/polis_equiv", math_env="clj-ref", + base_env={}, + ) + assert runner.cmd == ["clojure", "-M:run", "full"] + assert runner.cwd == pe._MATH_ROOT + assert runner.cwd.name == "math" + assert runner.env["DATABASE_URL"] == "postgres://x/polis_equiv" + assert runner.env["MATH_ENV"] == "clj-ref" + assert runner.env["POLL_FROM_DAYS_AGO"] == "10000" + assert runner._proc is None # never started + assert runner.pid is None + assert runner.is_alive() is False + + def test_py_poller_runner_cmd_cwd_env(self): + runner = pe.PyPollerRunner( + database_url="postgresql://x/polis_equiv", math_env="py-shadow", + base_env={}, + ) + assert runner.cmd == ["uv", "run", "python", "scripts/math_poller.py"] + # Equality with _DELPHI_ROOT is the contract; asserting the directory + # NAME was layout-fragile — CI mounts the delphi tree at /app, where + # .name == "app" (python-ci run 30071088647, 2026-07-24). + assert runner.cwd == pe._DELPHI_ROOT + assert runner.env["MATH_ENV"] == "py-shadow" + assert runner.env[ENGINE_MODE_ENV_VAR] == "clojure-legacy" + assert runner.env["DATABASE_SSL_MODE"] == "disable" + assert runner._proc is None + + def test_math_root_and_delphi_root_are_siblings(self): + assert pe._MATH_ROOT.parent == pe._DELPHI_ROOT.parent + + +class TestSubprocessRunnerKill: + """kill() lifecycle via a mocked Popen — no real process is ever spawned.""" + + def test_kill_is_noop_when_never_started(self): + runner = pe._SubprocessRunner(["true"], cwd=Path("."), env={}) + runner.kill(grace=0.01) # must not raise + + def test_kill_is_noop_when_already_exited(self): + runner = pe._SubprocessRunner(["true"], cwd=Path("."), env={}) + fake_proc = MagicMock() + fake_proc.poll.return_value = 0 # already exited + runner._proc = fake_proc + runner.kill(grace=0.01) + fake_proc.terminate.assert_not_called() + fake_proc.kill.assert_not_called() + + def test_kill_sigterm_succeeds_without_sigkill(self): + runner = pe._SubprocessRunner(["true"], cwd=Path("."), env={}) + fake_proc = MagicMock() + fake_proc.poll.return_value = None + fake_proc.wait.return_value = 0 # terminate() succeeds within grace + runner._proc = fake_proc + runner.kill(grace=5.0) + fake_proc.terminate.assert_called_once() + fake_proc.kill.assert_not_called() + + def test_kill_escalates_to_sigkill_after_grace_timeout(self): + runner = pe._SubprocessRunner(["true"], cwd=Path("."), env={}) + fake_proc = MagicMock() + fake_proc.poll.return_value = None + fake_proc.wait.side_effect = [ + subprocess.TimeoutExpired(cmd="true", timeout=0.01), + 0, + ] + runner._proc = fake_proc + runner.kill(grace=0.01) + fake_proc.terminate.assert_called_once() + fake_proc.kill.assert_called_once() + assert fake_proc.wait.call_count == 2 + + +# --------------------------------------------------------------------------- # +# RUNNER EVIDENCE — capture stdout+stderr to a log file under --out (real +# short-lived subprocesses; no PIPE is left undrained). REQUIRED FIX #3 from +# the 2026-07-24 live-debug task: without this, a runner crashing at startup +# (e.g. root causes #1-#3 above) leaves NO trace anywhere the harness looks. +# --------------------------------------------------------------------------- # +class TestSubprocessRunnerLogCapture: + def test_log_path_none_preserves_pipe_behavior(self, tmp_path): + """Default (no log_path) — same PIPE-based behavior the standalone + `run-clj`/`run-py` CLI subcommands stream from (unchanged contract).""" + runner = pe._SubprocessRunner( + ["sh", "-c", "echo hi"], cwd=tmp_path, env={"PATH": os.environ["PATH"]}, + ) + proc = runner.start() + assert proc.stdout is not None + out = proc.stdout.read() + proc.wait(timeout=5) + assert "hi" in out + + def test_log_path_captures_stdout_and_stderr_to_file(self, tmp_path): + log_path = tmp_path / "clj-ref.runner.log" + runner = pe._SubprocessRunner( + ["sh", "-c", "echo out-line; echo err-line 1>&2"], + cwd=tmp_path, env={"PATH": os.environ["PATH"]}, log_path=log_path, + ) + proc = runner.start() + assert proc.stdout is None # not piped — went straight to the file + proc.wait(timeout=5) + runner.kill(grace=0.01) # closes the log file handle + text = log_path.read_text() + assert "out-line" in text + assert "err-line" in text + + def test_log_path_parent_dir_is_created(self, tmp_path): + log_path = tmp_path / "nested" / "dir" / "py-shadow.runner.log" + runner = pe._SubprocessRunner( + ["sh", "-c", "echo hi"], cwd=tmp_path, env={"PATH": os.environ["PATH"]}, + log_path=log_path, + ) + runner.start().wait(timeout=5) + runner.kill(grace=0.01) + assert log_path.exists() + + def test_restart_appends_rather_than_truncating(self, tmp_path): + """A seam restart must not erase the PRE-seam evidence — the new + process's runner reuses the SAME log path and must append.""" + log_path = tmp_path / "py-shadow.runner.log" + first = pe._SubprocessRunner( + ["sh", "-c", "echo first-run"], cwd=tmp_path, env={"PATH": os.environ["PATH"]}, + log_path=log_path, + ) + first.start().wait(timeout=5) + first.kill(grace=0.01) + + second = pe._SubprocessRunner( + ["sh", "-c", "echo second-run"], cwd=tmp_path, env={"PATH": os.environ["PATH"]}, + log_path=log_path, + ) + second.start().wait(timeout=5) + second.kill(grace=0.01) + + text = log_path.read_text() + assert "first-run" in text + assert "second-run" in text + + def test_runner_constructors_accept_log_path(self, tmp_path): + log_path = tmp_path / "clj-ref.runner.log" + runner = pe.CljContainerRunner( + database_url="postgresql://x/polis_equiv", math_env="clj-ref", + base_env={}, log_path=log_path, + ) + assert runner.log_path == log_path + + py_log = tmp_path / "py-shadow.runner.log" + py_runner = pe.PyPollerRunner( + database_url="postgresql://x/polis_equiv", math_env="py-shadow", + base_env={}, log_path=py_log, + ) + assert py_runner.log_path == py_log + + +# --------------------------------------------------------------------------- # +# Stage B.3 — wait_for_tick (mocked connection, no real sleeping). +# --------------------------------------------------------------------------- # +class _FakeClock: + def __init__(self, start: float = 0.0, step: float = 1.0): + self.t = start + self.step = step + self.sleep_calls = 0 + + def now(self) -> float: + return self.t + + def sleep(self, seconds: float) -> None: + self.sleep_calls += 1 + self.t += self.step + + +class _SequenceConn: + """Returns rows from a fixed sequence, one per execute() call (clamped to + the last row once exhausted).""" + + def __init__(self, rows): + self._rows = list(rows) + self.n_calls = 0 + + def execute(self, stmt, params=None): + idx = min(self.n_calls, len(self._rows) - 1) + self.n_calls += 1 + return _FakeResult(self._rows[idx]) + + +class TestWaitForTick: + def test_returns_row_once_predicate_matches(self): + conn = _SequenceConn([None, {"caching_tick": 1}, {"caching_tick": 2}]) + clock = _FakeClock() + row = pe.wait_for_tick( + conn, "clj-ref", 1, lambda r: r["caching_tick"] >= 2, + timeout=100, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + ) + assert row == {"caching_tick": 2} + assert conn.n_calls == 3 + + def test_returns_none_on_timeout(self): + conn = _SequenceConn([{"caching_tick": 0}]) + clock = _FakeClock(start=0.0, step=1.0) + row = pe.wait_for_tick( + conn, "clj-ref", 1, lambda r: r["caching_tick"] >= 99, + timeout=3, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + ) + assert row is None + assert clock.sleep_calls == 3 + + def test_none_row_never_satisfies_predicate(self): + """A predicate that doesn't guard against None must not raise before + the row exists (no math_main row yet == a cold zid).""" + conn = _SequenceConn([None, None, {"caching_tick": 1}]) + clock = _FakeClock() + row = pe.wait_for_tick( + conn, "clj-ref", 1, lambda r: r.get("caching_tick") == 1, + timeout=100, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + ) + assert row == {"caching_tick": 1} + + def test_uses_zid_and_math_env_in_query_params(self): + captured = {} + + class _CapturingConn: + def execute(self, stmt, params=None): + captured.update(params or {}) + return _FakeResult({"caching_tick": 1}) + + clock = _FakeClock() + pe.wait_for_tick( + _CapturingConn(), "py-shadow", 42, lambda r: True, + timeout=1, sleep=clock.sleep, now=clock.now, + ) + assert captured == {"zid": 42, "math_env": "py-shadow"} + + +# --------------------------------------------------------------------------- # +# Quirk Q19 harness-level mitigation — wait-for-first-poll-cycle gate. +# ``conv_man.clj``'s ``queue-message-batch!`` has an unsynchronized +# check-then-act race spinning up TWO independent conv-actors for a +# brand-new zid whenever the :votes and :moderation pollers BOTH discover +# data for it on their very first poll tick (near-guaranteed by this +# harness's OWN timing: seed data + batch 0 land in the DB before the JVM +# even finishes booting). ``_poll_cycle_signal_seen``/ +# ``wait_for_first_poll_cycle`` delay feeding batch 0 until we've observed +# the clj runner log show AT LEAST ONE completed ``:votes`` poll cycle — by +# construction that cycle found ZERO rows (we haven't inserted any yet), so +# NO queue-message-batch! call happens from the votes side at all, meaning +# only ONE poller (moderation, discovering the already-seeded comments) can +# EVER be first to create the actor — no race, regardless of scheduling. +# ``polismath.poller/poll`` (poller.clj:24) emits ``"Polling > +# "`` UNCONDITIONALLY on every cycle (found rows or not), which +# is what makes this a reliable, log-based signal rather than a fixed sleep. +# --------------------------------------------------------------------------- # +class TestPollCycleSignalSeen: + def test_absent_when_log_is_empty(self): + assert pe._poll_cycle_signal_seen("") is False + + def test_absent_when_log_has_only_hikari_chatter(self): + text = "05:19:54.132 [main] INFO com.zaxxer.hikari.HikariDataSource - HikariPool-1 - Start completed.\n" + assert pe._poll_cycle_signal_seen(text) is False + + def test_present_after_a_real_poll_line(self): + text = "2026-07-24T03:57:30.497Z device-137.home INFO [polismath.poller:24] - Polling :votes > 1732029094000\n" + assert pe._poll_cycle_signal_seen(text) is True + + def test_message_type_is_selective(self): + text = "INFO [polismath.poller:24] - Polling :moderation > 123\n" + assert pe._poll_cycle_signal_seen(text, message_type="votes") is False + assert pe._poll_cycle_signal_seen(text, message_type="moderation") is True + + def test_default_message_type_is_votes(self): + text = "INFO [polismath.poller:24] - Polling :votes > 0\n" + assert pe._poll_cycle_signal_seen(text) is True + + +class TestWaitForFirstPollCycle: + def test_observed_true_once_the_signal_appears(self, tmp_path): + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("boot chatter only\n") + clock = _FakeClock() + + # Simulate the signal landing mid-wait by mutating the file from + # inside a custom sleep callback — the poll loop re-reads the file + # on every iteration, exactly like the real subprocess appending to + # it over time. + def sleep_then_append(seconds): + clock.sleep(seconds) + log_path.write_text(log_path.read_text() + "Polling :votes > 0\n") + + result = pe.wait_for_first_poll_cycle( + log_path, timeout=10, poll_interval=1.0, sleep=sleep_then_append, now=clock.now, + ) + assert result["observed"] is True + assert result["elapsed_s"] >= 0 + + def test_observed_false_on_timeout(self, tmp_path): + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("boot chatter only, never a poll line\n") + clock = _FakeClock() + result = pe.wait_for_first_poll_cycle( + log_path, timeout=3, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + ) + assert result["observed"] is False + assert result["reason"] == "timeout" + + def test_none_log_path_is_immediately_not_observed(self): + clock = _FakeClock() + result = pe.wait_for_first_poll_cycle( + None, timeout=10, sleep=clock.sleep, now=clock.now, + ) + assert result["observed"] is False + assert clock.sleep_calls == 0 # never even waits — nothing to poll + + def test_missing_log_file_is_treated_as_empty_not_an_error(self, tmp_path): + log_path = tmp_path / "never-created.log" + clock = _FakeClock() + result = pe.wait_for_first_poll_cycle( + log_path, timeout=2, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + ) + assert result["observed"] is False + assert result["reason"] == "timeout" + + def test_already_present_signal_returns_immediately_without_sleeping(self, tmp_path): + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("...\nPolling :votes > 0\n...\n") + clock = _FakeClock() + result = pe.wait_for_first_poll_cycle( + log_path, timeout=10, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + ) + assert result["observed"] is True + assert clock.sleep_calls == 0 + + def test_start_offset_ignores_stale_pre_offset_content(self, tmp_path): + """ROOT CAUSE (found live, 2026-07-24 session 2): the runner log is + opened in APPEND mode (:class:`_SubprocessRunner`, so a seam + restart's post-restart output lands in the SAME file as the + pre-restart run). A FRESH container's cold-start gate check must + NOT be satisfied by a "Polling :votes >" line left over from a + PREVIOUS attempt sitting earlier in the SAME file — that content + proves nothing about whether THIS instance has polled yet. Without + ``start_offset``, the gate is a no-op after the first-ever run in a + given --out directory (silently defeating the whole Q19 + mitigation) — reproduced live: the mitigation's very first + real-world run still hit quirk Q19, because the gate was satisfied + instantly by stale text.""" + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("Polling :votes > 999\n") # stale, from a PRIOR attempt + offset = log_path.stat().st_size + clock = _FakeClock() + result = pe.wait_for_first_poll_cycle( + log_path, timeout=3, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + start_offset=offset, + ) + assert result["observed"] is False + assert result["reason"] == "timeout" + + def test_start_offset_still_observes_genuinely_new_content(self, tmp_path): + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("stale boot chatter from a prior attempt\n") + offset = log_path.stat().st_size + + def sleep_then_append(seconds): + with open(log_path, "a") as fh: + fh.write("Polling :votes > 0\n") + + clock = _FakeClock() + result = pe.wait_for_first_poll_cycle( + log_path, timeout=10, poll_interval=1.0, sleep=sleep_then_append, now=clock.now, + start_offset=offset, + ) + assert result["observed"] is True + + def test_start_offset_defaults_to_zero_reading_the_whole_file(self, tmp_path): + """Backward-compatible default — every EXISTING caller/test above + (offset unspecified) reads from the start of the file, unchanged.""" + log_path = tmp_path / "clj-ref.runner.log" + log_path.write_text("Polling :votes > 0\n") + clock = _FakeClock() + result = pe.wait_for_first_poll_cycle( + log_path, timeout=10, poll_interval=1.0, sleep=clock.sleep, now=clock.now, + ) + assert result["observed"] is True + + +# --------------------------------------------------------------------------- # +# CLI stub smoke test — the click group loads and exposes the stage A/B +# subcommands (no real seeding/subprocess is exercised here). +# --------------------------------------------------------------------------- # +_CLI_PATH = Path(__file__).resolve().parents[2] / "scripts" / "poller_equiv.py" + + +def _cli_module(): + spec = importlib.util.spec_from_file_location("poller_equiv_cli", _CLI_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class TestCliStub: + def test_cli_help_lists_stage_ab_subcommands(self): + mod = _cli_module() + result = CliRunner().invoke(mod.cli, ["--help"]) + assert result.exit_code == 0 + assert "seed" in result.output + assert "run-clj" in result.output + assert "run-py" in result.output + + +# --------------------------------------------------------------------------- # +# Live-Postgres end-to-end (gated — self-skips without POLLER_EQUIV_PG_URL). +# --------------------------------------------------------------------------- # +@_needs_live_pg +class TestLiveEndToEnd: + """Exercises create_equiv_db + seed_conversation + insert_votes against a + REAL Postgres server. Set POLLER_EQUIV_PG_URL to an admin connection + (e.g. postgresql://postgres:postgres@localhost:15432/postgres) pointing at + a database OTHER than the equiv db itself to run these.""" + + DBNAME = "polis_equiv_test" + + def test_create_seed_and_reseed_is_idempotent(self): + target_url = pe.create_equiv_db(PG_URL, dbname=self.DBNAME) + engine = sa.create_engine(target_url) + try: + ds = _dataset() + with engine.begin() as conn: + pe.seed_conversation(conn, ds, zid=1) + pe.seed_conversation(conn, ds, zid=1) # re-seed: must not raise + + with engine.connect() as conn: + n_convs = conn.execute( + sa.text("SELECT COUNT(*) FROM conversations WHERE zid = 1") + ).scalar() + n_comments = conn.execute( + sa.text("SELECT COUNT(*) FROM comments WHERE zid = 1") + ).scalar() + assert n_convs == 1 + assert n_comments == len(ds.comments) + finally: + engine.dispose() + + def test_insert_votes_lands_raw_db_sign_convention(self): + target_url = pe.create_equiv_db(PG_URL, dbname=self.DBNAME) + engine = sa.create_engine(target_url) + try: + ds = _dataset() + with engine.begin() as conn: + pe.seed_conversation(conn, ds, zid=1) + pe.insert_votes(conn, ds, 0, ds.n, zid=1) + + with engine.connect() as conn: + rows = conn.execute( + sa.text("SELECT pid, tid, vote FROM votes WHERE zid = 1 " + "ORDER BY pid, tid") + ).mappings().all() + by_pid_tid = {(r["pid"], r["tid"]): r["vote"] for r in rows} + for v in ds.votes: + assert by_pid_tid[(v.pid, v.tid)] == delphi_vote_to_postgres(v.sign) + finally: + engine.dispose() + + def test_participants_table_exists_and_empty(self): + """The participants SHIM must exist (py poll_moderation depends on it) + but is never seeded.""" + target_url = pe.create_equiv_db(PG_URL, dbname=self.DBNAME) + engine = sa.create_engine(target_url) + try: + with engine.connect() as conn: + count = conn.execute(sa.text("SELECT COUNT(*) FROM participants")).scalar() + assert count == 0 + finally: + engine.dispose()