diff --git a/delphi/polismath/replay/certify.py b/delphi/polismath/replay/certify.py index d36db97a0..9453e9045 100644 --- a/delphi/polismath/replay/certify.py +++ b/delphi/polismath/replay/certify.py @@ -54,7 +54,13 @@ from polismath.replay import real_data from polismath.replay import schedule as sched from polismath.replay import store as st -from polismath.replay.crosslang import PREP_MAIN_KEYS, _kebab, load_clj_blobs, project_prep_main +from polismath.replay.crosslang import ( + PREP_MAIN_KEYS, + _kebab, + canonicalize_blob, + load_clj_blobs, + project_prep_main, +) from polismath.replay.stepcompare import DEFAULT_TOLERANT_STAT_KEYS, StepComparer from polismath.replay.types import ReplayDataset from polismath.utils.engine_mode import ( @@ -91,10 +97,15 @@ def project_acceptance(blob: dict[str, Any]) -> dict[str, Any]: """Project a math_main blob onto :data:`ACCEPTANCE_KEYS` (kebab-canonical). Reuses :func:`polismath.replay.crosslang.project_prep_main` for the - snake/kebab canonicalisation, then drops the dead subgroup-* trio. + snake/kebab canonicalisation, drops the dead subgroup-* trio, then + order-canonicalizes via :func:`polismath.replay.crosslang.canonicalize_blob` + so cross-engine-arbitrary array orderings (Clojure hash order vs Python + sorted) neither diverge in the comparer nor break the hash-first shortcut. """ proj = project_prep_main(blob) - return {k: v for k, v in proj.items() if k not in ACCEPTANCE_EXCLUDED_KEYS} + return canonicalize_blob( + {k: v for k, v in proj.items() if k not in ACCEPTANCE_EXCLUDED_KEYS} + ) def _acceptance_projecting_comparer(**kwargs: Any) -> StepComparer: diff --git a/delphi/polismath/replay/crosslang.py b/delphi/polismath/replay/crosslang.py index 7583e80c5..b8f1d9aca 100644 --- a/delphi/polismath/replay/crosslang.py +++ b/delphi/polismath/replay/crosslang.py @@ -76,6 +76,168 @@ def project_prep_main(blob: dict[str, Any]) -> dict[str, Any]: return {k: canon[k] for k in canon if k in PREP_MAIN_KEYS} +# --------------------------------------------------------------------------- +# Order canonicalization for cross-engine comparison. +# --------------------------------------------------------------------------- +# Clojure emits tids/in-conv (and everything positionally aligned to them) in +# hash/insertion order — :tids is (nm/colnames rating-mat) (conversation.clj:210), +# base-clusters emission preserves conv-state order (fold-clusters, +# clusters.clj:389) — while Python emits sorted order. Each blob is INTERNALLY +# consistent, so cross-engine array order is not a semantic divergence; the +# acceptance criterion (GOAL_R1_PARITY.md) is membership + value parity. +# NOTE: votes-base A/D/S bucket lists are aligned to sort-by-:id order on BOTH +# engines (bid-to-pid = (mapv :members (sort-by :id base-clusters)), +# conversation.clj:593) — already canonical, never permuted here. + +_SET_SEMANTIC_KEYS = ("in-conv", "mod-in", "mod-out", "meta-tids") + + +def _permutation(values: list) -> list[int]: + """Indices that sort ``values`` ascending (stable).""" + return sorted(range(len(values)), key=lambda i: values[i]) + + +def canonicalize_blob(blob: dict[str, Any]) -> dict[str, Any]: + """Return ``blob`` with cross-engine-arbitrary orderings normalized. + + - ``tids`` sorted; ``pca`` arrays indexed by tid (center, each comps / + comment-projection row, comment-extremity) re-indexed by the same + permutation. Arrays whose length does not match ``tids`` are left alone. + - ``in-conv`` / ``mod-in`` / ``mod-out`` / ``meta-tids`` sorted when lists + (``None`` passes through untouched — a None-vs-[] difference is a real + shape divergence and must stay visible). + - ``base-clusters`` columns re-indexed by sorted id; each ``members`` list + sorted. + - ``group-clusters`` sorted by id; each ``members`` list sorted. + + Purely structural: never rewrites values, only their order — a genuine + membership or numeric divergence survives canonicalization on both sides. + """ + b = dict(blob) + + tids = b.get("tids") + if isinstance(tids, list) and tids and all( + isinstance(t, (int, float)) for t in tids + ): + order = _permutation(tids) + b["tids"] = [tids[i] for i in order] + pca = b.get("pca") + if isinstance(pca, dict): + n = len(tids) + + def _by_tid(v: Any) -> Any: + if isinstance(v, list) and len(v) == n: + return [v[i] for i in order] + return v + + pca = dict(pca) + for key in ("center", "comment-extremity"): + if key in pca: + pca[key] = _by_tid(pca[key]) + for key in ("comps", "comment-projection"): + rows = pca.get(key) + if isinstance(rows, list): + pca[key] = [_by_tid(row) for row in rows] + b["pca"] = pca + + # PCA component signs are run-arbitrary: Clojure's first-tick power + # iteration has no start vectors, so its unseeded init flips component + # signs BETWEEN ITS OWN RUNS (observed 2026-07-22: a vw single-cut + # re-record negated comps[1] + base-clusters.y vs the prior recording). + # Canonicalize each component's sign deterministically — the max-|value| + # entry (first index on ties) made positive, evaluated AFTER the tid + # alignment above so both engines test the same column order — and flip + # every component-aligned array with it: comps row, comment-projection + # row, base-clusters x (comp 0) / y (comp 1), group-clusters center[k]. + # pca.center is a data mean, not sign-arbitrary — never flipped. A + # near-zero component row makes the flip choice noise-driven, but its + # projections are equally near-zero and fall inside numeric tolerance. + pca = b.get("pca") + if isinstance(pca, dict) and isinstance(pca.get("comps"), list): + flips = [] + for row in pca["comps"]: + if isinstance(row, list) and row and all( + isinstance(v, (int, float)) for v in row + ): + idx = max(range(len(row)), key=lambda i: (abs(row[i]), -i)) + flips.append(-1.0 if row[idx] < 0 else 1.0) + else: + flips.append(1.0) + if any(f < 0 for f in flips): + def _flip_rows(rows: Any) -> Any: + if not isinstance(rows, list): + return rows + return [ + [f * v for v in row] if isinstance(row, list) and f < 0 else row + for f, row in zip(flips, rows) + ] + + pca = dict(pca) + pca["comps"] = _flip_rows(pca["comps"]) + if "comment-projection" in pca: + pca["comment-projection"] = _flip_rows(pca["comment-projection"]) + b["pca"] = pca + + bc = b.get("base-clusters") + if isinstance(bc, dict): + bc = dict(bc) + for f, key in zip(flips, ("x", "y")): + if f < 0 and isinstance(bc.get(key), list): + bc[key] = [-v for v in bc[key]] + b["base-clusters"] = bc + + gc = b.get("group-clusters") + if isinstance(gc, list): + canon_gc = [] + for g in gc: + if isinstance(g, dict) and isinstance(g.get("center"), list): + g = dict(g) + g["center"] = [ + (flips[i] * v if i < len(flips) else v) + for i, v in enumerate(g["center"]) + ] + canon_gc.append(g) + b["group-clusters"] = canon_gc + + for key in _SET_SEMANTIC_KEYS: + v = b.get(key) + if isinstance(v, list) and all(isinstance(x, (int, float)) for x in v): + b[key] = sorted(v) + + bc = b.get("base-clusters") + if ( + isinstance(bc, dict) + and isinstance(bc.get("id"), list) + and all(isinstance(x, (int, float)) for x in bc["id"]) + ): + n = len(bc["id"]) + order = _permutation(bc["id"]) + bc = { + k: ([v[i] for i in order] if isinstance(v, list) and len(v) == n else v) + for k, v in bc.items() + } + members = bc.get("members") + if isinstance(members, list): + bc["members"] = [ + sorted(m) if isinstance(m, list) else m for m in members + ] + b["base-clusters"] = bc + + gc = b.get("group-clusters") + if isinstance(gc, list) and all(isinstance(g, dict) for g in gc): + canon_gc = [] + for g in gc: + g = dict(g) + if isinstance(g.get("members"), list): + g["members"] = sorted(g["members"]) + canon_gc.append(g) + if all(isinstance(g.get("id"), (int, float)) for g in canon_gc): + canon_gc.sort(key=lambda g: g["id"]) + b["group-clusters"] = canon_gc + + return b + + def clj_blob_files(clj_dir: str | Path) -> list[Path]: """The ``step-NNN.blob.json`` files in a clj recording dir, in step order. diff --git a/delphi/tests/replay_harness/test_certify_canonicalization.py b/delphi/tests/replay_harness/test_certify_canonicalization.py new file mode 100644 index 000000000..29276e63f --- /dev/null +++ b/delphi/tests/replay_harness/test_certify_canonicalization.py @@ -0,0 +1,235 @@ +"""Acceptance-projection canonicalization — ordering-artifact suppression. + +The first full battery run (journal 2026-07-22) showed 4/4 entries diverging at +step 0 with ~800+ "exact" divergences — nearly all of them ORDERING artifacts: +Clojure emits ``tids``/``in-conv`` (and everything positionally aligned to +them: pca.center, pca.comps rows, base-clusters columns, votes-base lists) in +hash/insertion order, while Python emits sorted order. Each blob is internally +consistent, so cross-engine array order is not a semantic divergence — the +acceptance criterion (GOAL_R1_PARITY.md) is MEMBERSHIP and value parity. + +``project_acceptance`` therefore canonicalizes both sides before hashing and +diffing: id-sets sorted, tid-aligned pca arrays re-indexed by sorted tid, +base-clusters columns re-indexed by sorted id (votes-base per-cluster lists +following the same permutation), group-clusters sorted by id with sorted +members. Real divergences (a differing pid, a differing center value for the +SAME tid) must still be reported — canonicalization must never mask them. +""" + +from __future__ import annotations + +import copy + +from polismath.replay import certify as cert + + +# --------------------------------------------------------------------------- +# Two semantically identical blobs, emitted in different orders. +# --------------------------------------------------------------------------- +# "Clojure-ordered": tids in hash order [2, 0, 1]; in-conv in hash order; +# base-clusters columns in conv-state order [1, 0] (fold-clusters preserves it, +# clusters.clj:389); group-clusters listed [1, 0] with unsorted members. +# votes-base A/D/S lists are ALWAYS aligned to sort-by-id bucket order on both +# engines (bid-to-pid = (mapv :members (sort-by :id base-clusters)), +# conversation.clj:593) — so they are identical across the two orderings and +# the canonicalizer must NOT permute them. +def _clj_ordered_blob() -> dict: + return { + "zid": "t", + "n": 4, + "n-cmts": 3, + "in-conv": [30, 10, 20], + "tids": [2, 0, 1], + "mod-in": [2, 1], + "mod-out": [5, 3], + "meta-tids": [7, 6], + "pca": { + # aligned to tids [2, 0, 1] + "center": [0.3, 0.1, 0.2], + "comps": [[0.32, 0.12, 0.22], [0.33, 0.13, 0.23]], + "comment-projection": [[3.2, 1.2, 2.2], [3.3, 1.3, 2.3]], + "comment-extremity": [3.0, 1.0, 2.0], + }, + "base-clusters": { + # columns aligned to id order [1, 0] + "id": [1, 0], + "x": [-0.1, 0.1], + "y": [-0.2, 0.2], + "count": [2, 1], + "members": [[30, 20], [10]], + }, + "votes-base": { + # per-tid A/D/S lists in sort-by-id bucket order [0, 1] — same as + # the py side, despite base-clusters columns being emitted [1, 0] + "0": {"A": [1, 2], "D": [0, 0], "S": [1, 2]}, + "1": {"A": [0, 1], "D": [1, 1], "S": [1, 2]}, + "2": {"A": [1, 0], "D": [0, 2], "S": [1, 2]}, + }, + "group-clusters": [ + {"id": 1, "members": [1], "center": [-0.1, -0.2]}, + {"id": 0, "members": [0], "center": [0.1, 0.2]}, + ], + "repness": {}, + } + + +# "Python-ordered": identical content, every set/alignment sorted ascending. +def _py_sorted_blob() -> dict: + return { + "zid": "t", + "n": 4, + "n-cmts": 3, + "in-conv": [10, 20, 30], + "tids": [0, 1, 2], + "mod-in": [1, 2], + "mod-out": [3, 5], + "meta-tids": [6, 7], + "pca": { + # aligned to tids [0, 1, 2] + "center": [0.1, 0.2, 0.3], + "comps": [[0.12, 0.22, 0.32], [0.13, 0.23, 0.33]], + "comment-projection": [[1.2, 2.2, 3.2], [1.3, 2.3, 3.3]], + "comment-extremity": [1.0, 2.0, 3.0], + }, + "base-clusters": { + # columns aligned to id order [0, 1] + "id": [0, 1], + "x": [0.1, -0.1], + "y": [0.2, -0.2], + "count": [1, 2], + "members": [[10], [20, 30]], + }, + "votes-base": { + # per-tid A/D/S lists aligned to base-clusters id order [0, 1] + "0": {"A": [1, 2], "D": [0, 0], "S": [1, 2]}, + "1": {"A": [0, 1], "D": [1, 1], "S": [1, 2]}, + "2": {"A": [1, 0], "D": [0, 2], "S": [1, 2]}, + }, + "group-clusters": [ + {"id": 0, "members": [0], "center": [0.1, 0.2]}, + {"id": 1, "members": [1], "center": [-0.1, -0.2]}, + ], + "repness": {}, + } + + +def _diverging_paths(blob_a: dict, blob_b: dict) -> list[str]: + cmp = cert._acceptance_projecting_comparer() + report = cmp.compare_step(blob_a, blob_b, 0) + return [d["path"] for fam in ("exact", "tolerant") for d in report["families"][fam]] + + +# --------------------------------------------------------------------------- +# Pure-ordering differences must vanish. +# --------------------------------------------------------------------------- +def test_ordering_only_differences_do_not_diverge(): + assert _diverging_paths(_clj_ordered_blob(), _py_sorted_blob()) == [] + + +def test_canonical_hashes_equal_after_reordering(): + ha = cert._canonical_hash(cert.project_acceptance(_clj_ordered_blob())) + hb = cert._canonical_hash(cert.project_acceptance(_py_sorted_blob())) + assert ha == hb + + +def test_canonicalization_is_idempotent_on_sorted_blob(): + blob = _py_sorted_blob() + assert cert.project_acceptance(copy.deepcopy(blob)) == cert.project_acceptance( + cert.project_acceptance(copy.deepcopy(blob)) + ) + + +# --------------------------------------------------------------------------- +# PCA component signs are run-arbitrary (Clojure's first-tick power iteration +# has no start vectors — unseeded init flips comps between ITS OWN runs; +# observed on the 2026-07-22 vw single-cut re-record: comps[1] and every +# comp-1-aligned array negated vs the previous recording). Canonicalization +# fixes each component's sign deterministically (max-|entry| positive, after +# tid alignment) and flips every component-aligned array with it. +# --------------------------------------------------------------------------- +def _flip_component(blob: dict, k: int) -> dict: + import copy + + b = copy.deepcopy(blob) + b["pca"]["comps"][k] = [-v for v in b["pca"]["comps"][k]] + b["pca"]["comment-projection"][k] = [ + -v for v in b["pca"]["comment-projection"][k] + ] + coord = ("x", "y")[k] + b["base-clusters"][coord] = [-v for v in b["base-clusters"][coord]] + for g in b["group-clusters"]: + g["center"][k] = -g["center"][k] + return b + + +def test_component_sign_flip_does_not_diverge(): + flipped = _flip_component(_py_sorted_blob(), 1) + assert _diverging_paths(_py_sorted_blob(), flipped) == [] + + +def test_component_sign_flip_of_comp0_does_not_diverge(): + flipped = _flip_component(_py_sorted_blob(), 0) + assert _diverging_paths(_py_sorted_blob(), flipped) == [] + + +def test_sign_flip_combined_with_reordering_does_not_diverge(): + flipped = _flip_component(_clj_ordered_blob(), 1) + assert _diverging_paths(flipped, _py_sorted_blob()) == [] + + +def test_inconsistent_flip_still_reported(): + """Negating base-clusters.y WITHOUT flipping comps[1] is a real + divergence (positions contradict the components) — must survive.""" + b = _py_sorted_blob() + b["base-clusters"]["y"] = [-v for v in b["base-clusters"]["y"]] + paths = _diverging_paths(_py_sorted_blob(), b) + assert any("base-clusters" in p for p in paths) + + +# --------------------------------------------------------------------------- +# Real divergences must SURVIVE canonicalization. +# --------------------------------------------------------------------------- +def test_membership_difference_still_reported(): + b = _py_sorted_blob() + b["in-conv"] = [10, 20, 40] # 30 -> 40: a real membership change + paths = _diverging_paths(_clj_ordered_blob(), b) + assert any("in-conv" in p for p in paths) + + +def test_center_value_difference_for_same_tid_still_reported(): + b = _py_sorted_blob() + b["pca"]["center"][2] = -0.3 # tid 2's mean flips sign: real divergence + paths = _diverging_paths(_clj_ordered_blob(), b) + assert any("pca.center" in p for p in paths) + + +def test_votes_base_count_difference_still_reported(): + b = _py_sorted_blob() + b["votes-base"]["1"]["A"] = [1, 1] # cluster-0 agree count 0 -> 1 + paths = _diverging_paths(_clj_ordered_blob(), b) + assert any("votes-base" in p for p in paths) + + +def test_base_cluster_membership_difference_still_reported(): + b = _py_sorted_blob() + b["base-clusters"]["members"] = [[10], [20, 40]] # 30 -> 40 in cluster 1 + paths = _diverging_paths(_clj_ordered_blob(), b) + assert any("base-clusters" in p for p in paths) + + +# --------------------------------------------------------------------------- +# Shape mismatches (None vs [], list vs int) must stay visible — they are the +# real blob-shape gaps the serializer port addresses, never masked here. +# --------------------------------------------------------------------------- +def test_none_vs_empty_list_still_reported(): + a = _clj_ordered_blob() + a["mod-in"] = None # Clojure emits null pre-moderation; [] must not match + paths = _diverging_paths(a, _py_sorted_blob()) + assert any("mod-in" in p for p in paths) + + +def test_votes_base_list_vs_int_still_reported(): + b = _py_sorted_blob() + b["votes-base"]["0"]["A"] = 3 # py's current scalar shape vs clj's list + paths = _diverging_paths(_clj_ordered_blob(), b) + assert any("votes-base" in p for p in paths)