diff --git a/delphi/polismath/conversation/conversation.py b/delphi/polismath/conversation/conversation.py index 9f80df692..0b1f11451 100644 --- a/delphi/polismath/conversation/conversation.py +++ b/delphi/polismath/conversation/conversation.py @@ -31,6 +31,7 @@ _NamedData as _LegacyNamedData, kmeans as legacy_kmeans, ) +from polismath.utils.clj_hash import clojure_hash_map_key_order from polismath.utils.engine_mode import resolve_engine_mode, ENGINE_MODE_LEGACY @@ -2050,17 +2051,24 @@ def _get_in_conv_participants(self) -> Set[Any]: in_conv = (set(self.in_conv) & set(vote_counts.keys())) | threshold_set # Greedy floor (conversation.clj:259-268): if under 15, admit the top - # remaining voters by count descending. Clojure sorts a hash-map with - # `(sort-by (comp - second))`, whose tie order among equal vote counts is - # hash-map iteration order (non-deterministic). We instead break ties by - # matrix ROW ORDER (vote_counts insertion order) via a STABLE sort — a - # deterministic, reproducible surrogate for an inherently underspecified - # Clojure tie case. Below-threshold participants ARE eligible here (the - # floor guarantees clustering has enough rows in tiny/early conversations). + # remaining voters by count descending. Clojure sorts its hash-map with + # `(sort-by (comp - second))` — a STABLE sort — so equal-count ties keep + # the map's ITERATION order, which is deterministic (Murmur3 hashLong + + # HAMT chunk order; validated against three recorded-blob oracles, see + # polismath/utils/clj_hash.py). Candidates are therefore pre-ordered by + # Clojure hash-map order before the stable count sort. Non-int pids fall + # back to matrix row order (clojure_hash_map_key_order passthrough). + # The ≤8-entry array-map regime (insertion order) can't affect the pick: + # a tie only matters with ≥16 participants, which guarantees hash-map. + # Below-threshold participants ARE eligible here (the floor guarantees + # clustering has enough rows in tiny/early conversations). greedy_n = self.IN_CONV_GREEDY_N if len(in_conv) < greedy_n: - candidates = [pid for pid in vote_counts if pid not in in_conv] - candidates.sort(key=lambda pid: -vote_counts[pid]) # stable -> row-order ties + candidates = [ + pid for pid in clojure_hash_map_key_order(vote_counts.keys()) + if pid not in in_conv + ] + candidates.sort(key=lambda pid: -vote_counts[pid]) # stable -> clj-map ties in_conv.update(candidates[:greedy_n - len(in_conv)]) # Persist for the next tick (Clojure returns this as the conv's new diff --git a/delphi/polismath/utils/clj_hash.py b/delphi/polismath/utils/clj_hash.py new file mode 100644 index 000000000..60c013a35 --- /dev/null +++ b/delphi/polismath/utils/clj_hash.py @@ -0,0 +1,109 @@ +"""Clojure PersistentHashMap iteration order for integer keys. + +Some Clojure-parity semantics depend on the ITERATION ORDER of a Clojure +hash-map — e.g. the in-conv greedy floor (conversation.clj:259-268) stable- +sorts the user-vote-counts map by count descending, so equal-count ties keep +the map's own order. That order is deterministic, not arbitrary: + +- Clojure's ``hasheq`` for a Long is ``Murmur3.hashLong`` (clojure.lang.Murmur3): + murmur3-32 finalization over the two 32-bit halves, seed 0, length 8. +- ``PersistentHashMap`` is a HAMT consuming the 32-bit hash in 5-bit chunks, + LOW bits first; each node iterates its entries in ascending chunk value. + Iteration order is therefore a sort by the tuple of successive chunks. + +Validated (2026-07-22) against three recorded-blob oracles — the raw JSON key +order of ``user-vote-counts`` written by Clojure's cheshire (which walks the +map in iteration order): n=18, n=30 and n=98 integer-pid maps, all exact. + +Caveats, deliberate and documented: +- Integer keys only. Other key types hash differently (e.g. String hasheq is + Murmur3 over ``String.hashCode``); :func:`clojure_hash_map_key_order` falls + back to the given order for them rather than guessing. +- Full-hash collisions land in a HashCollisionNode (insertion order). For + distinct realistic pid ranges Murmur3-32 collisions are vanishingly rare; + the sort is stable, so colliding keys keep their given relative order — + matching insertion order when the caller passes keys in insertion order. +- Clojure uses a PersistentArrayMap (insertion order) up to 8 entries. Callers + whose semantics only engage above 8 entries (the greedy floor needs ≥16 + participants before a tie can matter) never see that regime. +""" + +from __future__ import annotations + +from typing import Any, Iterable, List + +_MASK32 = 0xFFFFFFFF +_C1 = 0xCC9E2D51 +_C2 = 0x1B873593 + + +def _rotl32(x: int, r: int) -> int: + return ((x << r) | (x >> (32 - r))) & _MASK32 + + +def _mix_k1(k1: int) -> int: + k1 = (k1 * _C1) & _MASK32 + k1 = _rotl32(k1, 15) + return (k1 * _C2) & _MASK32 + + +def _mix_h1(h1: int, k1: int) -> int: + h1 ^= k1 + h1 = _rotl32(h1, 13) + return (h1 * 5 + 0xE6546B64) & _MASK32 + + +def _fmix(h1: int, length: int) -> int: + h1 ^= length + h1 ^= h1 >> 16 + h1 = (h1 * 0x85EBCA6B) & _MASK32 + h1 ^= h1 >> 13 + h1 = (h1 * 0xC2B2AE35) & _MASK32 + h1 ^= h1 >> 16 + return h1 + + +def clojure_long_hash(value: int) -> int: + """Clojure ``hasheq`` for a Long: ``Murmur3.hashLong`` (32-bit).""" + if value == 0: + return 0 + v = value & 0xFFFFFFFFFFFFFFFF # two's-complement view of the long + low = v & _MASK32 + high = (v >> 32) & _MASK32 + h1 = _mix_h1(0, _mix_k1(low)) + h1 = _mix_h1(h1, _mix_k1(high)) + return _fmix(h1, 8) + + +def _hamt_path(h: int) -> tuple: + # 7 chunks cover all 32 hash bits (5×7 = 35 ≥ 32). + return tuple((h >> shift) & 0x1F for shift in range(0, 35, 5)) + + +def _as_long(k: Any) -> Any: + """Numeric-string keys hash as their Long value: production pids are + strings python-side (poll_votes / run_math_pipeline cast ``str(pid)``) + while Clojure holds the DB's integer pid — parity requires ordering by + the integer's hash. Mirrors the ``int(tid) if tid.isdigit()`` idiom used + for tids in conversation.py. Non-numeric keys pass through unchanged.""" + if isinstance(k, str) and k.lstrip('-').isdigit(): + return int(k) + return k + + +def clojure_hash_map_key_order(keys: Iterable[Any]) -> List[Any]: + """Return ``keys`` in Clojure PersistentHashMap iteration order. + + Keys that are ints — or numeric STRINGS, normalized via :func:`_as_long` + for hashing only (the returned list keeps the original key objects) — + are ordered by their HAMT path (5-bit chunks of hasheq, low first). If + ANY key normalizes to something other than an int (bools excluded — they + are ints in Python but not Longs in Clojure), the given order is + returned unchanged: a wrong deterministic guess would be worse than the + caller's documented fallback order. + """ + key_list = list(keys) + normalized = [_as_long(k) for k in key_list] + if not all(isinstance(k, int) and not isinstance(k, bool) for k in normalized): + return key_list + return sorted(key_list, key=lambda k: _hamt_path(clojure_long_hash(_as_long(k)))) diff --git a/delphi/tests/test_clj_hash_order.py b/delphi/tests/test_clj_hash_order.py new file mode 100644 index 000000000..47c47d42e --- /dev/null +++ b/delphi/tests/test_clj_hash_order.py @@ -0,0 +1,121 @@ +"""Clojure hash-map iteration order (polismath.utils.clj_hash) + its use in +the legacy in-conv greedy tie-break. + +The oracle orders below are pure functions of Clojure's Murmur3/HAMT (no +dataset content): they were validated against the raw JSON key order of +``user-vote-counts`` maps written by Clojure's cheshire in the replay +recordings (journal 2026-07-22 session 3). +""" + +from __future__ import annotations + +import pytest + +from polismath.conversation.conversation import Conversation +from polismath.utils.clj_hash import ( + clojure_hash_map_key_order, + clojure_long_hash, +) +from polismath.utils.engine_mode import ENGINE_MODE_ENV_VAR + + +# Clojure REPL ground truth: (map hash (range 1 6)) and friends — hasheq of +# small Longs (Murmur3.hashLong). +def test_clojure_long_hash_known_values(): + assert clojure_long_hash(0) == 0 + # Verified against the HAMT-order oracle below (an incorrect hashLong + # cannot reproduce the recorded 18-key iteration order). + assert clojure_long_hash(1) != clojure_long_hash(2) + assert all(0 <= clojure_long_hash(v) <= 0xFFFFFFFF for v in range(-5, 40)) + + +def test_hash_map_order_matches_recorded_clojure_oracle(): + """Raw key order of a Clojure-serialized 18-key int map (vw front-loaded6 + step-0 user-vote-counts; same model validated on n=30 and n=98 maps).""" + assert clojure_hash_map_key_order(range(1, 19)) == [ + 7, 1, 4, 15, 13, 6, 17, 3, 12, 2, 11, 9, 5, 14, 16, 10, 18, 8, + ] + + +def test_hash_map_order_is_input_order_invariant(): + keys = [18, 3, 7, 1, 12, 5, 9, 2, 11, 4, 15, 13, 6, 17, 14, 16, 10, 8] + assert clojure_hash_map_key_order(keys) == clojure_hash_map_key_order( + sorted(keys) + ) + + +def test_hash_map_order_non_int_keys_fall_back_to_given_order(): + keys = ["p2", "p1", "p3"] + assert clojure_hash_map_key_order(keys) == keys + mixed = [2, "p1", 1] + assert clojure_hash_map_key_order(mixed) == mixed + + +# --------------------------------------------------------------------------- +# Legacy greedy floor: ties at the boundary follow Clojure hash-map order. +# --------------------------------------------------------------------------- +def _tie_conv(): + """16 participants: pid 1 votes a lot (over threshold), pids 2-13 vote + 3x each, pids 14-17 have exactly ONE vote each — the greedy floor (15) + must admit 13 sure candidates + 2 of the four tied 1-vote pids.""" + votes = [] + for j in range(8): + votes.append({"pid": 1, "tid": j, "vote": 1}) + for pid in range(2, 14): + for j in range(3): + votes.append({"pid": pid, "tid": j, "vote": -1}) + for pid in range(14, 18): + votes.append({"pid": pid, "tid": 0, "vote": 1}) + c = Conversation("greedy_tie") + return c.update_votes({"votes": votes}, recompute=False) + + +def test_legacy_greedy_tie_follows_clojure_hash_order(monkeypatch): + monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "clojure-legacy") + conv = _tie_conv() + in_conv = conv._get_in_conv_participants() + assert len(in_conv) == 15 + admitted_tied = in_conv & {14, 15, 16, 17} + # Clojure hash-map order of the tied pids decides which two get in. + expected = set(clojure_hash_map_key_order([14, 15, 16, 17])[:2]) + assert admitted_tied == expected + # Regression pin for the concrete order (15 before 17 before 14 before 16 + # — from the validated oracle above). + assert expected == {15, 17} + + +def test_hash_map_order_numeric_strings_hash_as_longs(): + """Production pids are STRINGS python-side (poll_votes / run_math_pipeline + cast str(pid)) while Clojure holds Longs — numeric strings must order by + their integer hash, not fall back (review finding on #2650).""" + assert clojure_hash_map_key_order([str(k) for k in range(1, 19)]) == [ + str(k) for k in [7, 1, 4, 15, 13, 6, 17, 3, 12, 2, 11, 9, 5, 14, 16, 10, 18, 8] + ] + + +def test_legacy_greedy_tie_follows_clojure_hash_order_string_pids(monkeypatch): + """Same greedy-floor tie as above but with the PRODUCTION data shape: + string pids (poll_votes casts str(pid); conversation preserves the type). + The tie must still resolve by Clojure hash order of the numeric value.""" + monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "clojure-legacy") + votes = [] + for j in range(8): + votes.append({"pid": "1", "tid": j, "vote": 1}) + for pid in range(2, 14): + for j in range(3): + votes.append({"pid": str(pid), "tid": j, "vote": -1}) + for pid in range(14, 18): + votes.append({"pid": str(pid), "tid": 0, "vote": 1}) + c = Conversation("greedy_tie_str") + conv = c.update_votes({"votes": votes}, recompute=False) + in_conv = conv._get_in_conv_participants() + assert len(in_conv) == 15 + assert in_conv & {"14", "15", "16", "17"} == {"15", "17"} + + +def test_improved_greedy_unaffected(monkeypatch): + monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "improved") + conv = _tie_conv() + in_conv = conv._get_in_conv_participants() + # Improved mode: threshold-only (min(7, n_cmts)=7 votes) — only pid 1. + assert in_conv == {1}