python-math #22: feat(math): Clojure hash-map iteration order for the in-conv greedy tie-break (legacy)

delphi/polismath/conversation/conversation.py

@@ -31,6 +31,7 @@
     _NamedData as _LegacyNamedData,
     kmeans as legacy_kmeans,
 )
+from polismath.utils.clj_hash import clojure_hash_map_key_order
 from polismath.utils.engine_mode import resolve_engine_mode, ENGINE_MODE_LEGACY
 
 
@@ -2050,17 +2051,24 @@ def _get_in_conv_participants(self) -> Set[Any]:
         in_conv = (set(self.in_conv) & set(vote_counts.keys())) | threshold_set
 
         # Greedy floor (conversation.clj:259-268): if under 15, admit the top
-        # remaining voters by count descending. Clojure sorts a hash-map with
-        # `(sort-by (comp - second))`, whose tie order among equal vote counts is
-        # hash-map iteration order (non-deterministic). We instead break ties by
-        # matrix ROW ORDER (vote_counts insertion order) via a STABLE sort — a
-        # deterministic, reproducible surrogate for an inherently underspecified
-        # Clojure tie case. Below-threshold participants ARE eligible here (the
-        # floor guarantees clustering has enough rows in tiny/early conversations).
+        # remaining voters by count descending. Clojure sorts its hash-map with
+        # `(sort-by (comp - second))` — a STABLE sort — so equal-count ties keep
+        # the map's ITERATION order, which is deterministic (Murmur3 hashLong +
+        # HAMT chunk order; validated against three recorded-blob oracles, see
+        # polismath/utils/clj_hash.py). Candidates are therefore pre-ordered by
+        # Clojure hash-map order before the stable count sort. Non-int pids fall
+        # back to matrix row order (clojure_hash_map_key_order passthrough).
+        # The ≤8-entry array-map regime (insertion order) can't affect the pick:
+        # a tie only matters with ≥16 participants, which guarantees hash-map.
+        # Below-threshold participants ARE eligible here (the floor guarantees
+        # clustering has enough rows in tiny/early conversations).
         greedy_n = self.IN_CONV_GREEDY_N
         if len(in_conv) < greedy_n:
-            candidates = [pid for pid in vote_counts if pid not in in_conv]
-            candidates.sort(key=lambda pid: -vote_counts[pid])  # stable -> row-order ties
+            candidates = [
+                pid for pid in clojure_hash_map_key_order(vote_counts.keys())
+                if pid not in in_conv
+            ]
+            candidates.sort(key=lambda pid: -vote_counts[pid])  # stable -> clj-map ties
             in_conv.update(candidates[:greedy_n - len(in_conv)])
 
         # Persist for the next tick (Clojure returns this as the conv's new

delphi/polismath/utils/clj_hash.py

@@ -0,0 +1,109 @@
+"""Clojure PersistentHashMap iteration order for integer keys.
+
+Some Clojure-parity semantics depend on the ITERATION ORDER of a Clojure
+hash-map — e.g. the in-conv greedy floor (conversation.clj:259-268) stable-
+sorts the user-vote-counts map by count descending, so equal-count ties keep
+the map's own order. That order is deterministic, not arbitrary:
+
+- Clojure's ``hasheq`` for a Long is ``Murmur3.hashLong`` (clojure.lang.Murmur3):
+  murmur3-32 finalization over the two 32-bit halves, seed 0, length 8.
+- ``PersistentHashMap`` is a HAMT consuming the 32-bit hash in 5-bit chunks,
+  LOW bits first; each node iterates its entries in ascending chunk value.
+  Iteration order is therefore a sort by the tuple of successive chunks.
+
+Validated (2026-07-22) against three recorded-blob oracles — the raw JSON key
+order of ``user-vote-counts`` written by Clojure's cheshire (which walks the
+map in iteration order): n=18, n=30 and n=98 integer-pid maps, all exact.
+
+Caveats, deliberate and documented:
+- Integer keys only. Other key types hash differently (e.g. String hasheq is
+  Murmur3 over ``String.hashCode``); :func:`clojure_hash_map_key_order` falls
+  back to the given order for them rather than guessing.
+- Full-hash collisions land in a HashCollisionNode (insertion order). For
+  distinct realistic pid ranges Murmur3-32 collisions are vanishingly rare;
+  the sort is stable, so colliding keys keep their given relative order —
+  matching insertion order when the caller passes keys in insertion order.
+- Clojure uses a PersistentArrayMap (insertion order) up to 8 entries. Callers
+  whose semantics only engage above 8 entries (the greedy floor needs ≥16
+  participants before a tie can matter) never see that regime.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, List
+
+_MASK32 = 0xFFFFFFFF
+_C1 = 0xCC9E2D51
+_C2 = 0x1B873593
+
+
+def _rotl32(x: int, r: int) -> int:
+    return ((x << r) | (x >> (32 - r))) & _MASK32
+
+
+def _mix_k1(k1: int) -> int:
+    k1 = (k1 * _C1) & _MASK32
+    k1 = _rotl32(k1, 15)
+    return (k1 * _C2) & _MASK32
+
+
+def _mix_h1(h1: int, k1: int) -> int:
+    h1 ^= k1
+    h1 = _rotl32(h1, 13)
+    return (h1 * 5 + 0xE6546B64) & _MASK32
+
+
+def _fmix(h1: int, length: int) -> int:
+    h1 ^= length
+    h1 ^= h1 >> 16
+    h1 = (h1 * 0x85EBCA6B) & _MASK32
+    h1 ^= h1 >> 13
+    h1 = (h1 * 0xC2B2AE35) & _MASK32
+    h1 ^= h1 >> 16
+    return h1
+
+
+def clojure_long_hash(value: int) -> int:
+    """Clojure ``hasheq`` for a Long: ``Murmur3.hashLong`` (32-bit)."""
+    if value == 0:
+        return 0
+    v = value & 0xFFFFFFFFFFFFFFFF  # two's-complement view of the long
+    low = v & _MASK32
+    high = (v >> 32) & _MASK32
+    h1 = _mix_h1(0, _mix_k1(low))
+    h1 = _mix_h1(h1, _mix_k1(high))
+    return _fmix(h1, 8)
+
+
+def _hamt_path(h: int) -> tuple:
+    # 7 chunks cover all 32 hash bits (5×7 = 35 ≥ 32).
+    return tuple((h >> shift) & 0x1F for shift in range(0, 35, 5))
+
+
+def _as_long(k: Any) -> Any:
+    """Numeric-string keys hash as their Long value: production pids are
+    strings python-side (poll_votes / run_math_pipeline cast ``str(pid)``)
+    while Clojure holds the DB's integer pid — parity requires ordering by
+    the integer's hash. Mirrors the ``int(tid) if tid.isdigit()`` idiom used
+    for tids in conversation.py. Non-numeric keys pass through unchanged."""
+    if isinstance(k, str) and k.lstrip('-').isdigit():
+        return int(k)
+    return k
+
+
+def clojure_hash_map_key_order(keys: Iterable[Any]) -> List[Any]:
+    """Return ``keys`` in Clojure PersistentHashMap iteration order.
+
+    Keys that are ints — or numeric STRINGS, normalized via :func:`_as_long`
+    for hashing only (the returned list keeps the original key objects) —
+    are ordered by their HAMT path (5-bit chunks of hasheq, low first). If
+    ANY key normalizes to something other than an int (bools excluded — they
+    are ints in Python but not Longs in Clojure), the given order is
+    returned unchanged: a wrong deterministic guess would be worse than the
+    caller's documented fallback order.
+    """
+    key_list = list(keys)
+    normalized = [_as_long(k) for k in key_list]
+    if not all(isinstance(k, int) and not isinstance(k, bool) for k in normalized):
+        return key_list
+    return sorted(key_list, key=lambda k: _hamt_path(clojure_long_hash(_as_long(k))))

delphi/tests/test_clj_hash_order.py

@@ -0,0 +1,121 @@
+"""Clojure hash-map iteration order (polismath.utils.clj_hash) + its use in
+the legacy in-conv greedy tie-break.
+
+The oracle orders below are pure functions of Clojure's Murmur3/HAMT (no
+dataset content): they were validated against the raw JSON key order of
+``user-vote-counts`` maps written by Clojure's cheshire in the replay
+recordings (journal 2026-07-22 session 3).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from polismath.conversation.conversation import Conversation
+from polismath.utils.clj_hash import (
+    clojure_hash_map_key_order,
+    clojure_long_hash,
+)
+from polismath.utils.engine_mode import ENGINE_MODE_ENV_VAR
+
+
+# Clojure REPL ground truth: (map hash (range 1 6)) and friends — hasheq of
+# small Longs (Murmur3.hashLong).
+def test_clojure_long_hash_known_values():
+    assert clojure_long_hash(0) == 0
+    # Verified against the HAMT-order oracle below (an incorrect hashLong
+    # cannot reproduce the recorded 18-key iteration order).
+    assert clojure_long_hash(1) != clojure_long_hash(2)
+    assert all(0 <= clojure_long_hash(v) <= 0xFFFFFFFF for v in range(-5, 40))
+
+
+def test_hash_map_order_matches_recorded_clojure_oracle():
+    """Raw key order of a Clojure-serialized 18-key int map (vw front-loaded6
+    step-0 user-vote-counts; same model validated on n=30 and n=98 maps)."""
+    assert clojure_hash_map_key_order(range(1, 19)) == [
+        7, 1, 4, 15, 13, 6, 17, 3, 12, 2, 11, 9, 5, 14, 16, 10, 18, 8,
+    ]
+
+
+def test_hash_map_order_is_input_order_invariant():
+    keys = [18, 3, 7, 1, 12, 5, 9, 2, 11, 4, 15, 13, 6, 17, 14, 16, 10, 8]
+    assert clojure_hash_map_key_order(keys) == clojure_hash_map_key_order(
+        sorted(keys)
+    )
+
+
+def test_hash_map_order_non_int_keys_fall_back_to_given_order():
+    keys = ["p2", "p1", "p3"]
+    assert clojure_hash_map_key_order(keys) == keys
+    mixed = [2, "p1", 1]
+    assert clojure_hash_map_key_order(mixed) == mixed
+
+
+# ---------------------------------------------------------------------------
+# Legacy greedy floor: ties at the boundary follow Clojure hash-map order.
+# ---------------------------------------------------------------------------
+def _tie_conv():
+    """16 participants: pid 1 votes a lot (over threshold), pids 2-13 vote
+    3x each, pids 14-17 have exactly ONE vote each — the greedy floor (15)
+    must admit 13 sure candidates + 2 of the four tied 1-vote pids."""
+    votes = []
+    for j in range(8):
+        votes.append({"pid": 1, "tid": j, "vote": 1})
+    for pid in range(2, 14):
+        for j in range(3):
+            votes.append({"pid": pid, "tid": j, "vote": -1})
+    for pid in range(14, 18):
+        votes.append({"pid": pid, "tid": 0, "vote": 1})
+    c = Conversation("greedy_tie")
+    return c.update_votes({"votes": votes}, recompute=False)
+
+
+def test_legacy_greedy_tie_follows_clojure_hash_order(monkeypatch):
+    monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "clojure-legacy")
+    conv = _tie_conv()
+    in_conv = conv._get_in_conv_participants()
+    assert len(in_conv) == 15
+    admitted_tied = in_conv & {14, 15, 16, 17}
+    # Clojure hash-map order of the tied pids decides which two get in.
+    expected = set(clojure_hash_map_key_order([14, 15, 16, 17])[:2])
+    assert admitted_tied == expected
+    # Regression pin for the concrete order (15 before 17 before 14 before 16
+    # — from the validated oracle above).
+    assert expected == {15, 17}
+
+
+def test_hash_map_order_numeric_strings_hash_as_longs():
+    """Production pids are STRINGS python-side (poll_votes / run_math_pipeline
+    cast str(pid)) while Clojure holds Longs — numeric strings must order by
+    their integer hash, not fall back (review finding on #2650)."""
+    assert clojure_hash_map_key_order([str(k) for k in range(1, 19)]) == [
+        str(k) for k in [7, 1, 4, 15, 13, 6, 17, 3, 12, 2, 11, 9, 5, 14, 16, 10, 18, 8]
+    ]
+
+
+def test_legacy_greedy_tie_follows_clojure_hash_order_string_pids(monkeypatch):
+    """Same greedy-floor tie as above but with the PRODUCTION data shape:
+    string pids (poll_votes casts str(pid); conversation preserves the type).
+    The tie must still resolve by Clojure hash order of the numeric value."""
+    monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "clojure-legacy")
+    votes = []
+    for j in range(8):
+        votes.append({"pid": "1", "tid": j, "vote": 1})
+    for pid in range(2, 14):
+        for j in range(3):
+            votes.append({"pid": str(pid), "tid": j, "vote": -1})
+    for pid in range(14, 18):
+        votes.append({"pid": str(pid), "tid": 0, "vote": 1})
+    c = Conversation("greedy_tie_str")
+    conv = c.update_votes({"votes": votes}, recompute=False)
+    in_conv = conv._get_in_conv_participants()
+    assert len(in_conv) == 15
+    assert in_conv & {"14", "15", "16", "17"} == {"15", "17"}
+
+
+def test_improved_greedy_unaffected(monkeypatch):
+    monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "improved")
+    conv = _tie_conv()
+    in_conv = conv._get_in_conv_participants()
+    # Improved mode: threshold-only (min(7, n_cmts)=7 votes) — only pid 1.
+    assert in_conv == {1}