diff --git a/src/lean_spec/spec/forks/lstar/spec.py b/src/lean_spec/spec/forks/lstar/spec.py index b36371fb..591e7d33 100644 --- a/src/lean_spec/spec/forks/lstar/spec.py +++ b/src/lean_spec/spec/forks/lstar/spec.py @@ -4,6 +4,8 @@ import math from collections import defaultdict from collections.abc import Iterable, Sequence, Set as AbstractSet +from dataclasses import dataclass +from enum import IntEnum from typing import Any, ClassVar from lean_spec.node.chain.clock import Interval @@ -47,7 +49,7 @@ ValidatorIndex, Validators, ) -from lean_spec.spec.ssz import ZERO_HASH, Boolean, Bytes32, SSZList, Uint8, Uint64 +from lean_spec.spec.ssz import ZERO_HASH, Boolean, Bytes32, SSZList, Uint64 from ..protocol import ForkProtocol, SpecBlockType, SpecStateType @@ -55,6 +57,45 @@ """Concrete Store specialization owned by the lstar fork.""" +class _Tier(IntEnum): + """Selection tier for a candidate attestation data entry. + + Declared in priority order: a lower value wins. + """ + + FINALIZE = 1 + """Applying the entry crosses two-thirds on target and finalizes the source.""" + JUSTIFY = 2 + """Applying the entry crosses two-thirds on target but does not finalize.""" + BUILD = 3 + """Adds marginal new voters toward target's two-thirds supermajority.""" + + +@dataclass(frozen=True) +class _EntryScore: + """Tiered score for a candidate attestation data entry during block building. + + Lower tier wins. + Within a tier, more new voters wins, then a smaller target slot, then a + smaller attestation slot, then the entry's data root for determinism. + """ + + tier: _Tier + new_voters: int + target_slot: Slot + att_slot: Slot + + def ordering_key(self, data_root: Bytes32) -> tuple[int, int, int, int, bytes]: + """Sort key where the smallest tuple is the best candidate.""" + return ( + int(self.tier), + -self.new_voters, + int(self.target_slot), + int(self.att_slot), + bytes(data_root), + ) + + class LstarSpec(ForkProtocol): """Lstar fork.""" @@ -594,6 +635,297 @@ def state_transition( return new_state + @staticmethod + def _build_running_votes(state: State) -> dict[Bytes32, set[ValidatorIndex]]: + """Deserialize the flat justification bitlist into a per-target-root voter map. + + The state layout is bit at index (i * N + j) means validator j voted for + justifications_roots[i], where N is the validator count. + Seeds the running voter set so scoring counts on-chain voters toward the + two-thirds threshold. + """ + num_validators = len(state.validators) + votes: dict[Bytes32, set[ValidatorIndex]] = {} + for i, root in enumerate(state.justifications_roots): + voters = { + ValidatorIndex(j) + for j in range(num_validators) + if state.justifications_validators[i * num_validators + j] + } + votes[root] = voters + return votes + + @staticmethod + def _is_genesis_self_vote(att_data: AttestationData) -> bool: + """Genesis self-votes (source and target at slot 0) are exempt from + target-after-source and target-already-justified filters. + + They cannot justify or finalize, but they carry fork-choice signal. + """ + return att_data.source.slot == Slot(0) and att_data.target.slot == Slot(0) + + @staticmethod + def _score_entry( + att_data: AttestationData, + proofs: AbstractSet[SingleMessageAggregate], + current_votes: dict[Bytes32, set[ValidatorIndex]], + projected_finalized_slot: Slot, + validator_count: int, + ) -> tuple[_EntryScore, set[ValidatorIndex]] | None: + """Score a single candidate entry under the current projected state. + + Returns None if the entry adds zero validators relative to the running + voter set for its target root. + On Some, the returned set is the new voters this entry contributes. + A genesis self-vote cannot justify or finalize and is always BUILD tier. + """ + prior_voters = current_votes.get(att_data.target.root, set()) + + # New voters: participants across all proofs not already recorded for the target. + new_voters: set[ValidatorIndex] = set() + for proof in proofs: + for vid in proof.participants.to_validator_indices(): + if vid not in prior_voters: + new_voters.add(vid) + if not new_voters: + return None + + # Threshold: total voters (prior plus new) crossing two-thirds. + total = len(prior_voters) + len(new_voters) + crosses_two_thirds = 3 * total >= 2 * validator_count + + is_genesis_self_vote = LstarSpec._is_genesis_self_vote(att_data) + + # 3SF-mini finalization requires no slot strictly between source and target + # to still be justifiable, so source and target are consecutive justified + # checkpoints in the projected post-state. + # + # The source must lie strictly past the projected finalized boundary. + # A source at or behind the boundary is already final. + # It may still justify a newer target, but it must not re-finalize. + # This mirrors the state transition, which advances finalization only when + # the source slot is strictly greater than the finalized slot. + # Scanning from one past the source also keeps every queried slot strictly + # above the boundary, where is_justifiable_after is defined. + finalizes = ( + crosses_two_thirds + and att_data.source.slot > projected_finalized_slot + and all( + not Slot(s).is_justifiable_after(projected_finalized_slot) + for s in range(int(att_data.source.slot) + 1, int(att_data.target.slot)) + ) + ) + + if is_genesis_self_vote or not crosses_two_thirds: + tier = _Tier.BUILD + elif finalizes: + tier = _Tier.FINALIZE + else: + tier = _Tier.JUSTIFY + + return ( + _EntryScore( + tier=tier, + new_voters=len(new_voters), + target_slot=att_data.target.slot, + att_slot=att_data.slot, + ), + new_voters, + ) + + @staticmethod + def _entry_passes_filters( + att_data: AttestationData, + known_block_roots: AbstractSet[Bytes32], + extended_historical_block_hashes: Sequence[Bytes32], + projected_justified_slots: JustifiedSlots, + projected_finalized_slot: Slot, + ) -> bool: + """Validate a candidate entry against the projected chain view. + + Mirrors the vote-validity rules: head must be known, source must be + justified, source and target must match the candidate-block chain view, + target must be after source, target must not already be justified, and + target must be justifiable relative to the projected finalized slot. + + The genesis self-vote (source.slot == target.slot == 0) is exempt from the + target-after-source and target-already-justified checks; the state + transition drops it but it carries fork-choice signal. + + Chain-match runs before the justified-slot queries: it rejects checkpoints + whose slot is past the chain view, which keeps the bounded justification + queries from raising IndexError. + """ + if att_data.head.root not in known_block_roots: + return False + if not LstarSpec._attestation_data_matches_chain( + att_data, extended_historical_block_hashes + ): + return False + if not projected_justified_slots.is_slot_justified( + projected_finalized_slot, att_data.source.slot + ): + return False + + # Genesis self-votes are exempt from the remaining checks. + # The state transition drops them, but they carry fork-choice signal. + if not LstarSpec._is_genesis_self_vote(att_data): + if att_data.target.slot <= att_data.source.slot: + return False + if projected_justified_slots.is_slot_justified( + projected_finalized_slot, att_data.target.slot + ): + return False + if not att_data.target.slot.is_justifiable_after(projected_finalized_slot): + return False + return True + + def _pick_best_candidate( + self, + aggregated_payloads: dict[AttestationData, set[SingleMessageAggregate]], + known_block_roots: AbstractSet[Bytes32], + extended_historical_block_hashes: Sequence[Bytes32], + processed: AbstractSet[AttestationData], + projected_justified_slots: JustifiedSlots, + projected_finalized_slot: Slot, + current_votes: dict[Bytes32, set[ValidatorIndex]], + validator_count: int, + ) -> tuple[AttestationData, _EntryScore, set[ValidatorIndex]] | None: + """Scan candidate entries and return the highest-scoring one. + + Skips entries already processed, those failing the projected-chain + filters, and those with zero new voters. + Returns the entry with the best + ordering key (smaller is better), or None when nothing scores. + """ + best: tuple[AttestationData, _EntryScore, set[ValidatorIndex]] | None = None + best_key: tuple[int, int, int, int, bytes] | None = None + + for att_data, proofs in aggregated_payloads.items(): + if att_data in processed: + continue + if not self._entry_passes_filters( + att_data, + known_block_roots, + extended_historical_block_hashes, + projected_justified_slots, + projected_finalized_slot, + ): + continue + scored = self._score_entry( + att_data, + proofs, + current_votes, + projected_finalized_slot, + validator_count, + ) + if scored is None: + continue + score, new_voters = scored + + candidate_key = score.ordering_key(hash_tree_root(att_data)) + if best_key is None or candidate_key < best_key: + best = (att_data, score, new_voters) + best_key = candidate_key + + return best + + def _select_attestations( + self, + head_state: State, + slot: Slot, + parent_root: Bytes32, + known_block_roots: AbstractSet[Bytes32], + aggregated_payloads: dict[AttestationData, set[SingleMessageAggregate]], + ) -> list[tuple[AggregatedAttestation, SingleMessageAggregate]]: + """Tiered greedy attestation selection for block proposal. + + Each round scores remaining candidates against a projected post-state and + picks the best: finalize beats justify beats build. + Justification and finalization are projected incrementally so dependent + attestations become eligible on the next round without re-running the + state transition. + Stops at the data-entry cap or when no remaining candidate scores. + """ + selected: list[tuple[AggregatedAttestation, SingleMessageAggregate]] = [] + if not aggregated_payloads: + return selected + + # Chain view the block header would produce: recorded history up to the + # parent, then the parent root at the parent slot, then zero hashes for any + # skipped slots up to the new block. + # + # Precondition: the new slot must lie strictly after the parent slot. + # Without the guard, Uint64 subtraction underflows and the empty-slot + # padding allocates an astronomically large list. + parent_slot = head_state.latest_block_header.slot + assert slot > parent_slot, f"Cannot build block at slot {slot} <= parent slot {parent_slot}" + num_empty_slots = int(slot - parent_slot - Slot(1)) + extended_historical_block_hashes: list[Bytes32] = ( + list(head_state.historical_block_hashes) + [parent_root] + [ZERO_HASH] * num_empty_slots + ) + validator_count = len(head_state.validators) + + # Projected post-state, updated incrementally as entries are selected. + finalized_slot = head_state.latest_finalized.slot + justified_slots = head_state.justified_slots.extend_to_slot(finalized_slot, slot - Slot(1)) + current_votes = self._build_running_votes(head_state) + processed: set[AttestationData] = set() + + for _round in range(int(MAX_ATTESTATIONS_DATA)): + best = self._pick_best_candidate( + aggregated_payloads, + known_block_roots, + extended_historical_block_hashes, + processed, + justified_slots, + finalized_slot, + current_votes, + validator_count, + ) + if best is None: + break + att_data, score, new_voters = best + processed.add(att_data) + + # Pack proofs that maximize new validator coverage for this entry. + selected_proofs, _ = select_greedily(aggregated_payloads[att_data]) + for proof in selected_proofs: + selected.append( + ( + self.aggregated_attestation_class( + aggregation_bits=proof.participants, + data=att_data, + ), + proof, + ) + ) + + target_root = att_data.target.root + + # Project justification and finalization. Finalize implies justify. + if score.tier <= _Tier.JUSTIFY: + justified_slots = justified_slots.extend_to_slot( + finalized_slot, att_data.target.slot + ) + justified_slots = justified_slots.with_justified( + finalized_slot, att_data.target.slot, Boolean(True) + ) + # A justified target can no longer be a candidate target, so its + # voter bucket is irrelevant for further scoring. + current_votes.pop(target_root, None) + else: + # BUILD tier: the target stays a candidate, so record its new + # voters to push it toward the threshold on a later round. + current_votes.setdefault(target_root, set()).update(new_voters) + if score.tier == _Tier.FINALIZE: + new_finalized = att_data.source.slot + delta = int(new_finalized) - int(finalized_slot) + justified_slots = justified_slots.shift_window(delta) + finalized_slot = new_finalized + + return selected + def build_block( self, state: State, @@ -608,152 +940,33 @@ def build_block( Computes the post-state and creates a block with the correct state root. - Uses a fixed-point algorithm: finds attestation_data entries whose source - matches the current justified checkpoint, greedily selects proofs maximizing - new validator coverage, then applies the STF. If justification advances, - repeats with the new checkpoint. + Uses a tiered greedy scorer: each round scores remaining candidate + attestation data entries against a projected post-state and selects the + best (finalize beats justify beats build). Justification and finalization + are projected incrementally, so the state transition runs only once at the + end to seal the state root. """ aggregated_attestations: list[AggregatedAttestation] = [] aggregated_signatures: list[SingleMessageAggregate] = [] if aggregated_payloads: - # Fixed-point loop: find attestation_data entries matching the current - # justified checkpoint and greedily select proofs. Processing attestations - # may advance justification, unlocking more entries. - # When building on top of genesis (slot 0), process_block_header - # updates the justified root to parent_root. Apply the same - # derivation here so attestation sources match. - if state.latest_block_header.slot == Slot(0): - current_justified = Checkpoint(slot=Slot(0), root=parent_root) - else: - current_justified = state.latest_justified - - # Track the justified-slot bitfield to skip already-justified targets. - # - # Extend the bitfield to cover every slot we might query. - # The range runs from the finalized boundary up to slot - 1 inclusive. - current_finalized_slot = state.latest_finalized.slot - current_justified_slots = state.justified_slots.extend_to_slot( - current_finalized_slot, slot - Slot(1) + selected = self._select_attestations( + state, + slot, + parent_root, + known_block_roots, + aggregated_payloads, ) - - # Build the chain view as it will appear on the candidate block. - # - # The view is the recorded history up to the parent. - # Then comes the parent root at the parent's slot. - # Then zero-hash entries for any skipped slots up to the new block. - # The chain-match helper uses this view to validate source and target roots. - num_empty_slots = int(slot - state.latest_block_header.slot - Slot(1)) - extended_historical_block_hashes: list[Bytes32] = ( - list(state.historical_block_hashes) + [parent_root] + [ZERO_HASH] * num_empty_slots - ) - - processed_attestation_data: set[AttestationData] = set() - - while True: - found_entries = False - - for attestation_data, proofs in sorted( - aggregated_payloads.items(), key=lambda item: item[0].target.slot - ): - if attestation_data in processed_attestation_data: - continue - - if Uint8(len(processed_attestation_data)) >= MAX_ATTESTATIONS_DATA: - break - - if attestation_data.head.root not in known_block_roots: - continue - - # Chain-match runs first. - # - # It rejects checkpoints whose slot is past the chain view. - # That prevents the bounded queries below from indexing out of range. - if not self._attestation_data_matches_chain( - attestation_data, extended_historical_block_hashes - ): - continue - - # The source slot must already be justified on this chain. - if not current_justified_slots.is_slot_justified( - current_finalized_slot, attestation_data.source.slot - ): - continue - - # Genesis-anchored votes have source.slot = target.slot = 0. - # - # They cannot advance justification: the state transition drops them. - # They still carry head-vote weight for fork choice. - # Including them in the body propagates them into peers' payload pool. - # The bypass below keeps them past the target-already-justified check, - # since slot 0 is implicitly justified and would otherwise filter them. - is_genesis_self_vote = attestation_data.source.slot == Slot(0) and ( - attestation_data.target.slot == Slot(0) - ) - - # Skip attestations whose target slot is already justified. - # - # Justification adds nothing for them. - # Entries the state transition will later drop are still kept here. - # They carry head-vote weight for fork choice. - if not is_genesis_self_vote and current_justified_slots.is_slot_justified( - current_finalized_slot, attestation_data.target.slot - ): - continue - - processed_attestation_data.add(attestation_data) - - found_entries = True - - selected, _ = select_greedily(proofs) - aggregated_signatures.extend(selected) - for proof in selected: - aggregated_attestations.append( - self.aggregated_attestation_class( - aggregation_bits=proof.participants, - data=attestation_data, - ) - ) - - if not found_entries: - break - - # Build candidate block and check if justification changed. - candidate_block = self.block_class( - slot=slot, - proposer_index=proposer_index, - parent_root=parent_root, - state_root=Bytes32.zero(), - body=self.block_body_class( - attestations=self.aggregated_attestations_class( - data=list(aggregated_attestations) - ) - ), - ) - post_state = self.process_block(self.process_slots(state, slot), candidate_block) - - # Re-run the filter when justification or finalization advanced. - # - # Both quantities are monotonic in 3SF-mini, so the loop is bounded. - # Finalization advancement shifts the justified window forward. - # That can unlock attestations whose target slot was outside it before. - if ( - post_state.latest_justified != current_justified - or post_state.latest_finalized.slot != current_finalized_slot - ): - current_justified = post_state.latest_justified - current_justified_slots = post_state.justified_slots - current_finalized_slot = post_state.latest_finalized.slot - continue - - break + for att, sig in selected: + aggregated_attestations.append(att) + aggregated_signatures.append(sig) # Compact: merge all proofs sharing the same AttestationData into one # using recursive children aggregation. # - # During the fixed-point loop above, multiple proofs may have been - # selected for the same AttestationData across iterations. Group them - # and merge each group into a single recursive proof. + # During selection above, multiple proofs may have been selected for + # the same AttestationData. Group them and merge each group into a + # single recursive proof. proof_groups: dict[AttestationData, list[SingleMessageAggregate]] = {} for attestation, signature in zip( aggregated_attestations, aggregated_signatures, strict=True @@ -1788,8 +2001,8 @@ def produce_block_with_signatures( 3. Build the block with maximal valid attestations 4. Store the block and update checkpoints - The block builder uses a fixed-point algorithm to collect attestations. - Each iteration may update the justified checkpoint. + The block builder uses a tiered greedy scorer to collect attestations. + Each round projects justification forward, unlocking dependent entries. Returns the per-attestation single-message aggregate proofs unmerged. The validator service signs the block root with the proposal key, wraps that into @@ -1819,9 +2032,9 @@ def produce_block_with_signatures( # Build the block. # - # The builder iteratively collects valid attestations from aggregated - # payloads matching the justified checkpoint. Each iteration may advance - # justification, unlocking more attestation data entries. + # The builder scores valid attestations from aggregated payloads against + # a projected post-state. Each round projects justification forward, + # unlocking more attestation data entries. final_block, final_post_state, _, signatures = self.build_block( head_state, slot=slot, @@ -1834,8 +2047,8 @@ def produce_block_with_signatures( # Invariant: the produced block must close any justified divergence. # # The store may have advanced its justified checkpoint from attestations - # on a minority fork that the head state never processed. The fixed-point - # loop above must incorporate those attestations from the pool, advancing + # on a minority fork that the head state never processed. The selection + # above must incorporate those attestations from the pool, advancing # the block's justified checkpoint to at least match the store. # # Without this, other nodes processing the block would never see the @@ -1845,7 +2058,7 @@ def produce_block_with_signatures( store_justified = store.latest_justified.slot assert block_justified >= store_justified, ( f"Produced block justified={block_justified} < store justified=" - f"{store_justified}. Fixed-point attestation loop did not converge." + f"{store_justified}. Attestation selection did not close the divergence." ) # Compute block hash for storage. diff --git a/tests/consensus/lstar/fc/test_attestation_source_divergence.py b/tests/consensus/lstar/fc/test_attestation_source_divergence.py index cf48382f..9612ea92 100644 --- a/tests/consensus/lstar/fc/test_attestation_source_divergence.py +++ b/tests/consensus/lstar/fc/test_attestation_source_divergence.py @@ -38,11 +38,11 @@ def test_justified_divergence_self_heals_in_next_block( Self-healing ------------ Block 5 is built on the head chain (no explicit attestations). - The builder's fixed-point loop resolves the divergence: + The tiered scorer resolves the divergence: - 1. Pool contains fork B's attestations (source=0, target=1) - 2. Builder starts with current_justified=0 (head state) - 3. Fork B's attestations match (source=0). Included. Justifies slot 1. + 1. Pool contains fork B's attestation (source=0, target=1) + 2. Builder projects from head state justified=0 + 3. Fork B's attestation matches (source=0). Selected. Projects slot 1 justified. 4. Divergence closed in one block. Expected post-state @@ -128,18 +128,17 @@ def test_justified_divergence_self_heals_in_next_block( # # No explicit attestations. The builder reads from the pool. # - # The pool has fork B's attestations (source=0, target=1) - # because on_block added them when processing fork_4. + # The pool has fork B's attestation (source=0, target=1) + # because on_block added it when processing fork_4. # - # Fixed-point loop: - # Pass 1: justified=0 -> fork B's attestations match (source=0) - # 3/4 supermajority -> justifies slot 1 - # justified advances to 1 - # Pass 2: nothing new -> break + # The tiered scorer projects justification forward: + # Round 1: source=0 is justified -> V1+V2+V3 target slot 1 + # 3/4 supermajority -> projects slot 1 justified + # Later rounds: nothing new scores -> stop # # Divergence closed: head state justified = 1 = store justified. # - # The produced block MUST carry the justifying attestations so that + # The produced block MUST carry the justifying attestation so that # other nodes processing it also advance their justified checkpoint. # Block production asserts this invariant. BlockStep( @@ -150,26 +149,21 @@ def test_justified_divergence_self_heals_in_next_block( # Both store and head agree: justified = slot 1. latest_justified_slot=Slot(1), latest_justified_root_label="common", - # The block body must contain fork B's attestations. + # The block body carries the one attestation that advances + # the chain: V1+V2+V3 targeting slot 1, the minority fork's + # justifying vote that closes the divergence. # - # The builder picks up all pool entries whose source matches - # the current justified checkpoint (genesis). Two match: - # - # 1. V1+V2+V3 targeting slot 1 — the minority fork's - # justifying attestation. This is the one that closes - # the divergence. - # 2. V0 targeting slot 2 — originally in block_3's body, - # still in the attestation pool. - block_attestation_count=2, + # V0 targeting slot 2 is also in the pool (originally in + # block_3's body), but every one of its voters is already + # recorded on the head chain for that target. It adds no new + # voters, so the scorer omits it rather than re-stating a + # vote the post-state already holds. + block_attestation_count=1, block_attestations=[ AggregatedAttestationCheck( participants={1, 2, 3}, target_slot=Slot(1), ), - AggregatedAttestationCheck( - participants={0}, - target_slot=Slot(2), - ), ], ), ), diff --git a/tests/consensus/lstar/fc/test_block_production.py b/tests/consensus/lstar/fc/test_block_production.py index 85e30327..3f3391c8 100644 --- a/tests/consensus/lstar/fc/test_block_production.py +++ b/tests/consensus/lstar/fc/test_block_production.py @@ -254,10 +254,23 @@ def test_produce_block_enforces_max_attestations_data_limit( Scenario -------- - Linear chain through MAX_ATTESTATIONS_DATA + 1 blocks. After building - the chain, the same number of aggregated attestations are gossiped — - each targeting a different block — producing one more distinct - AttestationData entry than the limit allows. + Gossip one more distinct AttestationData entry than the limit allows, + each targeting a different justifiable slot. The builder must drop the + surplus entry. + + Why justifiable slots + --------------------- + The builder only includes attestations whose target is justifiable after + the finalized boundary; the state transition skips the rest. Targeting + only justifiable slots ensures every gossiped entry is a real candidate, + so the cap is what bounds the count rather than the filter. + + Why one voter per entry + ----------------------- + A single validator never reaches the two-thirds supermajority, so no + entry justifies its target. The cap is then exercised purely by the + number of distinct entries, with no justification or finalization + dynamics shifting the candidate set mid-selection. Timing ------ @@ -267,24 +280,35 @@ def test_produce_block_enforces_max_attestations_data_limit( Block builder behavior ---------------------- - The builder sorts entries by target.slot and processes them in order. - After selecting MAX_ATTESTATIONS_DATA entries it breaks, excluding the - entries with the highest target slots. The proposer signature occupies - the remaining slot in the multi-message aggregate proof envelope. + Same-tier entries are ordered by target slot, so the builder selects the + MAX_ATTESTATIONS_DATA lowest target slots and drops the highest. The + proposer signature occupies the remaining slot in the multi-message + aggregate proof envelope. Expected post-state ------------------- The produced block contains exactly MAX_ATTESTATIONS_DATA attestations. """ limit = int(MAX_ATTESTATIONS_DATA) - num_target_blocks = limit + 1 - block_production_slot = num_target_blocks + 1 - validators = [ValidatorIndex(0), ValidatorIndex(1), ValidatorIndex(2)] + + # The first limit + 1 slots that are justifiable after genesis. + # One more than the cap, so exactly one candidate must be dropped. + target_slots: list[int] = [] + candidate = 1 + while len(target_slots) < limit + 1: + if Slot(candidate).is_justifiable_after(Slot(0)): + target_slots.append(candidate) + candidate += 1 + + # Build a contiguous chain up to the highest target slot, then produce one + # slot later. Every target block must exist on-chain to be a valid target. + chain_length = target_slots[-1] + block_production_slot = chain_length + 1 # Aggregate fires at interval 2 of the last chain slot. # With an empty pool this is a no-op, so no payloads are lost. # Compute the minimum integer second that reaches this interval. - aggregate_interval = num_target_blocks * int(INTERVALS_PER_SLOT) + 2 + aggregate_interval = chain_length * int(INTERVALS_PER_SLOT) + 2 aggregate_time = math.ceil(aggregate_interval * int(MILLISECONDS_PER_INTERVAL) / 1000) # Next slot start migrates gossip payloads from "new" to "known". next_slot_time = block_production_slot * int(SECONDS_PER_SLOT) @@ -294,25 +318,24 @@ def test_produce_block_enforces_max_attestations_data_limit( chain_steps: list[BlockStep] = [ BlockStep( block=BlockSpec(slot=Slot(n), label=f"block_{n}"), - checks=(StoreChecks(head_slot=Slot(n)) if n == 1 or n == num_target_blocks else None), + checks=(StoreChecks(head_slot=Slot(n)) if n == 1 or n == chain_length else None), ) - for n in range(1, num_target_blocks + 1) + for n in range(1, chain_length + 1) ] - # One gossip attestation per target block. - # Each has a different target checkpoint → num_target_blocks distinct - # AttestationData entries. + # One gossip attestation per target block, each from a single validator. + # Each has a different target checkpoint → limit + 1 distinct entries. # Source auto-resolves to the genesis justified checkpoint. attestation_steps: list[GossipAggregatedAttestationStep] = [ GossipAggregatedAttestationStep( attestation=GossipAggregatedAttestationSpec( - validator_indices=validators, - slot=Slot(num_target_blocks), + validator_indices=[ValidatorIndex(0)], + slot=Slot(chain_length), target_slot=Slot(n), target_root_label=f"block_{n}", ), ) - for n in range(1, num_target_blocks + 1) + for n in target_slots ] fork_choice_test( diff --git a/tests/lean_spec/spec/forks/lstar/state/test_state_aggregation.py b/tests/lean_spec/spec/forks/lstar/state/test_state_aggregation.py index 64e6e9cc..41944aa4 100644 --- a/tests/lean_spec/spec/forks/lstar/state/test_state_aggregation.py +++ b/tests/lean_spec/spec/forks/lstar/state/test_state_aggregation.py @@ -4,6 +4,7 @@ from consensus_testing.keys import XmssKeyManager +from lean_spec.node.chain.config import MAX_ATTESTATIONS_DATA from lean_spec.spec.crypto.merkleization import hash_tree_root from lean_spec.spec.forks import Checkpoint, Slot, ValidatorIndex, ValidatorIndices from lean_spec.spec.forks.lstar import AttestationSignatureEntry @@ -12,9 +13,12 @@ AttestationData, Block, BlockBody, + JustifiedSlots, + SingleMessageAggregate, + State, ) -from lean_spec.spec.forks.lstar.spec import LstarSpec -from lean_spec.spec.ssz import Bytes32 +from lean_spec.spec.forks.lstar.spec import LstarSpec, _Tier +from lean_spec.spec.ssz import Boolean, Bytes32 from tests.lean_spec.helpers import ( make_aggregated_proof, make_attestation_data_simple, @@ -24,6 +28,40 @@ ) +def _build_empty_chain( + spec: LstarSpec, + key_manager: XmssKeyManager, + num_validators: int, + num_blocks: int, +) -> tuple[State, list[Bytes32]]: + """Build genesis -> block_1 -> ... -> block_{num_blocks} with empty bodies. + + Returns the head state and a list of block roots indexed by slot, where + index 0 is the genesis root and index k is the root of the slot-k block. + """ + state = make_keyed_genesis_state(num_validators, key_manager) + roots: list[Bytes32] = [ + hash_tree_root( + state.latest_block_header.model_copy(update={"state_root": hash_tree_root(state)}) + ) + ] + for slot in range(1, num_blocks + 1): + block = Block( + slot=Slot(slot), + proposer_index=ValidatorIndex(slot % num_validators), + parent_root=roots[-1], + state_root=Bytes32.zero(), + body=BlockBody(attestations=AggregatedAttestations(data=[])), + ) + state = spec.process_block(spec.process_slots(state, Slot(slot)), block) + roots.append( + hash_tree_root( + state.latest_block_header.model_copy(update={"state_root": hash_tree_root(state)}) + ) + ) + return state, roots + + def test_aggregated_signatures_prefers_full_gossip_payload( container_key_manager: XmssKeyManager, spec: LstarSpec, @@ -384,6 +422,13 @@ def test_aggregate_with_no_signatures( assert results == [] +def test_build_running_votes_empty_for_fresh_genesis( + container_key_manager: XmssKeyManager, +) -> None: + state = make_keyed_genesis_state(3, container_key_manager) + assert LstarSpec._build_running_votes(state) == {} + + def test_build_block_fixed_point_closes_justified_divergence( container_key_manager: XmssKeyManager, spec: LstarSpec, @@ -490,3 +535,349 @@ def test_build_block_fixed_point_closes_justified_divergence( # Justification must have advanced: the fixed-point loop closed the gap. assert post_state.latest_justified.slot >= Slot(1) assert post_state.latest_justified == target + + +def test_score_entry_genesis_self_vote_is_build_tier( + container_key_manager: XmssKeyManager, +) -> None: + # Genesis self-vote: source.slot == target.slot == 0. + # Even with a supermajority it can never justify or finalize, so it scores BUILD. + genesis = Checkpoint(root=make_bytes32(7), slot=Slot(0)) + att_data = AttestationData(slot=Slot(1), head=genesis, target=genesis, source=genesis) + proof = make_aggregated_proof( + container_key_manager, [ValidatorIndex(0), ValidatorIndex(1)], att_data + ) + + scored = LstarSpec._score_entry( + att_data, + {proof}, + current_votes={}, + projected_finalized_slot=Slot(0), + validator_count=2, + ) + assert scored is not None + score, new_voters = scored + assert score.tier == _Tier.BUILD + assert new_voters == {ValidatorIndex(0), ValidatorIndex(1)} + + +def test_score_entry_returns_none_when_no_new_voters( + container_key_manager: XmssKeyManager, +) -> None: + genesis = Checkpoint(root=make_bytes32(7), slot=Slot(0)) + att_data = AttestationData(slot=Slot(1), head=genesis, target=genesis, source=genesis) + proof = make_aggregated_proof(container_key_manager, [ValidatorIndex(0)], att_data) + + # Validator 0 already recorded for this target root: zero new voters. + scored = LstarSpec._score_entry( + att_data, + {proof}, + current_votes={att_data.target.root: {ValidatorIndex(0)}}, + projected_finalized_slot=Slot(0), + validator_count=2, + ) + assert scored is None + + +def test_score_entry_finalize_tier_when_gap_slots_not_justifiable( + container_key_manager: XmssKeyManager, +) -> None: + # Source slot 6, target slot 9: slots 7 and 8 are not justifiable after + # finalized 0 (distances 7 and 8). + # Source and target are therefore consecutive justified checkpoints, so a + # supermajority entry finalizes its source. + source = Checkpoint(root=make_bytes32(1), slot=Slot(6)) + target = Checkpoint(root=make_bytes32(2), slot=Slot(9)) + head = Checkpoint(root=make_bytes32(3), slot=Slot(0)) + att_data = AttestationData(slot=Slot(9), head=head, target=target, source=source) + proof = make_aggregated_proof( + container_key_manager, [ValidatorIndex(i) for i in range(4)], att_data + ) + + scored = LstarSpec._score_entry( + att_data, + {proof}, + current_votes={}, + projected_finalized_slot=Slot(0), + validator_count=4, + ) + assert scored is not None + score, _ = scored + assert score.tier == _Tier.FINALIZE + + +def test_score_entry_older_source_after_finalization_does_not_raise( + container_key_manager: XmssKeyManager, +) -> None: + # Regression: with the finalized boundary advanced to slot 6, a candidate + # sourced at genesis (slot 0) must be scored without scanning slots at or + # below the finalized boundary. + # is_justifiable_after rejects a slot behind the finalized boundary, so an + # unclamped scan would raise here. + source = Checkpoint(root=make_bytes32(1), slot=Slot(0)) + target = Checkpoint(root=make_bytes32(2), slot=Slot(9)) + head = Checkpoint(root=make_bytes32(3), slot=Slot(0)) + att_data = AttestationData(slot=Slot(9), head=head, target=target, source=source) + proof = make_aggregated_proof( + container_key_manager, [ValidatorIndex(i) for i in range(4)], att_data + ) + + scored = LstarSpec._score_entry( + att_data, + {proof}, + current_votes={}, + projected_finalized_slot=Slot(6), + validator_count=4, + ) + assert scored is not None + score, _ = scored + # Slot 7 is justifiable after finalized 6 (distance 1), so this justifies + # rather than finalizes. + # The regression point is that scoring completes without raising. + assert score.tier == _Tier.JUSTIFY + + +def test_score_entry_older_source_with_short_gap_is_not_finalize( + container_key_manager: XmssKeyManager, +) -> None: + # Regression for the boundary-regression bug. + # Source at genesis (slot 0) is behind a projected finalized boundary at + # slot 5; target at slot 6 leaves an empty gap range (6, 6). + # An unguarded predicate would let all([]) vacuously hold and misclassify + # the entry as FINALIZE, after which _select_attestations would assign + # finalized_slot = 0 and corrupt the projection window. + # Finalization is monotonic, so this candidate must score as JUSTIFY. + source = Checkpoint(root=make_bytes32(1), slot=Slot(0)) + target = Checkpoint(root=make_bytes32(2), slot=Slot(6)) + head = Checkpoint(root=make_bytes32(3), slot=Slot(0)) + att_data = AttestationData(slot=Slot(6), head=head, target=target, source=source) + proof = make_aggregated_proof( + container_key_manager, [ValidatorIndex(i) for i in range(4)], att_data + ) + + scored = LstarSpec._score_entry( + att_data, + {proof}, + current_votes={}, + projected_finalized_slot=Slot(5), + validator_count=4, + ) + assert scored is not None + score, _ = scored + assert score.tier == _Tier.JUSTIFY + + +def test_score_entry_source_at_finalized_boundary_is_not_finalize( + container_key_manager: XmssKeyManager, +) -> None: + # Source sits exactly on the projected finalized boundary at slot 6. + # Target at slot 7 is the next justifiable slot, so the gap range (7, 7) is + # empty and a supermajority would otherwise look like a finalizing entry. + # A source at the boundary is already final, so it justifies the newer target + # but must not re-finalize. + # This mirrors the state transition, which advances finalization only when the + # source slot is strictly greater than the finalized slot. + source = Checkpoint(root=make_bytes32(1), slot=Slot(6)) + target = Checkpoint(root=make_bytes32(2), slot=Slot(7)) + head = Checkpoint(root=make_bytes32(3), slot=Slot(8)) + att_data = AttestationData(slot=Slot(8), head=head, target=target, source=source) + proof = make_aggregated_proof( + container_key_manager, [ValidatorIndex(i) for i in range(4)], att_data + ) + + scored = LstarSpec._score_entry( + att_data, + {proof}, + current_votes={}, + projected_finalized_slot=Slot(6), + validator_count=4, + ) + assert scored is not None + score, _ = scored + assert score.tier == _Tier.JUSTIFY + + +def test_entry_passes_filters_rejects_unknown_head() -> None: + chain = [make_bytes32(10), make_bytes32(11)] # slots 0, 1 + source = Checkpoint(root=chain[0], slot=Slot(0)) + target = Checkpoint(root=chain[1], slot=Slot(1)) + head = Checkpoint(root=make_bytes32(99), slot=Slot(0)) # not in known roots + att_data = AttestationData(slot=Slot(1), head=head, target=target, source=source) + + assert not LstarSpec._entry_passes_filters( + att_data, + known_block_roots=set(), + extended_historical_block_hashes=chain, + projected_justified_slots=JustifiedSlots(data=[]), + projected_finalized_slot=Slot(0), + ) + + +def test_entry_passes_filters_accepts_valid_gap_closer() -> None: + chain = [make_bytes32(10), make_bytes32(11), make_bytes32(12)] # slots 0, 1, 2 + source = Checkpoint(root=chain[0], slot=Slot(0)) # slot 0 is implicitly justified + target = Checkpoint(root=chain[2], slot=Slot(2)) + head = Checkpoint(root=chain[0], slot=Slot(0)) + att_data = AttestationData(slot=Slot(3), head=head, target=target, source=source) + + assert LstarSpec._entry_passes_filters( + att_data, + known_block_roots={chain[0]}, + extended_historical_block_hashes=chain, + projected_justified_slots=JustifiedSlots(data=[Boolean(False), Boolean(False)]), + projected_finalized_slot=Slot(0), + ) + + +def test_build_block_cascades_projected_justification_across_rounds( + container_key_manager: XmssKeyManager, + spec: LstarSpec, +) -> None: + # Round 1 selects A (source slot 0, target slot 1), projecting slot 1 + # justified in-loop. B has source slot 1, which is NOT justified against + # the initial state; the projection admits B in round 2 so the proposer + # packs both attestations without re-running the state transition. + num_validators = 4 + head_state, roots = _build_empty_chain(spec, container_key_manager, num_validators, 2) + parent_root = roots[2] # head is the slot-2 block + + all_validators = [ValidatorIndex(i) for i in range(num_validators)] + att_a = AttestationData( + slot=Slot(3), + head=Checkpoint(root=roots[0], slot=Slot(0)), + target=Checkpoint(root=roots[1], slot=Slot(1)), + source=Checkpoint(root=roots[0], slot=Slot(0)), + ) + att_b = AttestationData( + slot=Slot(3), + head=Checkpoint(root=roots[0], slot=Slot(0)), + target=Checkpoint(root=roots[2], slot=Slot(2)), + source=Checkpoint(root=roots[1], slot=Slot(1)), + ) + proof_a = make_aggregated_proof(container_key_manager, all_validators, att_a) + proof_b = make_aggregated_proof(container_key_manager, all_validators, att_b) + + block, post_state, _atts, _sigs = spec.build_block( + head_state, + slot=Slot(3), + proposer_index=ValidatorIndex(3), + parent_root=parent_root, + known_block_roots={roots[0], roots[1], roots[2]}, + aggregated_payloads={att_a: {proof_a}, att_b: {proof_b}}, + ) + + target_slots = {att.data.target.slot for att in block.body.attestations.data} + assert Slot(1) in target_slots, f"A (target slot 1) missing: {target_slots}" + assert Slot(2) in target_slots, ( + f"B (target slot 2) missing despite cascading projection: {target_slots}" + ) + # Both attestations justify their targets; the final STF lands on slot 2. + assert post_state.latest_justified.slot == Slot(2) + + +def test_build_block_absorbs_older_but_justified_source( + container_key_manager: XmssKeyManager, + spec: LstarSpec, +) -> None: + # Drive the head's latest_justified to slot 1, then feed a pool attestation + # whose source is genesis (slot 0, OLDER than latest_justified). The + # is_slot_justified(source.slot) filter still accepts it (slot 0 is behind + # the finalized boundary), so it is absorbed and justifies slot 2. + num_validators = 4 + supermajority = [ValidatorIndex(0), ValidatorIndex(1), ValidatorIndex(2)] # 3/4 >= ceil(8/3) + head_state, roots = _build_empty_chain(spec, container_key_manager, num_validators, 2) + + # Justify slot 1 on the head chain by processing a slot-3 block whose body + # carries 3/4 votes for the slot-1 block. + just_att = AttestationData( + slot=Slot(3), + head=Checkpoint(root=roots[1], slot=Slot(1)), + target=Checkpoint(root=roots[1], slot=Slot(1)), + source=Checkpoint(root=roots[0], slot=Slot(0)), + ) + just_proof = make_aggregated_proof(container_key_manager, supermajority, just_att) + justifying_block = Block( + slot=Slot(3), + proposer_index=ValidatorIndex(3), + parent_root=roots[2], + state_root=Bytes32.zero(), + body=BlockBody( + attestations=AggregatedAttestations( + data=[ + spec.aggregated_attestation_class( + aggregation_bits=just_proof.participants, data=just_att + ) + ] + ) + ), + ) + head_state = spec.process_block(spec.process_slots(head_state, Slot(3)), justifying_block) + block_3_root = hash_tree_root( + head_state.latest_block_header.model_copy(update={"state_root": hash_tree_root(head_state)}) + ) + assert head_state.latest_justified.slot == Slot(1) + + # Pool attestation: source = genesis (older than justified slot 1), + # target = slot 2. Build a block at slot 4 on the slot-3 head. + gap_att = AttestationData( + slot=Slot(4), + head=Checkpoint(root=roots[0], slot=Slot(0)), + target=Checkpoint(root=roots[2], slot=Slot(2)), + source=Checkpoint(root=roots[0], slot=Slot(0)), + ) + gap_proof = make_aggregated_proof( + container_key_manager, [ValidatorIndex(i) for i in range(num_validators)], gap_att + ) + + block, post_state, _atts, _sigs = spec.build_block( + head_state, + slot=Slot(4), + proposer_index=ValidatorIndex(0), + parent_root=block_3_root, + known_block_roots={roots[0], roots[1], roots[2], block_3_root}, + aggregated_payloads={gap_att: {gap_proof}}, + ) + + targets = {att.data.target for att in block.body.attestations.data} + assert gap_att.target in targets, f"missing gap-closing attestation: {targets}" + assert post_state.latest_justified.slot == Slot(2) + + +def test_build_block_caps_attestation_data_entries( + container_key_manager: XmssKeyManager, + spec: LstarSpec, +) -> None: + # Nine distinct entries each target a different justifiable slot with a single + # voter. With 8 validators the supermajority is 6, so no individual entry + # justifies its target (1/8 < 2/3), and selection stops at MAX_ATTESTATIONS_DATA (8). + # + # Justifiable slots after slot 0: 1, 2, 3, 4, 5, 6, 9, 12, 16 (first nine). + # Build chain to slot 16 so all target roots exist on-chain. + num_validators = 8 + target_slots = [1, 2, 3, 4, 5, 6, 9, 12, 16] # 9 slots, all justifiable after slot 0 + head_state, roots = _build_empty_chain(spec, container_key_manager, num_validators, 16) + parent_root = roots[16] # head is the slot-16 block + + aggregated_payloads: dict[AttestationData, set[SingleMessageAggregate]] = {} + for t in target_slots: + # One voter per entry so no target ever reaches supermajority (1/8 < 2/3). + att_data = AttestationData( + slot=Slot(17), # attestation slot 17, well within max_slot=20 + head=Checkpoint(root=roots[t], slot=Slot(t)), + target=Checkpoint(root=roots[t], slot=Slot(t)), + source=Checkpoint(root=roots[0], slot=Slot(0)), + ) + proof = make_aggregated_proof(container_key_manager, [ValidatorIndex(0)], att_data) + aggregated_payloads[att_data] = {proof} + + block, _post_state, _atts, _sigs = spec.build_block( + head_state, + slot=Slot(17), + proposer_index=ValidatorIndex(1), + parent_root=parent_root, + known_block_roots={roots[s] for s in range(17)}, + aggregated_payloads=aggregated_payloads, + ) + + distinct_data = {att.data for att in block.body.attestations.data} + assert len(distinct_data) == int(MAX_ATTESTATIONS_DATA)