diff --git a/dashboards/lodestar_beacon_chain.json b/dashboards/lodestar_beacon_chain.json index 04817bfbe768..15da60130f9d 100644 --- a/dashboards/lodestar_beacon_chain.json +++ b/dashboards/lodestar_beacon_chain.json @@ -948,26 +948,13 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_shuffling_cache_recalculated_shuffling_count[32m]) * 384", + "expr": "rate(lodestar_shuffling_cache_set_multiple_times_count[32m]) * 384", "hide": false, "instant": false, - "legendFormat": "built_multiple_times", + "legendFormat": "set_multiple_times", "range": true, "refId": "D" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(lodestar_shuffling_cache_promise_not_resolved_and_thrown_away_count[32m]) * 384", - "hide": false, - "instant": false, - "legendFormat": "not_resolved_thrown_away", - "range": true, - "refId": "E" - }, { "datasource": { "type": "prometheus", @@ -980,19 +967,6 @@ "legendFormat": "not_resolved", "range": true, "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(lodestar_shuffling_cache_next_shuffling_not_on_epoch_cache[32m]) * 384", - "hide": false, - "instant": false, - "legendFormat": "next_shuffling_not_on_cache", - "range": true, - "refId": "G" } ], "title": "Insert vs Hit vs Miss", @@ -1076,19 +1050,6 @@ "legendFormat": "resolution", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(lodestar_shuffling_cache_shuffling_calculation_time_seconds_sum[32m])\n/\nrate(lodestar_shuffling_cache_shuffling_calculation_time_seconds_count[32m])", - "hide": false, - "instant": false, - "legendFormat": "calculation_{{source}}", - "range": true, - "refId": "B" } ], "title": "Timing", diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 2d5e8fbf50b7..d7ddb588ee52 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -418,6 +418,13 @@ export async function importBlock( this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot}); } + // Cache shufflings when crossing an epoch boundary + const parentEpoch = computeEpochAtSlot(parentBlockSlot); + if (parentEpoch < blockEpoch) { + this.shufflingCache.processState(postState); + this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot}); + } + if (blockSlot % SLOTS_PER_EPOCH === 0) { // Cache state to preserve epoch transition work const checkpointState = postState; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 4e27c3354cdd..b7c7779618dc 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -52,6 +52,7 @@ import {computeNodeIdFromPrivateKey} from "../network/subnets/interface.js"; import {BufferPool} from "../util/bufferPool.js"; import {Clock, ClockEvent, IClock} from "../util/clock.js"; import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumns.js"; +import {callInNextEventLoop} from "../util/eventLoop.js"; import {ensureDir, writeIfNotExist} from "../util/file.js"; import {isOptimisticBlock} from "../util/forkChoice.js"; import {SerializedCache} from "../util/serializedCache.js"; @@ -291,7 +292,8 @@ export class BeaconChain implements IBeaconChain { }); this._earliestAvailableSlot = anchorState.slot; - this.shufflingCache = anchorState.epochCtx.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [ + + this.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [ { shuffling: anchorState.epochCtx.previousShuffling, decisionRoot: anchorState.epochCtx.previousDecisionRoot, @@ -417,6 +419,7 @@ export class BeaconChain implements IBeaconChain { clock.addListener(ClockEvent.epoch, this.onClockEpoch.bind(this)); emitter.addListener(ChainEvent.forkChoiceFinalized, this.onForkChoiceFinalized.bind(this)); emitter.addListener(ChainEvent.forkChoiceJustified, this.onForkChoiceJustified.bind(this)); + emitter.addListener(ChainEvent.checkpoint, this.onCheckpoint.bind(this)); } async init(): Promise { @@ -980,8 +983,8 @@ export class BeaconChain implements IBeaconChain { this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller}); state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller); } - - // should always be the current epoch of the active context so no need to await a result from the ShufflingCache + // resolve the promise to unblock other calls of the same epoch and dependent root + this.shufflingCache.processState(state); return state.epochCtx.getShufflingAtEpoch(attEpoch); } @@ -1165,6 +1168,13 @@ export class BeaconChain implements IBeaconChain { this.logger.verbose("Fork choice justified", {epoch: cp.epoch, root: cp.rootHex}); } + private onCheckpoint(this: BeaconChain, _checkpoint: phase0.Checkpoint, state: CachedBeaconStateAllForks): void { + // Defer to not block other checkpoint event handlers, which can cause lightclient update delays + callInNextEventLoop(() => { + this.shufflingCache.processState(state); + }); + } + private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise { this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex}); this.seenBlockProposers.prune(computeStartSlotAtEpoch(cp.epoch)); diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 24bbf399e859..3720feccea8c 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -117,12 +117,7 @@ export class PrepareNextSlotScheduler { // the slot 0 of next epoch will likely use this Previous Root Checkpoint state for state transition so we transfer cache here // the resulting state with cache will be cached in Checkpoint State Cache which is used for the upcoming block processing // for other slots dontTransferCached=true because we don't run state transition on this state - // - // Shuffling calculation will be done asynchronously when passing asyncShufflingCalculation=true. Shuffling will be queued in - // beforeProcessEpoch and should theoretically be ready immediately after the synchronous epoch transition finished and the - // event loop is free. In long periods of non-finality too many forks will cause the shufflingCache to throw an error for - // too many queued shufflings so only run async during normal epoch transition. See issue ChainSafe/lodestar#7244 - {dontTransferCache: !isEpochTransition, asyncShufflingCalculation: true}, + {dontTransferCache: !isEpochTransition}, RegenCaller.precomputeEpoch ); diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index dcb604747f4e..c027565a81a1 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -31,10 +31,6 @@ export enum RegenFnName { export type StateRegenerationOpts = { dontTransferCache: boolean; - /** - * Do not queue shuffling calculation async. Forces sync JIT calculation in afterProcessEpoch if not passed as `true` - */ - asyncShufflingCalculation?: boolean; }; export interface IStateRegenerator extends IStateRegeneratorInternal { diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index bdedddacf6db..19ddb7c1b763 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -1,11 +1,4 @@ -import { - BeaconStateAllForks, - EpochShuffling, - IShufflingCache, - ShufflingBuildProps, - computeEpochShuffling, - computeEpochShufflingAsync, -} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks, EpochShuffling} from "@lodestar/state-transition"; import {Epoch, RootHex} from "@lodestar/types"; import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; @@ -53,7 +46,7 @@ export type ShufflingCacheOpts = { * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice * - skip computing shuffling when loading state bytes from disk */ -export class ShufflingCache implements IShufflingCache { +export class ShufflingCache { /** LRU cache implemented as a map, pruned every time we add an item */ private readonly itemsByDecisionRootByEpoch: MapDef> = new MapDef( () => new Map() @@ -136,60 +129,20 @@ export class ShufflingCache implements IShufflingCache { } /** - * Gets a cached shuffling via the epoch and decision root. If the shuffling is not - * available it will build it synchronously and return the shuffling. - * - * NOTE: If a shuffling is already queued and not calculated it will build and resolve - * the promise but the already queued build will happen at some later time + * Process a state to extract and cache all shufflings (previous, current, next). + * Uses the stored decision roots from epochCtx. */ - getSync( - epoch: Epoch, - decisionRoot: RootHex, - buildProps?: T - ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); - if (!cacheItem) { - this.metrics?.shufflingCache.miss.inc(); - } else if (isShufflingCacheItem(cacheItem)) { - this.metrics?.shufflingCache.hit.inc(); - return cacheItem.shuffling; - } else if (buildProps) { - // TODO: (@matthewkeil) This should possible log a warning?? - this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway.inc(); - } else { - this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc(); - } + processState(state: CachedBeaconStateAllForks): void { + const {epochCtx} = state; - let shuffling: EpochShuffling | null = null; - if (buildProps) { - const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "getSync"}); - shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch); - timer?.(); - this.set(shuffling, decisionRoot); - } - return shuffling as T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; - } + // Cache previous shuffling + this.set(epochCtx.previousShuffling, epochCtx.previousDecisionRoot); - /** - * Queue asynchronous build for an EpochShuffling, triggered from state-transition - */ - build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: Uint32Array): void { - this.insertPromise(epoch, decisionRoot); - /** - * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations - * on a NICE thread - */ - const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "build"}); - computeEpochShufflingAsync(state, activeIndices, epoch) - .then((shuffling) => { - this.set(shuffling, decisionRoot); - }) - .catch((err) => - this.logger?.error(`error building shuffling for epoch ${epoch} at decisionRoot ${decisionRoot}`, {}, err) - ) - .finally(() => { - timer?.(); - }); + // Cache current shuffling + this.set(epochCtx.currentShuffling, epochCtx.currentDecisionRoot); + + // Cache next shuffling + this.set(epochCtx.nextShuffling, epochCtx.nextDecisionRoot); } /** @@ -207,7 +160,8 @@ export class ShufflingCache implements IShufflingCache { (Date.now() - cacheItem.timeInsertedMs) / 1000 ); } else { - this.metrics?.shufflingCache.shufflingBuiltMultipleTimes.inc(); + this.metrics?.shufflingCache.shufflingSetMultipleTimes.inc(); + return; } } // set the shuffling diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index dd3b96093a3c..2def2c231f42 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1308,33 +1308,19 @@ export function createLodestarMetrics( name: "lodestar_shuffling_cache_miss_count", help: "Count of shuffling cache miss", }), - shufflingBuiltMultipleTimes: register.gauge({ - name: "lodestar_shuffling_cache_recalculated_shuffling_count", - help: "Count of shuffling that were build multiple times", - }), - shufflingPromiseNotResolvedAndThrownAway: register.gauge({ - name: "lodestar_shuffling_cache_promise_not_resolved_and_thrown_away_count", - help: "Count of shuffling cache promises that were discarded and the shuffling was built synchronously", + shufflingSetMultipleTimes: register.gauge({ + name: "lodestar_shuffling_cache_set_multiple_times_count", + help: "Count of shuffling that were set multiple times", }), shufflingPromiseNotResolved: register.gauge({ name: "lodestar_shuffling_cache_promise_not_resolved_count", help: "Count of shuffling cache promises that were requested before the promise was resolved", }), - nextShufflingNotOnEpochCache: register.gauge({ - name: "lodestar_shuffling_cache_next_shuffling_not_on_epoch_cache", - help: "The next shuffling was not on the epoch cache before the epoch transition", - }), shufflingPromiseResolutionTime: register.histogram({ name: "lodestar_shuffling_cache_promise_resolution_time_seconds", help: "Time from promise insertion until promise resolution when shuffling was ready in seconds", buckets: [0.5, 1, 1.5, 2], }), - shufflingCalculationTime: register.histogram<{source: "build" | "getSync"}>({ - name: "lodestar_shuffling_cache_shuffling_calculation_time_seconds", - help: "Run time of shuffling calculation", - buckets: [0.5, 0.75, 1, 1.25, 1.5], - labelNames: ["source"], - }), }, seenCache: { diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index f47cd6acba4a..5054d9402754 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -32,7 +32,7 @@ import {getTotalSlashingsByIncrement} from "../epoch/processSlashings.js"; import {AttesterDuty, calculateCommitteeAssignments} from "../util/calculateCommitteeAssignments.js"; import { EpochShuffling, - IShufflingCache, + calculateDecisionRoot, calculateShufflingDecisionRoot, computeEpochShuffling, } from "../util/epochShuffling.js"; @@ -61,7 +61,7 @@ import { computeSyncCommitteeCache, getSyncCommitteeCache, } from "./syncCommitteeCache.js"; -import {BeaconStateAllForks, BeaconStateAltair, BeaconStateGloas} from "./types.js"; +import {BeaconStateAllForks, BeaconStateAltair, BeaconStateGloas, ShufflingGetter} from "./types.js"; /** `= PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)` */ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT); @@ -70,12 +70,12 @@ export type EpochCacheImmutableData = { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; - shufflingCache?: IShufflingCache; }; export type EpochCacheOpts = { skipSyncCommitteeCache?: boolean; skipSyncPubkeys?: boolean; + shufflingGetter?: ShufflingGetter; }; /** Defers computing proposers by persisting only the seed, and dropping it once indexes are computed */ @@ -117,12 +117,6 @@ export class EpochCache { * $VALIDATOR_COUNT x BLST deserialized pubkey (Jacobian coordinates) */ index2pubkey: Index2PubkeyCache; - /** - * ShufflingCache is passed in from `beacon-node` so should be available at runtime but may not be - * present during testing. - */ - shufflingCache?: IShufflingCache; - /** * Indexes of the block proposers for the current epoch. * For pre-fulu, this is computed and cached from the current shuffling. @@ -161,7 +155,7 @@ export class EpochCache { /** Same as previousShuffling */ currentShuffling: EpochShuffling; /** Same as previousShuffling */ - nextShuffling: EpochShuffling | null; + nextShuffling: EpochShuffling; /** * Cache nextActiveIndices so that in afterProcessEpoch the next shuffling can be build synchronously * in case it is not built or the ShufflingCache is not available @@ -254,7 +248,6 @@ export class EpochCache { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; - shufflingCache?: IShufflingCache; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; @@ -263,7 +256,7 @@ export class EpochCache { nextDecisionRoot: RootHex; previousShuffling: EpochShuffling; currentShuffling: EpochShuffling; - nextShuffling: EpochShuffling | null; + nextShuffling: EpochShuffling; nextActiveIndices: Uint32Array; effectiveBalanceIncrements: EffectiveBalanceIncrements; totalSlashingsByIncrement: number; @@ -286,7 +279,6 @@ export class EpochCache { this.config = data.config; this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; - this.shufflingCache = data.shufflingCache; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; @@ -324,7 +316,7 @@ export class EpochCache { */ static createFromState( state: BeaconStateAllForks, - {config, pubkey2index, index2pubkey, shufflingCache}: EpochCacheImmutableData, + {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { const currentEpoch = computeEpochAtSlot(state.slot); @@ -351,14 +343,15 @@ export class EpochCache { const currentActiveIndicesAsNumberArray: ValidatorIndex[] = []; const nextActiveIndicesAsNumberArray: ValidatorIndex[] = []; - // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch + // BeaconChain could provide a shuffling getter to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again + const shufflingGetter = opts?.shufflingGetter; const previousDecisionRoot = calculateShufflingDecisionRoot(config, state, previousEpoch); - const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot); + const cachedPreviousShuffling = shufflingGetter?.(previousEpoch, previousDecisionRoot); const currentDecisionRoot = calculateShufflingDecisionRoot(config, state, currentEpoch); - const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot); + const cachedCurrentShuffling = shufflingGetter?.(currentEpoch, currentDecisionRoot); const nextDecisionRoot = calculateShufflingDecisionRoot(config, state, nextEpoch); - const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot); + const cachedNextShuffling = shufflingGetter?.(nextEpoch, nextDecisionRoot); for (let i = 0; i < validatorCount; i++) { const validator = validators[i]; @@ -366,8 +359,7 @@ export class EpochCache { // Note: Not usable for fork-choice balances since in-active validators are not zero'ed effectiveBalanceIncrements[i] = Math.floor(validator.effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); - // we only need to track active indices for previous, current and next epoch if we have to compute shufflings - // skip doing that if we already have cached shufflings + // Collect active indices for each epoch to compute shufflings if (cachedPreviousShuffling == null && isActiveValidator(validator, previousEpoch)) { previousActiveIndicesAsNumberArray.push(i); } @@ -402,47 +394,19 @@ export class EpochCache { } const nextActiveIndices = new Uint32Array(nextActiveIndicesAsNumberArray); - let previousShuffling: EpochShuffling; - let currentShuffling: EpochShuffling; - let nextShuffling: EpochShuffling; - - if (!shufflingCache) { - // Only for testing. shufflingCache should always be available in prod - previousShuffling = computeEpochShuffling( - state, - new Uint32Array(previousActiveIndicesAsNumberArray), - previousEpoch - ); - currentShuffling = isGenesis - ? previousShuffling - : computeEpochShuffling(state, new Uint32Array(currentActiveIndicesAsNumberArray), currentEpoch); + // Use cached shufflings if available, otherwise compute + const currentShuffling = + cachedCurrentShuffling ?? + computeEpochShuffling(state, new Uint32Array(currentActiveIndicesAsNumberArray), currentEpoch); - nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); - } else { - currentShuffling = cachedCurrentShuffling - ? cachedCurrentShuffling - : shufflingCache.getSync(currentEpoch, currentDecisionRoot, { - state, - activeIndices: new Uint32Array(currentActiveIndicesAsNumberArray), - }); - - previousShuffling = cachedPreviousShuffling - ? cachedPreviousShuffling - : isGenesis - ? currentShuffling - : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { - state, - activeIndices: new Uint32Array(previousActiveIndicesAsNumberArray), - }); - - nextShuffling = cachedNextShuffling - ? cachedNextShuffling - : shufflingCache.getSync(nextEpoch, nextDecisionRoot, { - state, - activeIndices: nextActiveIndices, - }); - } + const previousShuffling = + cachedPreviousShuffling ?? + (isGenesis + ? currentShuffling + : computeEpochShuffling(state, new Uint32Array(previousActiveIndicesAsNumberArray), previousEpoch)); + + const nextShuffling = cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextEpoch); const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); @@ -549,7 +513,6 @@ export class EpochCache { config, pubkey2index, index2pubkey, - shufflingCache, proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, @@ -593,7 +556,6 @@ export class EpochCache { // Common append-only structures shared with all states, no need to clone pubkey2index: this.pubkey2index, index2pubkey: this.index2pubkey, - shufflingCache: this.shufflingCache, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, @@ -652,62 +614,26 @@ export class EpochCache { this.previousShuffling = this.currentShuffling; this.previousDecisionRoot = this.currentDecisionRoot; - // move next to current or calculate upcoming + // move next to current this.currentDecisionRoot = this.nextDecisionRoot; - if (this.nextShuffling) { - // was already pulled from the ShufflingCache to the EpochCache (should be in most cases) - this.currentShuffling = this.nextShuffling; - } else { - this.shufflingCache?.metrics?.shufflingCache.nextShufflingNotOnEpochCache.inc(); - this.currentShuffling = - this.shufflingCache?.getSync(upcomingEpoch, this.currentDecisionRoot, { - state, - // have to use the "nextActiveIndices" that were saved in the last transition here to calculate - // the upcoming shuffling if it is not already built (similar condition to the below computation) - activeIndices: this.nextActiveIndices, - }) ?? - // allow for this case during testing where the ShufflingCache is not present, may affect perf testing - // so should be taken into account when structuring tests. Should not affect unit or other tests though - computeEpochShuffling(state, this.nextActiveIndices, upcomingEpoch); - } + this.currentShuffling = this.nextShuffling; - // handle next values - this.nextDecisionRoot = epochTransitionCache.nextShufflingDecisionRoot; + // Compute shuffling for epoch n+2 + // + // Post-Fulu (EIP-7917), the beacon state includes a `proposer_lookahead` field that stores + // proposer indices for MIN_SEED_LOOKAHEAD + 1 epochs ahead (2 epochs with MIN_SEED_LOOKAHEAD=1). + // At each epoch boundary, processProposerLookahead() shifts out the current epoch's proposers + // and appends new proposers for epoch n + MIN_SEED_LOOKAHEAD + 1 (i.e., epoch n+2). + // + // processProposerLookahead() already computes the n+2 shuffling and stores it in + // epochTransitionCache.nextShuffling. Reuse it here to avoid duplicate computation. + // Pre-Fulu, we need to compute it here since processProposerLookahead doesn't run. + // + // See: https://eips.ethereum.org/EIPS/eip-7917 + this.nextDecisionRoot = calculateDecisionRoot(state, epochAfterUpcoming); this.nextActiveIndices = epochTransitionCache.nextShufflingActiveIndices; - if (this.shufflingCache) { - if (!epochTransitionCache.asyncShufflingCalculation) { - this.nextShuffling = this.shufflingCache.getSync(epochAfterUpcoming, this.nextDecisionRoot, { - state, - activeIndices: this.nextActiveIndices, - }); - } else { - this.nextShuffling = null; - // This promise will resolve immediately after the synchronous code of the state-transition runs. Until - // the build is done on a worker thread it will be calculated immediately after the epoch transition - // completes. Once the work is done concurrently it should be ready by time this get runs so the promise - // will resolve directly on the next spin of the event loop because the epoch transition and shuffling take - // about the same time to calculate so theoretically its ready now. Do not await here though in case it - // is not ready yet as the transition must not be asynchronous. - this.shufflingCache - .get(epochAfterUpcoming, this.nextDecisionRoot) - .then((shuffling) => { - if (!shuffling) { - throw new Error("EpochShuffling not returned from get in afterProcessEpoch"); - } - this.nextShuffling = shuffling; - }) - .catch((err) => { - this.shufflingCache?.logger?.error( - "EPOCH_CONTEXT_SHUFFLING_BUILD_ERROR", - {epoch: epochAfterUpcoming, decisionRoot: epochTransitionCache.nextShufflingDecisionRoot}, - err - ); - }); - } - } else { - // Only for testing. shufflingCache should always be available in prod - this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, epochAfterUpcoming); - } + this.nextShuffling = + epochTransitionCache.nextShuffling ?? computeEpochShuffling(state, this.nextActiveIndices, epochAfterUpcoming); // TODO: DEDUPLICATE from createEpochCache // @@ -1100,10 +1026,6 @@ export class EpochCache { case this.epoch: return this.currentShuffling; case this.nextEpoch: - if (!this.nextShuffling) { - this.nextShuffling = - this.shufflingCache?.getSync(this.nextEpoch, this.getShufflingDecisionRoot(this.nextEpoch)) ?? null; - } return this.nextShuffling; default: return null; diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index a3ec69956162..01d7e94ff153 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,12 +1,6 @@ -import { - EPOCHS_PER_SLASHINGS_VECTOR, - FAR_FUTURE_EPOCH, - ForkSeq, - MIN_ACTIVATION_BALANCE, - SLOTS_PER_HISTORICAL_ROOT, -} from "@lodestar/params"; -import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; -import {intDiv, toRootHex} from "@lodestar/utils"; +import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; +import {Epoch, ValidatorIndex} from "@lodestar/types"; +import {intDiv} from "@lodestar/utils"; import {processPendingAttestations} from "../epoch/processPendingAttestations.js"; import { CachedBeaconStateAllForks, @@ -26,16 +20,13 @@ import { FLAG_UNSLASHED, hasMarkers, } from "../util/attesterStatus.js"; +import {EpochShuffling} from "../util/epochShuffling.js"; export type EpochTransitionCacheOpts = { /** * Assert progressive balances the same to EpochTransitionCache */ assertCorrectProgressiveBalances?: boolean; - /** - * Do not queue shuffling calculation async. Forces sync JIT calculation in afterProcessEpoch - */ - asyncShufflingCalculation?: boolean; }; /** @@ -162,9 +153,10 @@ export interface EpochTransitionCache { nextShufflingActiveIndices: Uint32Array; /** - * Shuffling decision root that gets set on the EpochCache in afterProcessEpoch + * Pre-computed shuffling for epoch N+2, populated by processProposerLookahead (Fulu+). + * Used by afterProcessEpoch to avoid recomputing the same shuffling. */ - nextShufflingDecisionRoot: RootHex; + nextShuffling: EpochShuffling | null; /** * Altair specific, this is total active balances for the next epoch. @@ -179,12 +171,6 @@ export interface EpochTransitionCache { */ nextEpochTotalActiveBalanceByIncrement: number; - /** - * Compute the shuffling sync or async. Defaults to synchronous. Need to pass `true` with the - * `EpochTransitionCacheOpts` - */ - asyncShufflingCalculation: boolean; - /** * Track by validator index if it's active in the prev epoch. * Used in metrics @@ -379,12 +365,7 @@ export function beforeProcessEpoch( } }); - // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) - const epochAfterNext = state.epochCtx.nextEpoch + 1; - // cannot call calculateShufflingDecisionRoot here because spec prevent getting current slot - // as a decision block. we are part way through the transition though and this was added in - // process slot beforeProcessEpoch happens so it available and valid - const nextShufflingDecisionRoot = toRootHex(state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT)); + // Prepare shuffling data for epoch after next (nextShuffling post epoch transition) const nextShufflingActiveIndices = new Uint32Array(nextEpochShufflingActiveIndicesLength); if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { throw new Error( @@ -396,11 +377,6 @@ export function beforeProcessEpoch( nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; } - const asyncShufflingCalculation = opts?.asyncShufflingCalculation ?? false; - if (asyncShufflingCalculation) { - state.epochCtx.shufflingCache?.build(epochAfterNext, nextShufflingDecisionRoot, state, nextShufflingActiveIndices); - } - if (totalActiveStakeByIncrement < 1) { totalActiveStakeByIncrement = 1; } else if (totalActiveStakeByIncrement >= Number.MAX_SAFE_INTEGER) { @@ -524,9 +500,8 @@ export function beforeProcessEpoch( indicesEligibleForActivationQueue, indicesEligibleForActivation: indicesEligibleForActivation.map(({validatorIndex}) => validatorIndex), indicesToEject, - nextShufflingDecisionRoot, nextShufflingActiveIndices, - asyncShufflingCalculation, + nextShuffling: null, // to be updated in processEffectiveBalanceUpdates nextEpochTotalActiveBalanceByIncrement: 0, isActivePrevEpoch, diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index 231a921d966b..f69cd2a44972 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -180,7 +180,7 @@ export function loadCachedBeaconState; - }; -} -export interface IShufflingCache { - metrics: PublicShufflingCacheMetrics | null; - logger: Logger | null; - /** - * Gets a cached shuffling via the epoch and decision root. If the state and - * activeIndices are passed and a shuffling is not available it will be built - * synchronously. If the state is not passed and the shuffling is not available - * nothing will be returned. - * - * NOTE: If a shuffling is already queued and not calculated it will build and resolve - * the promise but the already queued build will happen at some later time - */ - getSync( - epoch: Epoch, - decisionRoot: RootHex, - buildProps?: T - ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; - - /** - * Gets a cached shuffling via the epoch and decision root. Returns a promise - * for the shuffling if it hs not calculated yet. Returns null if a build has - * not been queued nor a shuffling was calculated. - */ - get(epoch: Epoch, decisionRoot: RootHex): Promise; - - /** - * Queue asynchronous build for an EpochShuffling - */ - build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: Uint32Array): void; -} - /** * Readonly interface for EpochShuffling. */ @@ -164,7 +123,7 @@ export async function computeEpochShufflingAsync( }; } -function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch): RootHex { +export function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch): RootHex { const pivotSlot = computeStartSlotAtEpoch(epoch - 1) - 1; return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index b4b2bebc5336..526b02bd30e5 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -57,15 +57,15 @@ describe("loadState", () => { pubkey2index.set(pubkey, validatorIndex); index2pubkey[validatorIndex] = PublicKey.fromBytes(pubkey); } + const shufflingGetter = () => seedState.epochCtx.currentShuffling; createCachedBeaconState( migratedState, { config: seedState.config, pubkey2index, index2pubkey, - shufflingCache: seedState.epochCtx.shufflingCache, }, - {skipSyncPubkeys: true, skipSyncCommitteeCache: true} + {skipSyncPubkeys: true, skipSyncCommitteeCache: true, shufflingGetter} ); }, }); diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 61375bf772a9..bbad6831414c 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -3,9 +3,10 @@ import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {fromHexString} from "@chainsafe/ssz"; import {createBeaconConfig} from "@lodestar/config"; import {config as defaultConfig} from "@lodestar/config/default"; -import {ssz} from "@lodestar/types"; +import {Epoch, RootHex, ssz} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; +import {EpochShuffling, calculateShufflingDecisionRoot} from "../../src/util/epochShuffling.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; import {interopPubkeysCached} from "../utils/interop.js"; import {createCachedBeaconStateTest} from "../utils/state.js"; @@ -159,8 +160,33 @@ describe("CachedBeaconState", () => { // confirm loadState() result const stateBytes = state.serialize(); + const shufflingGetter = (shufflingEpoch: Epoch, dependentRoot: RootHex): EpochShuffling | null => { + if ( + shufflingEpoch === seedState.epochCtx.epoch - 1 && + dependentRoot === calculateShufflingDecisionRoot(config, seedState, shufflingEpoch) + ) { + return seedState.epochCtx.previousShuffling; + } + + if ( + shufflingEpoch === seedState.epochCtx.epoch && + dependentRoot === calculateShufflingDecisionRoot(config, seedState, shufflingEpoch) + ) { + return seedState.epochCtx.currentShuffling; + } + + if ( + shufflingEpoch === seedState.epochCtx.epoch + 1 && + dependentRoot === calculateShufflingDecisionRoot(config, seedState, shufflingEpoch) + ) { + return seedState.epochCtx.nextShuffling; + } + + return null; + }; const newCachedState = loadCachedBeaconState(seedState, stateBytes, { skipSyncCommitteeCache: true, + shufflingGetter, }); const newStateBytes = newCachedState.serialize(); expect(newStateBytes).toEqual(stateBytes); @@ -171,9 +197,8 @@ describe("CachedBeaconState", () => { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], - shufflingCache: seedState.epochCtx.shufflingCache, }, - {skipSyncCommitteeCache: true} + {skipSyncCommitteeCache: true, shufflingGetter} ); // validatorCountDelta < 0 is unrealistic and shuffling computation results in a different result if (validatorCountDelta >= 0) {