Dev to main

README.md

@@ -178,6 +178,7 @@ So, according to the algorithm, there are the following statements:
 | TX_STRIKES_PAYLOAD_MAX_BATCH_SIZE       | Max number of strikes packed into a single transaction                  | no                     | 10                            |
 | STRIKES_MAX_REQUEST_FEE_GWEI            | Abort bad-performance proof if EIP-7002 withdrawal request fee ≥ this    | no                     | 6000                          |
 | WORKER_TIMEOUT_MS                       | Timeout before a proof worker is forcibly terminated                    | no                     | 1_800_000 (30 min)            |
+| WORKER_MAX_OLD_GENERATION_SIZE_MB       | Max V8 heap of a worker in MB; up to 2 workers run at once, so keep 2 × value + ~1.7 GB within the container memory. A worker over it fails with ERR_WORKER_OUT_OF_MEMORY | no                     | 3072                          |
 | BALANCE_PROOF_MIN_DELTA_GWEI            | Minimum balance increase above confirmed level to trigger a proof       | no                     | 512_000_000_000 (512 ETH)     |
 | BALANCE_PROOF_TOPUP_STEP_GWEI           | Report a balance change this far below MaxEB (top-up step); 0 = at MaxEB | no                     | 2_000_000_000 (2 ETH)         |
 | KEYS_INDEXER_RUNNING_PERIOD_MS          | How often the keys indexer re-syncs with KeysAPI                        | no                     | 10_800_000 (3 hours)          |

src/common/config/env.validation.ts

@@ -146,6 +146,11 @@ export class EnvironmentVariables {
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
   public WORKER_TIMEOUT_MS = 30 * MINUTE_MS;
 
+  @IsNumber()
+  @Min(1)
+  @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
+  public WORKER_MAX_OLD_GENERATION_SIZE_MB = 3072;
+
   @IsNumber()
   @Transform(({ value }) => parseInt(value, 10), { toClassOnly: true })
   public TX_CONFIRMATIONS = 1;

src/common/helpers/proofs.ts

@@ -7,6 +7,7 @@ import type { RootHex } from '@lodestar/types';
 import type { BeaconBlockHeaderStruct, ValidatorStruct, WithdrawalStruct } from '../contracts/types/Verifier.js';
 import { epochToBigInt } from '../providers/consensus/epoch.js';
 import type {
+  SupportedBlockRootsView,
   SupportedBlockView,
   SupportedStateView,
   SupportedValidatorView,
@@ -46,7 +47,7 @@ export async function generateBalanceProof(
 
 export async function generateHistoricalStateProof(
   finalizedStateView: SupportedStateView,
-  summaryStateView: SupportedStateView,
+  summaryBlockRoots: SupportedBlockRootsView,
   summaryIndex: number,
   rootIndex: number,
 ): Promise<SingleProof> {
@@ -57,8 +58,8 @@ export async function generateHistoricalStateProof(
     summaryIndex,
     'blockSummaryRoot',
   ]).gindex;
-  patchedTree.setNode(blockSummaryRootGI, summaryStateView.blockRoots.node);
-  const blockRootsGI = summaryStateView.blockRoots.type.getPropertyGindex(rootIndex) as bigint;
+  patchedTree.setNode(blockSummaryRootGI, summaryBlockRoots.node);
+  const blockRootsGI = summaryBlockRoots.type.getPropertyGindex(rootIndex) as bigint;
   const gI = concatGindices([blockSummaryRootGI, blockRootsGI]);
   return createProof(patchedTree.rootNode, {
     type: ProofType.single,

src/common/prover/duties/balances.service.ts

@@ -65,8 +65,10 @@ export class BalancesService {
     const keysCount = Object.keys(keys).length;
     if (keysCount === 0) return {};
 
-    const currentBalances = await this.getValidatorBalances(currentState);
-    const { slot, valExitEpochs, valWithdrawableEpochs } = await this.getValidatorEpochs(currentState);
+    const { slot, validators: validatorsInfo } = await this.workers.getValidatorsInfo({
+      state: currentState,
+      indices: Object.keys(keys).map(Number),
+    });
     const stateEpoch = BigInt(this.consensus.slotToEpoch(slot));
     const minActivationBalanceGwei = BigInt(this.consensus.beaconConfig.MIN_ACTIVATION_BALANCE);
     const provable: InvolvedKeys = {};
@@ -75,19 +77,19 @@ export class BalancesService {
     // processing; they retry next block. A balance <= MIN_ACTIVATION_BALANCE can never be provable (confirmed
     // balance is always >= MIN_ACTIVATION_BALANCE), so drop those before the on-chain confirmed-balance reads.
     const entries = Object.entries(keys).filter(([valIndex]) => {
-      const balance = currentBalances[Number(valIndex)];
-      return balance !== undefined && balance > minActivationBalanceGwei;
+      const info = validatorsInfo[Number(valIndex)];
+      return info !== undefined && info.balance > minActivationBalanceGwei;
     });
     const addedBalances = await this.stakingModule.getKeyAddedBalances(entries.map(([, keyInfo]) => keyInfo));
 
     entries.forEach(([valIndex, keyInfo], i) => {
-      const index = Number(valIndex);
+      const { balance, exitEpoch, withdrawableEpoch } = validatorsInfo[Number(valIndex)];
       const provableBalance = this.isProvableBalance(
         valIndex,
         addedBalances[i],
-        currentBalances[index],
-        valExitEpochs[index],
-        valWithdrawableEpochs[index],
+        balance,
+        exitEpoch,
+        withdrawableEpoch,
         stateEpoch,
       );
       if (provableBalance) {
@@ -136,14 +138,6 @@ export class BalancesService {
     return payloads.length;
   }
 
-  private async getValidatorBalances(state: State): Promise<bigint[]> {
-    return await this.workers.getValidatorBalances({ state });
-  }
-
-  private async getValidatorEpochs(state: State) {
-    return await this.workers.getValidatorExitEpochs({ state });
-  }
-
   private async sendGeneralBalanceProofs(
     blockHeader: BlockHeaderResponse,
     state: State,

src/common/providers/consensus/consensus.ts

@@ -56,10 +56,8 @@ export interface State {
 
 @Injectable()
 export class Consensus extends BaseRestProvider implements OnModuleInit {
-  // Max distinct states commonly needed in a single root-processing cycle:
-  // epoch current state, epoch previous state, parent state for pre-event proofs,
-  // finalized/recent state for historical proofs, and summary state.
-  private readonly stateCache = new LruCache<StateId, State>(5, { shouldCache: isCacheableConsensusId });
+  // Last state only: the next consumer often reuses it; more won't fit the memory budget.
+  private readonly stateCache = new LruCache<StateId, State>(1, { shouldCache: isCacheableConsensusId });
   private readonly blockInfoCache = new LruCache<BlockId, SupportedBlock>(16, { shouldCache: isCacheableConsensusId });
   private readonly beaconHeaderCache = new LruCache<BlockId, BlockHeaderResponse>(16, {
     shouldCache: isCacheableConsensusId,
@@ -210,8 +208,11 @@ export class Consensus extends BaseRestProvider implements OnModuleInit {
         await body.dump().catch(() => {});
         throw e;
       }
-      const bodyBytes = await body.bytes();
-      this.assertSerializedState(bodyBytes, forkName, stateId);
+      const downloaded = await body.bytes();
+      this.assertSerializedState(downloaded, forkName, stateId);
+      // Shared with workers by reference instead of a structured-clone copy per worker.
+      const bodyBytes = new Uint8Array(new SharedArrayBuffer(downloaded.byteLength));
+      bodyBytes.set(downloaded);
       return { bodyBytes, forkName };
     });
     if (this.progress) {

src/common/providers/consensus/forks.ts

@@ -1,16 +1,20 @@
 import { ForkName } from '@lodestar/params';
-import { sszTypesFor } from '@lodestar/types';
+import { ssz, sszTypesFor } from '@lodestar/types';
+
+import type { State } from './consensus.js';
 
 const SUPPORTED_FORKS = [ForkName.capella, ForkName.deneb, ForkName.electra, ForkName.fulu] as const;
 
 export type SupportedForkKey = (typeof SUPPORTED_FORKS)[number];
 type ForkSsz = ReturnType<typeof sszTypesFor<SupportedForkKey>>;
+type StateFields = (typeof ssz)[SupportedForkKey]['BeaconState']['fields'];
 
 export type SupportedBlock = ReturnType<ForkSsz['BeaconBlock']['fromJson']>;
 export type SupportedBlockView = ReturnType<ForkSsz['BeaconBlock']['toView']>;
 export type SupportedStateView = ReturnType<ForkSsz['BeaconState']['toView']>;
 export type SupportedWithdrawal = ReturnType<ForkSsz['Withdrawal']['fromJson']>;
 export type SupportedValidatorView = ReturnType<ForkSsz['Validator']['toView']>;
+export type SupportedBlockRootsView = SupportedStateView['blockRoots'];
 
 export function parseFork(forkName: string): SupportedForkKey {
   if (!(SUPPORTED_FORKS as readonly string[]).includes(forkName)) {
@@ -22,3 +26,13 @@ export function parseFork(forkName: string): SupportedForkKey {
 export function getSsz(forkName: SupportedForkKey): ForkSsz {
   return sszTypesFor(forkName);
 }
+
+export function getStateFieldsBytes(state: State): Record<keyof StateFields, Uint8Array> {
+  const { bodyBytes, forkName } = state;
+  const stateType = ssz[forkName].BeaconState;
+  const view = new DataView(bodyBytes.buffer, bodyBytes.byteOffset, bodyBytes.byteLength);
+  const ranges = stateType.getFieldRanges(view, 0, bodyBytes.byteLength);
+  return Object.fromEntries(
+    stateType.fieldsEntries.map(({ fieldName }, i) => [fieldName, bodyBytes.subarray(ranges[i].start, ranges[i].end)]),
+  ) as Record<keyof StateFields, Uint8Array>;
+}

src/common/workers/historical-state-proof.ts

@@ -0,0 +1,31 @@
+import type { SingleProof } from '@chainsafe/persistent-merkle-tree';
+
+import { WorkerLogger } from './worker-logger.js';
+import { generateHistoricalStateProof, verifyProof } from '../helpers/proofs.js';
+import type { State } from '../providers/consensus/consensus.js';
+import { type SupportedBlockRootsView, getSsz } from '../providers/consensus/forks.js';
+
+// The full state view lives only in this frame so it can be collected before the target state is deserialized.
+export async function buildHistoricalStateProof(
+  finalizedState: State,
+  summaryBlockRoots: SupportedBlockRootsView,
+  summaryIndex: number,
+  rootIndexInSummary: number,
+): Promise<SingleProof> {
+  const finalizedStateView = getSsz(finalizedState.forkName).BeaconState.deserializeToView(finalizedState.bodyBytes);
+  WorkerLogger.log('Generating historical state proof');
+  const historicalStateProof = await generateHistoricalStateProof(
+    finalizedStateView,
+    summaryBlockRoots,
+    summaryIndex,
+    rootIndexInSummary,
+  );
+  WorkerLogger.log('Verifying historical state proof locally');
+  verifyProof(
+    finalizedStateView.hashTreeRoot(),
+    historicalStateProof.gindex,
+    historicalStateProof.witnesses,
+    summaryBlockRoots.getReadonly(rootIndexInSummary),
+  );
+  return historicalStateProof;
+}

src/common/workers/items/build-historical-balance-proof-payloads.ts

@@ -1,9 +1,10 @@
 import { parentPort, workerData } from 'node:worker_threads';
 
+import { ssz } from '@lodestar/types';
+
 import type { IVerifier } from '../../contracts/types/Verifier.js';
 import {
   generateBalanceProof,
-  generateHistoricalStateProof,
   generateValidatorProof,
   toBeaconHeaderStruct,
   toHex,
@@ -12,8 +13,9 @@ import {
 } from '../../helpers/proofs.js';
 import type { InvolvedKeys } from '../../prover/duties/balances.service.js';
 import type { State } from '../../providers/consensus/consensus.js';
-import { getSsz } from '../../providers/consensus/forks.js';
+import { getSsz, getStateFieldsBytes } from '../../providers/consensus/forks.js';
 import type { BlockHeaderResponse } from '../../providers/consensus/response.interface.js';
+import { buildHistoricalStateProof } from '../historical-state-proof.js';
 import { WorkerLogger } from '../worker-logger.js';
 
 export type BuildHistoricalBalanceProofArgs = {
@@ -41,25 +43,17 @@ async function buildHistoricalBalanceProofPayloads(): Promise<IVerifier.ProcessH
     keys,
   } = workerData as BuildHistoricalBalanceProofArgs;
 
-  const stateWithBalancesView = getSsz(stateWithBalances.forkName).BeaconState.deserializeToView(
-    stateWithBalances.bodyBytes,
+  const summaryBlockRoots = ssz[summaryState.forkName].BeaconState.fields.blockRoots.deserializeToView(
+    getStateFieldsBytes(summaryState).blockRoots,
   );
-  const recentStateView = getSsz(recentState.forkName).BeaconState.deserializeToView(recentState.bodyBytes);
-  const summaryStateView = getSsz(summaryState.forkName).BeaconState.deserializeToView(summaryState.bodyBytes);
-
-  WorkerLogger.log('Generating historical state proof');
-  const historicalStateProof = await generateHistoricalStateProof(
-    recentStateView,
-    summaryStateView,
+  const historicalStateProof = await buildHistoricalStateProof(
+    recentState,
+    summaryBlockRoots,
     summaryIndex,
     rootIndexInSummary,
   );
-  WorkerLogger.log('Verifying historical state proof locally');
-  verifyProof(
-    recentStateView.hashTreeRoot(),
-    historicalStateProof.gindex,
-    historicalStateProof.witnesses,
-    summaryStateView.blockRoots.getReadonly(rootIndexInSummary),
+  const stateWithBalancesView = getSsz(stateWithBalances.forkName).BeaconState.deserializeToView(
+    stateWithBalances.bodyBytes,
   );
 
   const payloads: IVerifier.ProcessHistoricalBalanceProofInputStruct[] = [];

src/common/workers/items/build-historical-wd-proof-payloads.ts

@@ -1,8 +1,9 @@
 import { parentPort, workerData } from 'node:worker_threads';
 
+import { ssz } from '@lodestar/types';
+
 import type { IVerifier } from '../../contracts/types/Verifier.js';
 import {
-  generateHistoricalStateProof,
   generateValidatorProof,
   generateWithdrawalProof,
   toBeaconHeaderStruct,
@@ -13,8 +14,9 @@ import {
 } from '../../helpers/proofs.js';
 import type { InvolvedKeysWithWithdrawal } from '../../prover/duties/withdrawals.service.js';
 import type { State } from '../../providers/consensus/consensus.js';
-import { type SupportedBlock, getSsz } from '../../providers/consensus/forks.js';
+import { type SupportedBlock, getSsz, getStateFieldsBytes } from '../../providers/consensus/forks.js';
 import type { BlockHeaderResponse } from '../../providers/consensus/response.interface.js';
+import { buildHistoricalStateProof } from '../historical-state-proof.js';
 import { WorkerLogger } from '../worker-logger.js';
 
 export type BuildHistoricalWithdrawalProofArgs = {
@@ -48,8 +50,15 @@ async function buildHistoricalWithdrawalsProofPayloads(): Promise<IVerifier.Proc
   //
   // Get views
   //
-  const finalizedStateView = getSsz(finalizedState.forkName).BeaconState.deserializeToView(finalizedState.bodyBytes);
-  const summaryStateView = getSsz(summaryState.forkName).BeaconState.deserializeToView(summaryState.bodyBytes);
+  const summaryBlockRoots = ssz[summaryState.forkName].BeaconState.fields.blockRoots.deserializeToView(
+    getStateFieldsBytes(summaryState).blockRoots,
+  );
+  const historicalStateProof = await buildHistoricalStateProof(
+    finalizedState,
+    summaryBlockRoots,
+    summaryIndex,
+    rootIndexInSummary,
+  );
   const stateWithWdsView = getSsz(stateWithWds.forkName).BeaconState.deserializeToView(stateWithWds.bodyBytes);
   const blockWithWdsView = getSsz(stateWithWds.forkName).BeaconBlock.toView(blockWithWds);
   //
@@ -84,13 +93,6 @@ async function buildHistoricalWithdrawalsProofPayloads(): Promise<IVerifier.Proc
       blockWithWdsView,
       keyWithWithdrawalInfo.withdrawal.offset,
     );
-    WorkerLogger.log('Generating historical state proof');
-    const historicalStateProof = await generateHistoricalStateProof(
-      finalizedStateView,
-      summaryStateView,
-      summaryIndex,
-      rootIndexInSummary,
-    );
     WorkerLogger.log('Verifying validator proof locally');
     verifyProof(
       stateWithWdsView.hashTreeRoot(),
@@ -107,13 +109,6 @@ async function buildHistoricalWithdrawalsProofPayloads(): Promise<IVerifier.Proc
         .getReadonly(keyWithWithdrawalInfo.withdrawal.offset)
         .hashTreeRoot(),
     );
-    WorkerLogger.log('Verifying historical state proof locally');
-    verifyProof(
-      finalizedStateView.hashTreeRoot(),
-      historicalStateProof.gindex,
-      historicalStateProof.witnesses,
-      summaryStateView.blockRoots.getReadonly(rootIndexInSummary),
-    );
     payloads.push({
       withdrawal: {
         offset: Number(keyWithWithdrawalInfo.withdrawal.offset),

src/common/workers/items/get-new-validator-keys.ts

@@ -1,10 +1,11 @@
 import { parentPort, workerData } from 'node:worker_threads';
 
 import { iterateNodesAtDepth } from '@chainsafe/persistent-merkle-tree';
+import { ssz } from '@lodestar/types';
 
 import { toHex } from '../../helpers/proofs.js';
 import type { State } from '../../providers/consensus/consensus.js';
-import { getSsz } from '../../providers/consensus/forks.js';
+import { getStateFieldsBytes } from '../../providers/consensus/forks.js';
 import { WorkerLogger } from '../worker-logger.js';
 
 export type GetNewValidatorKeysArgs = {
@@ -22,25 +23,26 @@ async function getNewValidatorKeys(): Promise<GetNewValidatorKeysResult> {
   //
   // Get views
   //
-  const stateView = getSsz(state.forkName).BeaconState.deserializeToView(state.bodyBytes);
+  const { fields } = ssz[state.forkName].BeaconState;
+  const validatorsView = fields.validators.deserializeToView(getStateFieldsBytes(state).validators);
   //
   //
   //
-  const totalValLength = stateView.validators.length;
+  const totalValLength = validatorsView.length;
   const appearedValsCount = totalValLength - lastValidatorsCount;
   if (appearedValsCount === 0) {
     return { totalValLength, valKeys: [] };
   }
   const iterator = iterateNodesAtDepth(
-    stateView.validators.type.tree_getChunksNode(stateView.validators.node),
-    stateView.validators.type.chunkDepth,
+    validatorsView.type.tree_getChunksNode(validatorsView.node),
+    validatorsView.type.chunkDepth,
     lastValidatorsCount,
     appearedValsCount,
   );
   const valKeys = [];
   for (let i = lastValidatorsCount; i < totalValLength; i++) {
     const node = iterator.next().value;
-    const v = stateView.validators.type.elementType.tree_toValue(node);
+    const v = validatorsView.type.elementType.tree_toValue(node);
     valKeys.push(toHex(v.pubkey));
   }
   iterator.return?.();