From be03ef1429c78cb6a002f73367d1f57d997a2508 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 1 Aug 2024 16:24:04 -0400 Subject: [PATCH 001/145] chore: add more discovery logs (#6996) --- packages/beacon-node/src/network/peers/discover.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index 1cb084846f61..79603f780e3d 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -249,13 +249,21 @@ export class PeerDiscovery { } // Run a discv5 subnet query to try to discover new peers - if (subnetsToDiscoverPeers.length > 0 || cachedENRsToDial.size < peersToConnect) { + const shouldRunFindRandomNodeQuery = subnetsToDiscoverPeers.length > 0 || cachedENRsToDial.size < peersToConnect; + if (shouldRunFindRandomNodeQuery) { void this.runFindRandomNodeQuery(); } + + this.logger.debug("Discover peers outcome", { + peersToConnect, + peersAvailableToDial: cachedENRsToDial.size, + subnetsToDiscover: subnetsToDiscoverPeers.length, + shouldRunFindRandomNodeQuery, + }); } /** - * Request to find peers. First, looked at cached peers in peerStore + * Request discv5 to find peers if there is no query in progress */ private async runFindRandomNodeQuery(): Promise { // Delay the 1st query after starting discv5 @@ -305,6 +313,7 @@ export class PeerDiscovery { const attnets = zeroAttnets; const syncnets = zeroSyncnets; const status = this.handleDiscoveredPeer(id, multiaddrs[0], attnets, syncnets); + this.logger.debug("Discovered peer via libp2p", {peer: prettyPrintPeerId(id), status}); this.metrics?.discovery.discoveredStatus.inc({status}); }; @@ -336,6 +345,7 @@ export class PeerDiscovery { const syncnets = syncnetsBytes ? deserializeEnrSubnets(syncnetsBytes, SYNC_COMMITTEE_SUBNET_COUNT) : zeroSyncnets; const status = this.handleDiscoveredPeer(peerId, multiaddrTCP, attnets, syncnets); + this.logger.debug("Discovered peer via discv5", {peer: prettyPrintPeerId(peerId), status}); this.metrics?.discovery.discoveredStatus.inc({status}); }; From e1dec02b1016dccea6d7e7a8a118a48b3530b4fb Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 2 Aug 2024 09:15:48 -0400 Subject: [PATCH 002/145] chore: refactor chain event emits (#6993) * chore: refactor chain event emits * chore: wrap emits in callInNextEventLoop --- .../src/chain/blocks/importBlock.ts | 103 +++++++++--------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 906be51434c2..205d6bb361cc 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -101,34 +101,6 @@ export async function importBlock( this.metrics?.importBlock.bySource.inc({source}); this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex}); - // We want to import block asap so call all event handler in the next event loop - callInNextEventLoop(async () => { - this.emitter.emit(routes.events.EventType.block, { - block: blockRootHex, - slot: blockSlot, - executionOptimistic: blockSummary != null && isOptimisticBlock(blockSummary), - }); - - // dataPromise will not end up here, but preDeneb could. In future we might also allow syncing - // out of data range blocks and import then in forkchoice although one would not be able to - // attest and propose with such head similar to optimistic sync - if (blockInput.type === BlockInputType.availableData) { - const {blobsSource, blobs} = blockInput.blockData; - - this.metrics?.importBlock.blobsBySource.inc({blobsSource}); - for (const blobSidecar of blobs) { - const {index, kzgCommitment} = blobSidecar; - this.emitter.emit(routes.events.EventType.blobSidecar, { - blockRoot: blockRootHex, - slot: blockSlot, - index, - kzgCommitment: toHexString(kzgCommitment), - versionedHash: toHexString(kzgCommitmentToVersionedHash(kzgCommitment)), - }); - } - } - }); - // 3. Import attestations to fork choice // // - For each attestation @@ -413,32 +385,58 @@ export async function importBlock( // Send block events, only for recent enough blocks if (this.clock.currentSlot - blockSlot < EVENTSTREAM_EMIT_RECENT_BLOCK_SLOTS) { - // NOTE: Skip looping if there are no listeners from the API - if (this.emitter.listenerCount(routes.events.EventType.voluntaryExit)) { - for (const voluntaryExit of block.message.body.voluntaryExits) { - this.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit); + // We want to import block asap so call all event handler in the next event loop + callInNextEventLoop(() => { + // NOTE: Skip emitting if there are no listeners from the API + if (this.emitter.listenerCount(routes.events.EventType.block)) { + this.emitter.emit(routes.events.EventType.block, { + block: blockRootHex, + slot: blockSlot, + executionOptimistic: blockSummary != null && isOptimisticBlock(blockSummary), + }); } - } - if (this.emitter.listenerCount(routes.events.EventType.blsToExecutionChange)) { - for (const blsToExecutionChange of (block.message.body as capella.BeaconBlockBody).blsToExecutionChanges ?? []) { - this.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange); + if (this.emitter.listenerCount(routes.events.EventType.voluntaryExit)) { + for (const voluntaryExit of block.message.body.voluntaryExits) { + this.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit); + } } - } - if (this.emitter.listenerCount(routes.events.EventType.attestation)) { - for (const attestation of block.message.body.attestations) { - this.emitter.emit(routes.events.EventType.attestation, attestation); + if (this.emitter.listenerCount(routes.events.EventType.blsToExecutionChange)) { + for (const blsToExecutionChange of (block.message as capella.BeaconBlock).body.blsToExecutionChanges ?? []) { + this.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange); + } } - } - if (this.emitter.listenerCount(routes.events.EventType.attesterSlashing)) { - for (const attesterSlashing of block.message.body.attesterSlashings) { - this.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing); + if (this.emitter.listenerCount(routes.events.EventType.attestation)) { + for (const attestation of block.message.body.attestations) { + this.emitter.emit(routes.events.EventType.attestation, attestation); + } } - } - if (this.emitter.listenerCount(routes.events.EventType.proposerSlashing)) { - for (const proposerSlashing of block.message.body.proposerSlashings) { - this.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing); + if (this.emitter.listenerCount(routes.events.EventType.attesterSlashing)) { + for (const attesterSlashing of block.message.body.attesterSlashings) { + this.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing); + } } - } + if (this.emitter.listenerCount(routes.events.EventType.proposerSlashing)) { + for (const proposerSlashing of block.message.body.proposerSlashings) { + this.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing); + } + } + if ( + blockInput.type === BlockInputType.availableData && + this.emitter.listenerCount(routes.events.EventType.blobSidecar) + ) { + const {blobs} = blockInput.blockData; + for (const blobSidecar of blobs) { + const {index, kzgCommitment} = blobSidecar; + this.emitter.emit(routes.events.EventType.blobSidecar, { + blockRoot: blockRootHex, + slot: blockSlot, + index, + kzgCommitment: toHexString(kzgCommitment), + versionedHash: toHexString(kzgCommitmentToVersionedHash(kzgCommitment)), + }); + } + } + }); } // Register stat metrics about the block after importing it @@ -452,6 +450,13 @@ export async function importBlock( fullyVerifiedBlock.postState.epochCtx.currentSyncCommitteeIndexed.validatorIndices ); } + // dataPromise will not end up here, but preDeneb could. In future we might also allow syncing + // out of data range blocks and import then in forkchoice although one would not be able to + // attest and propose with such head similar to optimistic sync + if (blockInput.type === BlockInputType.availableData) { + const {blobsSource} = blockInput.blockData; + this.metrics?.importBlock.blobsBySource.inc({blobsSource}); + } const advancedSlot = this.clock.slotWithFutureTolerance(REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC); From 156e4340fa8b388fcfd258bc285b9e006bf06f4b Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 2 Aug 2024 20:19:02 +0700 Subject: [PATCH 003/145] chore: more log on updateHeadState() (#6997) * chore: more log on updateHeadState() * chore: also log maybeHeadStateRoot * fix: dontTransferCache option when regen head state --- .../src/chain/blocks/importBlock.ts | 2 +- .../beacon-node/src/chain/regen/interface.ts | 2 +- .../beacon-node/src/chain/regen/queued.ts | 36 ++++++++++++------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 205d6bb361cc..9d46410a8638 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -213,7 +213,7 @@ export async function importBlock( if (newHead.blockRoot !== oldHead.blockRoot) { // Set head state as strong reference - this.regen.updateHeadState(newHead.stateRoot, postState); + this.regen.updateHeadState(newHead, postState); this.emitter.emit(routes.events.EventType.head, { block: newHead.blockRoot, diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index 341625c9ff1d..031d19860789 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -44,7 +44,7 @@ export interface IStateRegenerator extends IStateRegeneratorInternal { pruneOnFinalized(finalizedEpoch: Epoch): void; processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void; addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void; - updateHeadState(newHeadStateRoot: RootHex, maybeHeadState: CachedBeaconStateAllForks): void; + updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void; updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; } diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 358a37e6e638..ad673b334bd1 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -165,28 +165,40 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.checkpointStateCache.add(cp, item); } - updateHeadState(newHeadStateRoot: RootHex, maybeHeadState: CachedBeaconStateAllForks): void { - // the resulting state will be added to block state cache so we transfer the cache in this flow - const cloneOpts = {dontTransferCache: true}; + updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void { + const {stateRoot: newHeadStateRoot, blockRoot: newHeadBlockRoot, slot: newHeadSlot} = newHead; + const maybeHeadStateRoot = toHexString(maybeHeadState.hashTreeRoot()); + const logCtx = { + newHeadSlot, + newHeadBlockRoot, + newHeadStateRoot, + maybeHeadSlot: maybeHeadState.slot, + maybeHeadStateRoot, + }; const headState = - newHeadStateRoot === toHexString(maybeHeadState.hashTreeRoot()) + newHeadStateRoot === maybeHeadStateRoot ? maybeHeadState - : this.blockStateCache.get(newHeadStateRoot, cloneOpts); + : // maybeHeadState was already in block state cache so we don't transfer the cache + this.blockStateCache.get(newHeadStateRoot, {dontTransferCache: true}); if (headState) { this.blockStateCache.setHeadState(headState); } else { // Trigger regen on head change if necessary - this.logger.warn("Head state not available, triggering regen", {stateRoot: newHeadStateRoot}); - // it's important to reload state to regen head state here - const allowDiskReload = true; - // head has changed, so the existing cached head state is no longer useful. Set strong reference to null to free - // up memory for regen step below. During regen, node won't be functional but eventually head will be available - // for legacy StateContextCache only + this.logger.warn("Head state not available, triggering regen", logCtx); + // for the old BlockStateCacheImpl only + // - head has changed, so the existing cached head state is no longer useful. Set strong reference to null to free + // up memory for regen step below. During regen, node won't be functional but eventually head will be available + // for the new FIFOBlockStateCache, this has no affect this.blockStateCache.setHeadState(null); + + // for the new FIFOBlockStateCache, it's important to reload state to regen head state here if needed + const allowDiskReload = true; + // transfer cache here because we want to regen state asap + const cloneOpts = {dontTransferCache: false}; this.regen.getState(newHeadStateRoot, RegenCaller.processBlock, cloneOpts, allowDiskReload).then( (headStateRegen) => this.blockStateCache.setHeadState(headStateRegen), - (e) => this.logger.error("Error on head state regen", {}, e) + (e) => this.logger.error("Error on head state regen", logCtx, e) ); } } From cf00c3f67e91b27dec865e9d82f20fa86d844eb1 Mon Sep 17 00:00:00 2001 From: g11tech Date: Fri, 2 Aug 2024 20:01:05 +0530 Subject: [PATCH 004/145] fix: refactor the blockcontent types as fork aware types for api (#6999) * fix: refactor the blockcontent types as fork aware types for api * Update packages/types/src/deneb/sszTypes.ts Co-authored-by: Nico Flaig --------- Co-authored-by: Cayman Co-authored-by: Nico Flaig --- .../api/src/beacon/routes/beacon/block.ts | 84 ++++++++----------- packages/types/src/deneb/sszTypes.ts | 18 ++++ packages/types/src/deneb/types.ts | 7 +- packages/types/src/types.ts | 8 +- 4 files changed, 60 insertions(+), 57 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index 73680ac0afc2..99fbf7d45645 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -11,8 +11,10 @@ import { BeaconBlockBody, SignedBeaconBlockOrContents, SignedBlindedBeaconBlock, + SignedBlockContents, + sszTypesFor, } from "@lodestar/types"; -import {ForkName, ForkPreExecution, ForkSeq, isForkExecution} from "@lodestar/params"; +import {ForkName, ForkPreExecution, isForkBlobs, isForkExecution} from "@lodestar/params"; import {Endpoint, RequestCodec, RouteDefinitions, Schema} from "../../../utils/index.js"; import {EmptyMeta, EmptyResponseCodec, EmptyResponseData, WithVersion} from "../../../utils/codecs.js"; import { @@ -37,19 +39,10 @@ export const BlockHeadersResponseType = new ListCompositeType(BlockHeaderRespons export const RootResponseType = new ContainerType({ root: ssz.Root, }); -export const SignedBlockContentsType = new ContainerType( - { - signedBlock: ssz.deneb.SignedBeaconBlock, - kzgProofs: ssz.deneb.KZGProofs, - blobs: ssz.deneb.Blobs, - }, - {jsonCase: "eth2"} -); export type BlockHeaderResponse = ValueOf; export type BlockHeadersResponse = ValueOf; export type RootResponse = ValueOf; -export type SignedBlockContents = ValueOf; export type BlockId = RootHex | Slot | "head" | "genesis" | "finalized" | "justified"; @@ -297,11 +290,12 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const slot = isSignedBlockContents(signedBlockOrContents) ? signedBlockOrContents.signedBlock.message.slot : signedBlockOrContents.message.slot; + const fork = config.getForkName(slot); + return { - body: - config.getForkSeq(slot) < ForkSeq.deneb - ? config.getForkTypes(slot).SignedBeaconBlock.serialize(signedBlockOrContents as SignedBeaconBlock) - : SignedBlockContentsType.serialize(signedBlockOrContents as SignedBlockContents), + body: isForkBlobs(fork) + ? sszTypesFor(fork).SignedBlockContents.serialize(signedBlockOrContents as SignedBlockContents) + : sszTypesFor(fork).SignedBeaconBlock.serialize(signedBlockOrContents as SignedBeaconBlock), headers: { [MetaHeader.Version]: config.getForkName(slot), }, @@ -345,12 +338,10 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const forkName = toForkName(fromHeaders(headers, MetaHeader.Version)); - const forkSeq = config.forks[forkName].seq; return { - signedBlockOrContents: - forkSeq < ForkSeq.deneb - ? ssz[forkName].SignedBeaconBlock.deserialize(body) - : SignedBlockContentsType.deserialize(body), + signedBlockOrContents: isForkBlobs(forkName) + ? sszTypesFor(forkName).SignedBlockContents.deserialize(body) + : ssz[forkName].SignedBeaconBlock.deserialize(body), }; }, schema: { @@ -371,25 +362,23 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const forkName = toForkName(fromHeaders(headers, MetaHeader.Version)); - const forkSeq = config.forks[forkName].seq; return { - signedBlockOrContents: - forkSeq < ForkSeq.deneb - ? ssz[forkName].SignedBeaconBlock.fromJson(body) - : SignedBlockContentsType.fromJson(body), + signedBlockOrContents: isForkBlobs(forkName) + ? sszTypesFor(forkName).SignedBlockContents.fromJson(body) + : ssz[forkName].SignedBeaconBlock.fromJson(body), broadcastValidation: query.broadcast_validation as BroadcastValidation, }; }, @@ -397,25 +386,24 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const forkName = toForkName(fromHeaders(headers, MetaHeader.Version)); - const forkSeq = config.forks[forkName].seq; return { - signedBlockOrContents: - forkSeq < ForkSeq.deneb - ? ssz[forkName].SignedBeaconBlock.deserialize(body) - : SignedBlockContentsType.deserialize(body), + signedBlockOrContents: isForkBlobs(forkName) + ? sszTypesFor(forkName).SignedBlockContents.deserialize(body) + : ssz[forkName].SignedBeaconBlock.deserialize(body), broadcastValidation: query.broadcast_validation as BroadcastValidation, }; }, diff --git a/packages/types/src/deneb/sszTypes.ts b/packages/types/src/deneb/sszTypes.ts index c93c3f145beb..076973bba579 100644 --- a/packages/types/src/deneb/sszTypes.ts +++ b/packages/types/src/deneb/sszTypes.ts @@ -312,3 +312,21 @@ export const SSEPayloadAttributes = new ContainerType( }, {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} ); + +export const BlockContents = new ContainerType( + { + block: BeaconBlock, + kzgProofs: KZGProofs, + blobs: Blobs, + }, + {typeName: "BlockContents", jsonCase: "eth2"} +); + +export const SignedBlockContents = new ContainerType( + { + signedBlock: SignedBeaconBlock, + kzgProofs: KZGProofs, + blobs: Blobs, + }, + {typeName: "SignedBlockContents", jsonCase: "eth2"} +); diff --git a/packages/types/src/deneb/types.ts b/packages/types/src/deneb/types.ts index 9a901c9a1a81..7ee6648aeaf2 100644 --- a/packages/types/src/deneb/types.ts +++ b/packages/types/src/deneb/types.ts @@ -1,6 +1,4 @@ import {ValueOf} from "@chainsafe/ssz"; -import {ForkName} from "@lodestar/params"; -import type {BlockContents} from "../types.js"; import * as ssz from "./sszTypes.js"; export type KZGProof = ValueOf; @@ -49,4 +47,7 @@ export type LightClientOptimisticUpdate = ValueOf; export type ProducedBlobSidecars = Omit; -export type Contents = Omit, "block">; + +export type BlockContents = ValueOf; +export type SignedBlockContents = ValueOf; +export type Contents = Omit; diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index 46641d55667e..3602299ae5e1 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -124,12 +124,8 @@ type TypesByFork = { BuilderBid: deneb.BuilderBid; SignedBuilderBid: deneb.SignedBuilderBid; SSEPayloadAttributes: deneb.SSEPayloadAttributes; - BlockContents: {block: BeaconBlock; kzgProofs: deneb.KZGProofs; blobs: deneb.Blobs}; - SignedBlockContents: { - signedBlock: SignedBeaconBlock; - kzgProofs: deneb.KZGProofs; - blobs: deneb.Blobs; - }; + BlockContents: deneb.BlockContents; + SignedBlockContents: deneb.SignedBlockContents; ExecutionPayloadAndBlobsBundle: deneb.ExecutionPayloadAndBlobsBundle; BlobsBundle: deneb.BlobsBundle; Contents: deneb.Contents; From 3d2c2b3b1759e947528d1df3a919e1a02a679d69 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 2 Aug 2024 23:02:49 +0100 Subject: [PATCH 005/145] feat: track syncing status and fetch duties on resynced (#6995) * feat: track syncing status and fetch duties on synced * Rename scheduling function to runOnResynced * Consider prev offline and syncing to trigger resynced event handlers * Add comment to error handler * Add note about el offline and sycning not considered * Align syncing status logs with existing node is syncing logs * Cleanup * Add ssz support to syncing status api * Align beacon node code to return proper types * Keep track of error in prev syncing status * Print slot in error log * Skip on first slot of epoch since tasks are already scheduled * Update api test data * Fix endpoint tests * await scheduled tasks, mostly relevant for testing * Add unit tests * Move beacon heath metric to syncing status tracker * Add beacon health panel to validator client dashboard * Formatting * Improve info called once assertion * Reset mocks after each test --- dashboards/lodestar_validator_client.json | 134 +++++++++++++--- packages/api/src/beacon/routes/node.ts | 34 ++-- .../api/test/unit/beacon/testData/node.ts | 2 +- packages/beacon-node/src/sync/sync.ts | 12 +- .../api/impl/beacon/node/endpoints.test.ts | 7 +- packages/cli/test/sim/endpoints.test.ts | 4 +- packages/state-transition/src/util/epoch.ts | 7 + packages/validator/src/metrics.ts | 6 +- .../validator/src/services/attestation.ts | 17 +- .../src/services/attestationDuties.ts | 10 +- .../validator/src/services/syncCommittee.ts | 17 +- .../src/services/syncCommitteeDuties.ts | 16 +- .../src/services/syncingStatusTracker.ts | 74 +++++++++ packages/validator/src/validator.ts | 32 ++-- .../test/unit/services/attestation.test.ts | 5 + .../unit/services/attestationDuties.test.ts | 124 ++++++++++++--- .../unit/services/syncCommitteDuties.test.ts | 129 +++++++++++++-- .../test/unit/services/syncCommittee.test.ts | 5 + .../services/syncingStatusTracker.test.ts | 149 ++++++++++++++++++ .../validator/test/unit/utils/metrics.test.ts | 2 +- packages/validator/test/utils/apiStub.ts | 3 + packages/validator/test/utils/logger.ts | 14 ++ 22 files changed, 693 insertions(+), 110 deletions(-) create mode 100644 packages/validator/src/services/syncingStatusTracker.ts create mode 100644 packages/validator/test/unit/services/syncingStatusTracker.test.ts diff --git a/dashboards/lodestar_validator_client.json b/dashboards/lodestar_validator_client.json index 8ec6a04437b1..7ede7ba0bcff 100644 --- a/dashboards/lodestar_validator_client.json +++ b/dashboards/lodestar_validator_client.json @@ -71,6 +71,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "msg / slot", @@ -211,10 +212,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "name" + "textMode": "name", + "wideLayout": true }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -231,7 +234,6 @@ } ], "title": "Lodestar version", - "transformations": [], "type": "stat" }, { @@ -267,10 +269,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "name" + "textMode": "name", + "wideLayout": true }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -321,10 +325,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "name" + "textMode": "name", + "wideLayout": true }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -359,7 +365,7 @@ }, "gridPos": { "h": 2, - "w": 4, + "w": 2, "x": 20, "y": 0 }, @@ -376,9 +382,11 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -395,6 +403,83 @@ "title": "VC indices", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "Ready" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "Syncing" + }, + "2": { + "color": "red", + "index": 2, + "text": "Error" + } + }, + "type": "value" + } + ] + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 0 + }, + "id": 47, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "vc_beacon_health", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Beacon health", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -429,9 +514,11 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -482,9 +569,11 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -622,7 +711,7 @@ }, "showHeader": false }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -701,6 +790,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -866,6 +956,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -1019,7 +1110,8 @@ }, "showValue": "never", "tooltip": { - "show": true, + "mode": "single", + "showColorScale": false, "yHistogram": false }, "yAxis": { @@ -1028,7 +1120,7 @@ "unit": "s" } }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "reverseYBuckets": false, "targets": [ { @@ -1133,7 +1225,8 @@ }, "showValue": "never", "tooltip": { - "show": true, + "mode": "single", + "showColorScale": false, "yHistogram": false }, "yAxis": { @@ -1142,7 +1235,7 @@ "unit": "s" } }, - "pluginVersion": "10.1.1", + "pluginVersion": "10.4.1", "reverseYBuckets": false, "targets": [ { @@ -2344,8 +2437,7 @@ ], "refresh": "10s", "revision": 1, - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [ "lodestar" ], diff --git a/packages/api/src/beacon/routes/node.ts b/packages/api/src/beacon/routes/node.ts index 1ff0378c3330..0744b5f07452 100644 --- a/packages/api/src/beacon/routes/node.ts +++ b/packages/api/src/beacon/routes/node.ts @@ -43,6 +43,22 @@ export const PeerCountType = new ContainerType( {jsonCase: "eth2"} ); +export const SyncingStatusType = new ContainerType( + { + /** Head slot node is trying to reach */ + headSlot: ssz.Slot, + /** How many slots node needs to process to reach head. 0 if synced. */ + syncDistance: ssz.Slot, + /** Set to true if the node is syncing, false if the node is synced. */ + isSyncing: ssz.Boolean, + /** Set to true if the node is optimistically tracking head. */ + isOptimistic: ssz.Boolean, + /** Set to true if the connected el client is offline */ + elOffline: ssz.Boolean, + }, + {jsonCase: "eth2"} +); + export type NetworkIdentity = ValueOf; export type PeerState = "disconnected" | "connecting" | "connected" | "disconnecting"; @@ -66,18 +82,7 @@ export type FilterGetPeers = { direction?: PeerDirection[]; }; -export type SyncingStatus = { - /** Head slot node is trying to reach */ - headSlot: string; - /** How many slots node needs to process to reach head. 0 if synced. */ - syncDistance: string; - /** Set to true if the node is syncing, false if the node is synced. */ - isSyncing: boolean; - /** Set to true if the node is optimistically tracking head. */ - isOptimistic: boolean; - /** Set to true if the connected el client is offline */ - elOffline: boolean; -}; +export type SyncingStatus = ValueOf; export enum NodeHealth { READY = HttpStatusCode.OK, @@ -243,7 +248,10 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions = { }, getSyncingStatus: { args: undefined, - res: {data: {headSlot: "1", syncDistance: "2", isSyncing: false, isOptimistic: true, elOffline: false}}, + res: {data: {headSlot: 1, syncDistance: 2, isSyncing: false, isOptimistic: true, elOffline: false}}, }, getHealth: { args: {syncingStatus: 206}, diff --git a/packages/beacon-node/src/sync/sync.ts b/packages/beacon-node/src/sync/sync.ts index c7f01e1eae78..cc8ddc6eb499 100644 --- a/packages/beacon-node/src/sync/sync.ts +++ b/packages/beacon-node/src/sync/sync.ts @@ -93,8 +93,8 @@ export class BeaconSync implements IBeaconSync { // If we are pre/at genesis, signal ready if (currentSlot <= GENESIS_SLOT) { return { - headSlot: "0", - syncDistance: "0", + headSlot: 0, + syncDistance: 0, isSyncing: false, isOptimistic: false, elOffline, @@ -107,16 +107,16 @@ export class BeaconSync implements IBeaconSync { case SyncState.SyncingHead: case SyncState.Stalled: return { - headSlot: String(head.slot), - syncDistance: String(currentSlot - head.slot), + headSlot: head.slot, + syncDistance: currentSlot - head.slot, isSyncing: true, isOptimistic: isOptimisticBlock(head), elOffline, }; case SyncState.Synced: return { - headSlot: String(head.slot), - syncDistance: "0", + headSlot: head.slot, + syncDistance: 0, isSyncing: false, isOptimistic: isOptimisticBlock(head), elOffline, diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts index d85fdb80720f..2d2a0a37c59e 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts @@ -1,6 +1,7 @@ import {describe, beforeAll, afterAll, it, expect, vi} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; +import {routes} from "@lodestar/api"; import {ApiClient, getClient} from "@lodestar/api/beacon"; import {sleep} from "@lodestar/utils"; import {LogLevel, testLogger} from "../../../../../utils/logger.js"; @@ -46,9 +47,9 @@ describe("beacon node api", function () { it("should return valid syncing status", async () => { const res = await client.node.getSyncingStatus(); - expect(res.value()).toEqual({ - headSlot: "0", - syncDistance: "0", + expect(res.value()).toEqual({ + headSlot: 0, + syncDistance: 0, isSyncing: false, isOptimistic: false, elOffline: false, diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts index a40a18e379eb..07cd5fec0cc4 100644 --- a/packages/cli/test/sim/endpoints.test.ts +++ b/packages/cli/test/sim/endpoints.test.ts @@ -107,8 +107,8 @@ await env.tracker.assert( await env.tracker.assert("BN Not Synced", async () => { const expectedSyncStatus: routes.node.SyncingStatus = { - headSlot: "2", - syncDistance: "0", + headSlot: 2, + syncDistance: 0, isSyncing: false, isOptimistic: false, elOffline: false, diff --git a/packages/state-transition/src/util/epoch.ts b/packages/state-transition/src/util/epoch.ts index bb66fb04eb94..ba182f627de0 100644 --- a/packages/state-transition/src/util/epoch.ts +++ b/packages/state-transition/src/util/epoch.ts @@ -70,3 +70,10 @@ export function computeSyncPeriodAtSlot(slot: Slot): SyncPeriod { export function computeSyncPeriodAtEpoch(epoch: Epoch): SyncPeriod { return Math.floor(epoch / EPOCHS_PER_SYNC_COMMITTEE_PERIOD); } + +/** + * Determine if the given slot is start slot of an epoch + */ +export function isStartSlotOfEpoch(slot: Slot): boolean { + return slot % SLOTS_PER_EPOCH === 0; +} diff --git a/packages/validator/src/metrics.ts b/packages/validator/src/metrics.ts index dc7d1a11ffac..a437328e8d5f 100644 --- a/packages/validator/src/metrics.ts +++ b/packages/validator/src/metrics.ts @@ -8,9 +8,7 @@ export enum MessageSource { export enum BeaconHealth { READY = 0, SYNCING = 1, - NOT_INITIALIZED_OR_ISSUES = 2, - UNKNOWN = 3, - ERROR = 4, + ERROR = 2, } export type Metrics = ReturnType; @@ -279,7 +277,7 @@ export function getMetrics(register: MetricsRegisterExtra, gitData: LodestarGitD beaconHealth: register.gauge({ name: "vc_beacon_health", - help: `Current health status of the beacon(s) the validator is connected too. ${renderEnumNumeric(BeaconHealth)}`, + help: `Current health status of the beacon(s) the validator is connected to. ${renderEnumNumeric(BeaconHealth)}`, }), restApiClient: { diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 57a8a7621a97..9191b251a6de 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -10,6 +10,7 @@ import {ValidatorStore} from "./validatorStore.js"; import {AttestationDutiesService, AttDutyAndProof} from "./attestationDuties.js"; import {groupAttDutiesByCommitteeIndex} from "./utils.js"; import {ChainHeaderTracker} from "./chainHeaderTracker.js"; +import {SyncingStatusTracker} from "./syncingStatusTracker.js"; import {ValidatorEventEmitter} from "./emitter.js"; export type AttestationServiceOpts = { @@ -40,12 +41,22 @@ export class AttestationService { private readonly validatorStore: ValidatorStore, private readonly emitter: ValidatorEventEmitter, chainHeadTracker: ChainHeaderTracker, + syncingStatusTracker: SyncingStatusTracker, private readonly metrics: Metrics | null, private readonly opts?: AttestationServiceOpts ) { - this.dutiesService = new AttestationDutiesService(logger, api, clock, validatorStore, chainHeadTracker, metrics, { - distributedAggregationSelection: opts?.distributedAggregationSelection, - }); + this.dutiesService = new AttestationDutiesService( + logger, + api, + clock, + validatorStore, + chainHeadTracker, + syncingStatusTracker, + metrics, + { + distributedAggregationSelection: opts?.distributedAggregationSelection, + } + ); // At most every slot, check existing duties from AttestationDutiesService and run tasks clock.runEverySlot(this.runAttestationTasks); diff --git a/packages/validator/src/services/attestationDuties.ts b/packages/validator/src/services/attestationDuties.ts index 1f278aebbd89..ea82bf0a4c72 100644 --- a/packages/validator/src/services/attestationDuties.ts +++ b/packages/validator/src/services/attestationDuties.ts @@ -1,7 +1,7 @@ import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; -import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; +import {computeEpochAtSlot, isAggregatorFromCommitteeLength, isStartSlotOfEpoch} from "@lodestar/state-transition"; import {BLSSignature, Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; import {ApiClient, routes} from "@lodestar/api"; import {batchItems, IClock, LoggerVc} from "../util/index.js"; @@ -9,6 +9,7 @@ import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; import {ValidatorStore} from "./validatorStore.js"; import {ChainHeaderTracker, HeadEventData} from "./chainHeaderTracker.js"; +import {SyncingStatusTracker} from "./syncingStatusTracker.js"; /** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch. */ const HISTORICAL_DUTIES_EPOCHS = 2; @@ -52,6 +53,7 @@ export class AttestationDutiesService { private clock: IClock, private readonly validatorStore: ValidatorStore, chainHeadTracker: ChainHeaderTracker, + syncingStatusTracker: SyncingStatusTracker, private readonly metrics: Metrics | null, private readonly opts?: AttestationDutiesServiceOpts ) { @@ -60,6 +62,12 @@ export class AttestationDutiesService { clock.runEveryEpoch(this.runDutiesTasks); clock.runEverySlot(this.prepareForNextEpoch); chainHeadTracker.runOnNewHead(this.onNewHead); + syncingStatusTracker.runOnResynced(async (slot) => { + // Skip on first slot of epoch since tasks are already scheduled + if (!isStartSlotOfEpoch(slot)) { + return this.runDutiesTasks(computeEpochAtSlot(slot)); + } + }); if (metrics) { metrics.attesterDutiesCount.addCollect(() => { diff --git a/packages/validator/src/services/syncCommittee.ts b/packages/validator/src/services/syncCommittee.ts index 06926724141c..c960adc6986b 100644 --- a/packages/validator/src/services/syncCommittee.ts +++ b/packages/validator/src/services/syncCommittee.ts @@ -11,6 +11,7 @@ import {SyncCommitteeDutiesService, SyncDutyAndProofs} from "./syncCommitteeDuti import {groupSyncDutiesBySubcommitteeIndex, SubcommitteeDuty} from "./utils.js"; import {ChainHeaderTracker} from "./chainHeaderTracker.js"; import {ValidatorEventEmitter} from "./emitter.js"; +import {SyncingStatusTracker} from "./syncingStatusTracker.js"; export type SyncCommitteeServiceOpts = { scAfterBlockDelaySlotFraction?: number; @@ -31,12 +32,22 @@ export class SyncCommitteeService { private readonly validatorStore: ValidatorStore, private readonly emitter: ValidatorEventEmitter, private readonly chainHeaderTracker: ChainHeaderTracker, + readonly syncingStatusTracker: SyncingStatusTracker, private readonly metrics: Metrics | null, private readonly opts?: SyncCommitteeServiceOpts ) { - this.dutiesService = new SyncCommitteeDutiesService(config, logger, api, clock, validatorStore, metrics, { - distributedAggregationSelection: opts?.distributedAggregationSelection, - }); + this.dutiesService = new SyncCommitteeDutiesService( + config, + logger, + api, + clock, + validatorStore, + syncingStatusTracker, + metrics, + { + distributedAggregationSelection: opts?.distributedAggregationSelection, + } + ); // At most every slot, check existing duties from SyncCommitteeDutiesService and run tasks clock.runEverySlot(this.runSyncCommitteeTasks); diff --git a/packages/validator/src/services/syncCommitteeDuties.ts b/packages/validator/src/services/syncCommitteeDuties.ts index edc62dea575c..dd663528f751 100644 --- a/packages/validator/src/services/syncCommitteeDuties.ts +++ b/packages/validator/src/services/syncCommitteeDuties.ts @@ -1,6 +1,12 @@ import {toHexString} from "@chainsafe/ssz"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; -import {computeSyncPeriodAtEpoch, computeSyncPeriodAtSlot, isSyncCommitteeAggregator} from "@lodestar/state-transition"; +import { + computeEpochAtSlot, + computeSyncPeriodAtEpoch, + computeSyncPeriodAtSlot, + isStartSlotOfEpoch, + isSyncCommitteeAggregator, +} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {BLSSignature, Epoch, Slot, SyncPeriod, ValidatorIndex} from "@lodestar/types"; import {ApiClient, routes} from "@lodestar/api"; @@ -9,6 +15,7 @@ import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; import {ValidatorStore} from "./validatorStore.js"; import {syncCommitteeIndicesToSubnets} from "./utils.js"; +import {SyncingStatusTracker} from "./syncingStatusTracker.js"; /** Only retain `HISTORICAL_DUTIES_PERIODS` duties prior to the current periods. */ const HISTORICAL_DUTIES_PERIODS = 2; @@ -80,12 +87,19 @@ export class SyncCommitteeDutiesService { private readonly api: ApiClient, clock: IClock, private readonly validatorStore: ValidatorStore, + syncingStatusTracker: SyncingStatusTracker, metrics: Metrics | null, private readonly opts?: SyncCommitteeDutiesServiceOpts ) { // Running this task every epoch is safe since a re-org of many epochs is very unlikely // TODO: If the re-org event is reliable consider re-running then clock.runEveryEpoch(this.runDutiesTasks); + syncingStatusTracker.runOnResynced(async (slot) => { + // Skip on first slot of epoch since tasks are already scheduled + if (!isStartSlotOfEpoch(slot)) { + return this.runDutiesTasks(computeEpochAtSlot(slot)); + } + }); if (metrics) { metrics.syncCommitteeDutiesCount.addCollect(() => { diff --git a/packages/validator/src/services/syncingStatusTracker.ts b/packages/validator/src/services/syncingStatusTracker.ts new file mode 100644 index 000000000000..4c38e670092d --- /dev/null +++ b/packages/validator/src/services/syncingStatusTracker.ts @@ -0,0 +1,74 @@ +import {ApiClient, routes} from "@lodestar/api"; +import {Logger} from "@lodestar/utils"; +import {Slot} from "@lodestar/types"; +import {IClock} from "../util/clock.js"; +import {BeaconHealth, Metrics} from "../metrics.js"; + +export type SyncingStatus = routes.node.SyncingStatus; + +type RunOnResyncedFn = (slot: Slot, signal: AbortSignal) => Promise; + +/** + * Track the syncing status of connected beacon node(s) + */ +export class SyncingStatusTracker { + private prevSyncingStatus?: SyncingStatus | Error; + + private readonly fns: RunOnResyncedFn[] = []; + + constructor( + private readonly logger: Logger, + private readonly api: ApiClient, + private readonly clock: IClock, + private readonly metrics: Metrics | null + ) { + this.clock.runEverySlot(this.checkSyncingStatus); + } + + /** + * Run function when node status changes from syncing to synced + * + * Note: does not consider if execution client is offline or syncing and + * hence it is not useful to schedule tasks that require a non-optimistic node. + */ + runOnResynced(fn: RunOnResyncedFn): void { + this.fns.push(fn); + } + + private checkSyncingStatus = async (slot: Slot, signal: AbortSignal): Promise => { + try { + const syncingStatus = (await this.api.node.getSyncingStatus()).value(); + const {isSyncing, headSlot, syncDistance, isOptimistic, elOffline} = syncingStatus; + const prevErrorOrSyncing = this.prevSyncingStatus instanceof Error || this.prevSyncingStatus?.isSyncing === true; + + if (isSyncing === true) { + this.logger.warn("Node is syncing", {slot, headSlot, syncDistance}); + } else if (this.prevSyncingStatus === undefined || prevErrorOrSyncing) { + this.logger.info("Node is synced", {slot, headSlot, isOptimistic, elOffline}); + } + this.logger.verbose("Node syncing status", {slot, ...syncingStatus}); + + this.prevSyncingStatus = syncingStatus; + + this.metrics?.beaconHealth.set( + !isSyncing && !isOptimistic && !elOffline ? BeaconHealth.READY : BeaconHealth.SYNCING + ); + + if (prevErrorOrSyncing && isSyncing === false) { + await Promise.all( + this.fns.map((fn) => + fn(slot, signal).catch((e) => this.logger.error("Error calling resynced event handler", e)) + ) + ); + } + } catch (e) { + // Error likely due to node being offline. In any case, handle failure to + // check syncing status the same way as if node was previously syncing + this.prevSyncingStatus = e as Error; + + this.metrics?.beaconHealth.set(BeaconHealth.ERROR); + + this.logger.error("Failed to check syncing status", {slot}, this.prevSyncingStatus); + } + }; +} diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 461762560fee..1732f54ababf 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -16,10 +16,11 @@ import {ExternalSignerOptions, pollExternalSignerPubkeys} from "./services/exter import {Interchange, InterchangeFormatVersion, ISlashingProtection} from "./slashingProtection/index.js"; import {assertEqualParams, getLoggerVc, NotEqualParamsError} from "./util/index.js"; import {ChainHeaderTracker} from "./services/chainHeaderTracker.js"; +import {SyncingStatusTracker} from "./services/syncingStatusTracker.js"; import {ValidatorEventEmitter} from "./services/emitter.js"; import {ValidatorStore, Signer, ValidatorProposerConfig, defaultOptions} from "./services/validatorStore.js"; import {LodestarValidatorDatabaseController, ProcessShutdownCallback, PubkeyHex} from "./types.js"; -import {BeaconHealth, Metrics} from "./metrics.js"; +import {Metrics} from "./metrics.js"; import {MetaDataRepository} from "./repositories/metaDataRepository.js"; import {DoppelgangerService} from "./services/doppelgangerService.js"; @@ -35,6 +36,7 @@ export type ValidatorModules = { api: ApiClient; clock: IClock; chainHeaderTracker: ChainHeaderTracker; + syncingStatusTracker: SyncingStatusTracker; logger: Logger; db: LodestarValidatorDatabaseController; metrics: Metrics | null; @@ -89,6 +91,7 @@ export class Validator { private readonly api: ApiClient; private readonly clock: IClock; private readonly chainHeaderTracker: ChainHeaderTracker; + readonly syncingStatusTracker: SyncingStatusTracker; private readonly logger: Logger; private readonly db: LodestarValidatorDatabaseController; private state: Status; @@ -106,6 +109,7 @@ export class Validator { api, clock, chainHeaderTracker, + syncingStatusTracker, logger, db, metrics, @@ -121,6 +125,7 @@ export class Validator { this.api = api; this.clock = clock; this.chainHeaderTracker = chainHeaderTracker; + this.syncingStatusTracker = syncingStatusTracker; this.logger = logger; this.controller = controller; this.db = db; @@ -145,12 +150,6 @@ export class Validator { if (metrics) { this.db.setMetrics(metrics.db); - - this.clock.runEverySlot(() => - this.fetchBeaconHealth() - .then((health) => metrics.beaconHealth.set(health)) - .catch((e) => this.logger.error("Error on fetchBeaconHealth", {}, e)) - ); } // "start" the validator @@ -225,6 +224,7 @@ export class Validator { emitter.setMaxListeners(Infinity); const chainHeaderTracker = new ChainHeaderTracker(logger, api, emitter); + const syncingStatusTracker = new SyncingStatusTracker(logger, api, clock, metrics); const blockProposingService = new BlockProposingService(config, loggerVc, api, clock, validatorStore, metrics, { useProduceBlockV3: opts.useProduceBlockV3, @@ -239,6 +239,7 @@ export class Validator { validatorStore, emitter, chainHeaderTracker, + syncingStatusTracker, metrics, { afterBlockDelaySlotFraction: opts.afterBlockDelaySlotFraction, @@ -255,6 +256,7 @@ export class Validator { validatorStore, emitter, chainHeaderTracker, + syncingStatusTracker, metrics, { scAfterBlockDelaySlotFraction: opts.scAfterBlockDelaySlotFraction, @@ -274,6 +276,7 @@ export class Validator { api, clock, chainHeaderTracker, + syncingStatusTracker, logger, db, metrics, @@ -382,21 +385,6 @@ export class Validator { return this.validatorStore.signVoluntaryExit(publicKey, validator.index, exitEpoch); } - - private async fetchBeaconHealth(): Promise { - try { - const {status: healthCode} = await this.api.node.getHealth(); - - if (healthCode === routes.node.NodeHealth.READY) return BeaconHealth.READY; - if (healthCode === routes.node.NodeHealth.SYNCING) return BeaconHealth.SYNCING; - if (healthCode === routes.node.NodeHealth.NOT_INITIALIZED_OR_ISSUES) - return BeaconHealth.NOT_INITIALIZED_OR_ISSUES; - else return BeaconHealth.UNKNOWN; - } catch (e) { - // TODO: Filter by network error type - return BeaconHealth.ERROR; - } - } } /** Assert the same genesisValidatorRoot and genesisTime */ diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index 0ffec323ee30..11779ee496b3 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -10,12 +10,14 @@ import {getApiClientStub, mockApiResponse} from "../../utils/apiStub.js"; import {loggerVc} from "../../utils/logger.js"; import {ClockMock} from "../../utils/clock.js"; import {ChainHeaderTracker} from "../../../src/services/chainHeaderTracker.js"; +import {SyncingStatusTracker} from "../../../src/services/syncingStatusTracker.js"; import {ValidatorEventEmitter} from "../../../src/services/emitter.js"; import {ZERO_HASH, ZERO_HASH_HEX} from "../../utils/types.js"; vi.mock("../../../src/services/validatorStore.js"); vi.mock("../../../src/services/emitter.js"); vi.mock("../../../src/services/chainHeaderTracker.js"); +vi.mock("../../../src/services/syncingStatusTracker.js"); describe("AttestationService", function () { const api = getApiClientStub(); @@ -24,6 +26,8 @@ describe("AttestationService", function () { const emitter = vi.mocked(new ValidatorEventEmitter()); // @ts-expect-error - Mocked class don't need parameters const chainHeadTracker = vi.mocked(new ChainHeaderTracker()); + // @ts-expect-error - Mocked class don't need parameters + const syncingStatusTracker = vi.mocked(new SyncingStatusTracker()); let pubkeys: Uint8Array[]; // Initialize pubkeys in before() so bls is already initialized @@ -62,6 +66,7 @@ describe("AttestationService", function () { validatorStore, emitter, chainHeadTracker, + syncingStatusTracker, null, opts ); diff --git a/packages/validator/test/unit/services/attestationDuties.test.ts b/packages/validator/test/unit/services/attestationDuties.test.ts index ad54c0735eea..751ebcf11205 100644 --- a/packages/validator/test/unit/services/attestationDuties.test.ts +++ b/packages/validator/test/unit/services/attestationDuties.test.ts @@ -13,6 +13,7 @@ import {loggerVc} from "../../utils/logger.js"; import {ClockMock} from "../../utils/clock.js"; import {initValidatorStore} from "../../utils/validatorStore.js"; import {ChainHeaderTracker} from "../../../src/services/chainHeaderTracker.js"; +import {SyncingStatusTracker} from "../../../src/services/syncingStatusTracker.js"; import {ZERO_HASH_HEX} from "../../utils/types.js"; vi.mock("../../../src/services/chainHeaderTracker.js"); @@ -45,10 +46,6 @@ describe("AttestationDutiesService", function () { let controller: AbortController; // To stop clock beforeEach(() => { controller = new AbortController(); - }); - afterEach(() => controller.abort()); - - it("Should fetch indexes and duties", async function () { // Reply with an active validator that has an index const validatorResponse = { ...defaultValidator, @@ -58,7 +55,13 @@ describe("AttestationDutiesService", function () { api.beacon.getStateValidators.mockResolvedValue( mockApiResponse({data: [validatorResponse], meta: {executionOptimistic: false, finalized: false}}) ); + }); + afterEach(() => { + vi.restoreAllMocks(); + controller.abort(); + }); + it("Should fetch indexes and duties", async function () { // Reply with some duties const slot = 1; const epoch = computeEpochAtSlot(slot); @@ -78,9 +81,18 @@ describe("AttestationDutiesService", function () { // Accept all subscriptions api.validator.prepareBeaconCommitteeSubnet.mockResolvedValue(mockApiResponse({})); - // Clock will call runAttesterDutiesTasks() immediately + // Clock will call runDutiesTasks() immediately const clock = new ClockMock(); - const dutiesService = new AttestationDutiesService(loggerVc, api, clock, validatorStore, chainHeadTracker, null); + const syncingStatusTracker = new SyncingStatusTracker(loggerVc, api, clock, null); + const dutiesService = new AttestationDutiesService( + loggerVc, + api, + clock, + validatorStore, + chainHeadTracker, + syncingStatusTracker, + null + ); // Trigger clock onSlot for slot 0 await clock.tickEpochFns(0, controller.signal); @@ -107,16 +119,6 @@ describe("AttestationDutiesService", function () { }); it("Should remove signer from attestation duties", async function () { - // Reply with an active validator that has an index - const validatorResponse = { - ...defaultValidator, - index, - validator: {...defaultValidator.validator, pubkey: pubkeys[0]}, - }; - api.beacon.getStateValidators.mockResolvedValue( - mockApiResponse({data: [validatorResponse], meta: {executionOptimistic: false, finalized: false}}) - ); - // Reply with some duties const slot = 1; const duty: routes.validator.AttesterDuty = { @@ -135,9 +137,18 @@ describe("AttestationDutiesService", function () { // Accept all subscriptions api.validator.prepareBeaconCommitteeSubnet.mockResolvedValue(mockApiResponse({})); - // Clock will call runAttesterDutiesTasks() immediately + // Clock will call runDutiesTasks() immediately const clock = new ClockMock(); - const dutiesService = new AttestationDutiesService(loggerVc, api, clock, validatorStore, chainHeadTracker, null); + const syncingStatusTracker = new SyncingStatusTracker(loggerVc, api, clock, null); + const dutiesService = new AttestationDutiesService( + loggerVc, + api, + clock, + validatorStore, + chainHeadTracker, + syncingStatusTracker, + null + ); // Trigger clock onSlot for slot 0 await clock.tickEpochFns(0, controller.signal); @@ -153,4 +164,81 @@ describe("AttestationDutiesService", function () { dutiesService.removeDutiesForKey(toHexString(pubkeys[0])); expect(Object.fromEntries(dutiesService["dutiesByIndexByEpoch"])).toEqual({}); }); + + it("Should fetch duties when node is resynced", async function () { + // Node is syncing + api.node.getSyncingStatus.mockResolvedValue( + mockApiResponse({data: {headSlot: 0, syncDistance: 1, isSyncing: true, isOptimistic: false, elOffline: false}}) + ); + api.validator.getAttesterDuties.mockRejectedValue(Error("Node is syncing")); + api.validator.prepareBeaconCommitteeSubnet.mockRejectedValue(Error("Node is syncing")); + + // Clock will call runDutiesTasks() immediately + const clock = new ClockMock(); + const syncingStatusTracker = new SyncingStatusTracker(loggerVc, api, clock, null); + const dutiesService = new AttestationDutiesService( + loggerVc, + api, + clock, + validatorStore, + chainHeadTracker, + syncingStatusTracker, + null + ); + + // Trigger clock for slot and epoch + await clock.tickEpochFns(0, controller.signal); + await clock.tickSlotFns(1, controller.signal); + + const dutySlot = 3; + const epoch = computeEpochAtSlot(dutySlot); + + // Duties for slot should be empty as node is still syncing + expect(dutiesService.getDutiesAtSlot(dutySlot)).toEqual([]); + + // Node is synced now + api.node.getSyncingStatus.mockResolvedValue( + mockApiResponse({data: {headSlot: 1, syncDistance: 0, isSyncing: false, isOptimistic: false, elOffline: false}}) + ); + + // Reply with some duties on next call + const duty: routes.validator.AttesterDuty = { + slot: dutySlot, + committeeIndex: 1, + committeeLength: 120, + committeesAtSlot: 120, + validatorCommitteeIndex: 1, + validatorIndex: index, + pubkey: pubkeys[0], + }; + api.validator.getAttesterDuties.mockResolvedValue( + mockApiResponse({data: [duty], meta: {dependentRoot: ZERO_HASH_HEX, executionOptimistic: false}}) + ); + + // Accept all subscriptions + api.validator.prepareBeaconCommitteeSubnet.mockResolvedValue(mockApiResponse({})); + + // Only tick clock for slot to not trigger regular polling + await clock.tickSlotFns(2, controller.signal); + + // Validator index should be persisted + expect(validatorStore.getAllLocalIndices()).toEqual([index]); + expect(validatorStore.getPubkeyOfIndex(index)).toBe(toHexString(pubkeys[0])); + + // Duties for this and next epoch should be persisted + expect(Object.fromEntries(dutiesService["dutiesByIndexByEpoch"].get(epoch)?.dutiesByIndex || new Map())).toEqual({ + // Since the ZERO_HASH won't pass the isAggregator test, selectionProof is null + [index]: {duty, selectionProof: null}, + }); + expect( + Object.fromEntries(dutiesService["dutiesByIndexByEpoch"].get(epoch + 1)?.dutiesByIndex || new Map()) + ).toEqual({ + // Since the ZERO_HASH won't pass the isAggregator test, selectionProof is null + [index]: {duty, selectionProof: null}, + }); + + expect(dutiesService.getDutiesAtSlot(dutySlot)).toEqual([{duty, selectionProof: null}]); + + expect(api.validator.prepareBeaconCommitteeSubnet).toHaveBeenCalledOnce(); + }); }); diff --git a/packages/validator/test/unit/services/syncCommitteDuties.test.ts b/packages/validator/test/unit/services/syncCommitteDuties.test.ts index 52f2071f9102..10bac3ab2fe9 100644 --- a/packages/validator/test/unit/services/syncCommitteDuties.test.ts +++ b/packages/validator/test/unit/services/syncCommitteDuties.test.ts @@ -1,4 +1,4 @@ -import {describe, it, expect, beforeAll, beforeEach, afterEach} from "vitest"; +import {describe, it, expect, beforeAll, beforeEach, afterEach, vi} from "vitest"; import {when} from "vitest-when"; import {toBufferBE} from "bigint-buffer"; import {toHexString} from "@chainsafe/ssz"; @@ -13,6 +13,7 @@ import { SyncDutySubnet, } from "../../../src/services/syncCommitteeDuties.js"; import {ValidatorStore} from "../../../src/services/validatorStore.js"; +import {SyncingStatusTracker} from "../../../src/services/syncingStatusTracker.js"; import {getApiClientStub, mockApiResponse} from "../../utils/apiStub.js"; import {loggerVc} from "../../utils/logger.js"; import {ClockMock} from "../../utils/clock.js"; @@ -63,7 +64,10 @@ describe("SyncCommitteeDutiesService", function () { mockApiResponse({data: validatorResponses, meta: {executionOptimistic: false, finalized: false}}) ); }); - afterEach(() => controller.abort()); + afterEach(() => { + vi.restoreAllMocks(); + controller.abort(); + }); it("Should fetch indexes and duties", async function () { // Reply with some duties @@ -80,14 +84,22 @@ describe("SyncCommitteeDutiesService", function () { // Accept all subscriptions api.validator.prepareSyncCommitteeSubnets.mockResolvedValue(mockApiResponse({})); - // Clock will call runAttesterDutiesTasks() immediately + // Clock will call runDutiesTasks() immediately const clock = new ClockMock(); - const dutiesService = new SyncCommitteeDutiesService(altair0Config, loggerVc, api, clock, validatorStore, null); + const syncingStatusTracker = new SyncingStatusTracker(loggerVc, api, clock, null); + const dutiesService = new SyncCommitteeDutiesService( + altair0Config, + loggerVc, + api, + clock, + validatorStore, + syncingStatusTracker, + null + ); // Trigger clock onSlot for slot 0 await clock.tickEpochFns(0, controller.signal); - // Validator index should be persisted // Validator index should be persisted expect(validatorStore.getAllLocalIndices()).toEqual(indices); for (let i = 0; i < indices.length; i++) { @@ -107,9 +119,9 @@ describe("SyncCommitteeDutiesService", function () { 1: {[indices[0]]: {duty: toSyncDutySubnet(duty)}}, } as typeof dutiesByIndexByPeriodObj); - expect(await dutiesService.getDutiesAtSlot(slot)).toEqual([ + expect(await dutiesService.getDutiesAtSlot(slot)).toEqual([ {duty: toSyncDutySubnet(duty), selectionProofs: [{selectionProof: null, subcommitteeIndex: 0}]}, - ] as SyncDutyAndProofs[]); + ]); expect(api.validator.prepareSyncCommitteeSubnets).toHaveBeenCalledOnce(); }); @@ -143,9 +155,18 @@ describe("SyncCommitteeDutiesService", function () { .calledWith({epoch: 1, indices}) .thenResolve(mockApiResponse({data: [duty2], meta: {executionOptimistic: false}})); - // Clock will call runAttesterDutiesTasks() immediately + // Clock will call runDutiesTasks() immediately const clock = new ClockMock(); - const dutiesService = new SyncCommitteeDutiesService(altair0Config, loggerVc, api, clock, validatorStore, null); + const syncingStatusTracker = new SyncingStatusTracker(loggerVc, api, clock, null); + const dutiesService = new SyncCommitteeDutiesService( + altair0Config, + loggerVc, + api, + clock, + validatorStore, + syncingStatusTracker, + null + ); // Trigger clock onSlot for slot 0 await clock.tickEpochFns(0, controller.signal); @@ -196,9 +217,18 @@ describe("SyncCommitteeDutiesService", function () { // Accept all subscriptions api.validator.prepareSyncCommitteeSubnets.mockResolvedValue(mockApiResponse({})); - // Clock will call runAttesterDutiesTasks() immediately + // Clock will call runDutiesTasks() immediately const clock = new ClockMock(); - const dutiesService = new SyncCommitteeDutiesService(altair0Config, loggerVc, api, clock, validatorStore, null); + const syncingStatusTracker = new SyncingStatusTracker(loggerVc, api, clock, null); + const dutiesService = new SyncCommitteeDutiesService( + altair0Config, + loggerVc, + api, + clock, + validatorStore, + syncingStatusTracker, + null + ); // Trigger clock onSlot for slot 0 await clock.tickEpochFns(0, controller.signal); @@ -236,6 +266,83 @@ describe("SyncCommitteeDutiesService", function () { 1: {[indices[1]]: {duty: toSyncDutySubnet(duty2)}}, } as typeof dutiesByIndexByPeriodObjAfterRemoval); }); + + it("Should fetch duties when node is resynced", async function () { + // Node is syncing + api.node.getSyncingStatus.mockResolvedValue( + mockApiResponse({data: {headSlot: 0, syncDistance: 1, isSyncing: true, isOptimistic: false, elOffline: false}}) + ); + api.validator.getSyncCommitteeDuties.mockRejectedValue(Error("Node is syncing")); + api.validator.prepareSyncCommitteeSubnets.mockRejectedValue(Error("Node is syncing")); + + // Clock will call runDutiesTasks() immediately + const clock = new ClockMock(); + const syncingStatusTracker = new SyncingStatusTracker(loggerVc, api, clock, null); + const dutiesService = new SyncCommitteeDutiesService( + altair0Config, + loggerVc, + api, + clock, + validatorStore, + syncingStatusTracker, + null + ); + + // Trigger clock for slot and epoch + await clock.tickEpochFns(0, controller.signal); + await clock.tickSlotFns(1, controller.signal); + + const dutySlot = 1; + + // Duties for slot should be empty as node is still syncing + expect(await dutiesService.getDutiesAtSlot(dutySlot)).toEqual([]); + + // Node is synced now + api.node.getSyncingStatus.mockResolvedValue( + mockApiResponse({data: {headSlot: 1, syncDistance: 0, isSyncing: false, isOptimistic: false, elOffline: false}}) + ); + + // Reply with some duties + const duty: routes.validator.SyncDuty = { + pubkey: pubkeys[0], + validatorIndex: indices[0], + validatorSyncCommitteeIndices: [7], + }; + api.validator.getSyncCommitteeDuties.mockResolvedValue( + mockApiResponse({data: [duty], meta: {executionOptimistic: false}}) + ); + + // Accept all subscriptions + api.validator.prepareSyncCommitteeSubnets.mockResolvedValue(mockApiResponse({})); + + // Only tick clock for slot to not trigger regular polling + await clock.tickSlotFns(2, controller.signal); + + // Validator index should be persisted + expect(validatorStore.getAllLocalIndices()).toEqual(indices); + for (let i = 0; i < indices.length; i++) { + expect(validatorStore.getPubkeyOfIndex(indices[i])).toBe(toHexString(pubkeys[i])); + } + + // Duties for this and next epoch should be persisted + const dutiesByIndexByPeriodObj = Object.fromEntries( + Array.from(dutiesService["dutiesByIndexByPeriod"].entries()).map(([period, dutiesByIndex]) => [ + period, + Object.fromEntries(dutiesByIndex), + ]) + ); + + expect(dutiesByIndexByPeriodObj).toEqual({ + 0: {[indices[0]]: {duty: toSyncDutySubnet(duty)}}, + 1: {[indices[0]]: {duty: toSyncDutySubnet(duty)}}, + } as typeof dutiesByIndexByPeriodObj); + + expect(await dutiesService.getDutiesAtSlot(dutySlot)).toEqual([ + {duty: toSyncDutySubnet(duty), selectionProofs: [{selectionProof: null, subcommitteeIndex: 0}]}, + ]); + + expect(api.validator.prepareSyncCommitteeSubnets).toHaveBeenCalledOnce(); + }); }); function toSyncDutySubnet(duty: routes.validator.SyncDuty): SyncDutySubnet { diff --git a/packages/validator/test/unit/services/syncCommittee.test.ts b/packages/validator/test/unit/services/syncCommittee.test.ts index c65912f12e9a..66e63ab72a6c 100644 --- a/packages/validator/test/unit/services/syncCommittee.test.ts +++ b/packages/validator/test/unit/services/syncCommittee.test.ts @@ -12,12 +12,14 @@ import {getApiClientStub, mockApiResponse} from "../../utils/apiStub.js"; import {loggerVc} from "../../utils/logger.js"; import {ClockMock} from "../../utils/clock.js"; import {ChainHeaderTracker} from "../../../src/services/chainHeaderTracker.js"; +import {SyncingStatusTracker} from "../../../src/services/syncingStatusTracker.js"; import {ZERO_HASH} from "../../utils/types.js"; import {ValidatorEventEmitter} from "../../../src/services/emitter.js"; vi.mock("../../../src/services/validatorStore.js"); vi.mock("../../../src/services/emitter.js"); vi.mock("../../../src/services/chainHeaderTracker.js"); +vi.mock("../../../src/services/syncingStatusTracker.js"); /* eslint-disable @typescript-eslint/naming-convention */ @@ -28,6 +30,8 @@ describe("SyncCommitteeService", function () { const emitter = vi.mocked(new ValidatorEventEmitter()); // @ts-expect-error - Mocked class don't need parameters const chainHeaderTracker = vi.mocked(new ChainHeaderTracker()); + // @ts-expect-error - Mocked class don't need parameters + const syncingStatusTracker = vi.mocked(new SyncingStatusTracker()); let pubkeys: Uint8Array[]; // Initialize pubkeys in before() so bls is already initialized const config = createChainForkConfig({ @@ -71,6 +75,7 @@ describe("SyncCommitteeService", function () { validatorStore, emitter, chainHeaderTracker, + syncingStatusTracker, null, opts ); diff --git a/packages/validator/test/unit/services/syncingStatusTracker.test.ts b/packages/validator/test/unit/services/syncingStatusTracker.test.ts new file mode 100644 index 000000000000..59029e1b9c51 --- /dev/null +++ b/packages/validator/test/unit/services/syncingStatusTracker.test.ts @@ -0,0 +1,149 @@ +import {describe, it, expect, vi, beforeEach, afterEach, MockedFunction} from "vitest"; +import {getApiClientStub, mockApiResponse} from "../../utils/apiStub.js"; +import {getMockedLogger} from "../../utils/logger.js"; +import {ClockMock} from "../../utils/clock.js"; +import {SyncingStatus, SyncingStatusTracker} from "../../../src/services/syncingStatusTracker.js"; + +describe("SyncingStatusTracker", function () { + const api = getApiClientStub(); + const logger = getMockedLogger(); + + let controller: AbortController; + let clock: ClockMock; + let syncingStatusTracker: SyncingStatusTracker; + let callOnResynced: MockedFunction<() => Promise>; + + beforeEach(() => { + controller = new AbortController(); + clock = new ClockMock(); + syncingStatusTracker = new SyncingStatusTracker(logger, api, clock, null); + callOnResynced = vi.fn().mockResolvedValue(undefined); + syncingStatusTracker.runOnResynced(callOnResynced); + }); + + afterEach(() => { + vi.resetAllMocks(); + controller.abort(); + }); + + it("should handle transition from syncing to synced", async function () { + // Node is syncing + const syncing: SyncingStatus = { + headSlot: 0, + syncDistance: 1, + isSyncing: true, + isOptimistic: false, + elOffline: false, + }; + api.node.getSyncingStatus.mockResolvedValue(mockApiResponse({data: syncing})); + + await clock.tickSlotFns(1, controller.signal); + + expect(logger.warn).toHaveBeenCalledWith("Node is syncing", {slot: 1, headSlot: 0, syncDistance: 1}); + expect(logger.verbose).toHaveBeenCalledWith("Node syncing status", { + slot: 1, + headSlot: 0, + syncDistance: 1, + isSyncing: true, + isOptimistic: false, + elOffline: false, + }); + expect(syncingStatusTracker["prevSyncingStatus"]).toBe(syncing); + + // Transition to synced + const synced: SyncingStatus = { + headSlot: 2, + syncDistance: 0, + isSyncing: false, + isOptimistic: false, + elOffline: false, + }; + api.node.getSyncingStatus.mockResolvedValue(mockApiResponse({data: synced})); + + await clock.tickSlotFns(2, controller.signal); + + expect(logger.info).toHaveBeenCalledWith("Node is synced", { + slot: 2, + headSlot: 2, + isOptimistic: false, + elOffline: false, + }); + expect(logger.verbose).toHaveBeenCalledWith("Node syncing status", { + slot: 2, + headSlot: 2, + syncDistance: 0, + isSyncing: false, + isOptimistic: false, + elOffline: false, + }); + expect(syncingStatusTracker["prevSyncingStatus"]).toBe(synced); + expect(callOnResynced).toHaveBeenCalledOnce(); + }); + + it("should handle errors when checking syncing status", async function () { + // Node is offline + const error = new Error("ECONNREFUSED"); + api.node.getSyncingStatus.mockRejectedValue(error); + + await clock.tickSlotFns(1, controller.signal); + + expect(logger.error).toHaveBeenCalledWith("Failed to check syncing status", {slot: 1}, error); + expect(syncingStatusTracker["prevSyncingStatus"]).toBe(error); + expect(callOnResynced).not.toHaveBeenCalled(); + }); + + it("should not call scheduled tasks if already synced", async function () { + // Node is already synced + const syncedHead1: SyncingStatus = { + headSlot: 1, + syncDistance: 0, + isSyncing: false, + isOptimistic: false, + elOffline: false, + }; + api.node.getSyncingStatus.mockResolvedValue(mockApiResponse({data: syncedHead1})); + + await clock.tickSlotFns(1, controller.signal); + + expect(logger.info).toHaveBeenCalledWith("Node is synced", { + slot: 1, + headSlot: 1, + isOptimistic: false, + elOffline: false, + }); + expect(logger.verbose).toHaveBeenCalledWith("Node syncing status", { + slot: 1, + headSlot: 1, + syncDistance: 0, + isSyncing: false, + isOptimistic: false, + elOffline: false, + }); + expect(syncingStatusTracker["prevSyncingStatus"]).toBe(syncedHead1); + + // Still synced on next tick + const syncedHead2: SyncingStatus = { + headSlot: 2, + syncDistance: 0, + isSyncing: false, + isOptimistic: false, + elOffline: false, + }; + api.node.getSyncingStatus.mockResolvedValue(mockApiResponse({data: syncedHead2})); + + await clock.tickSlotFns(2, controller.signal); + + // info log should only be printed out once, not every slot + expect(logger.info).toHaveBeenCalledOnce(); + expect(logger.verbose).toHaveBeenCalledWith("Node syncing status", { + slot: 2, + headSlot: 2, + syncDistance: 0, + isSyncing: false, + isOptimistic: false, + elOffline: false, + }); + expect(syncingStatusTracker["prevSyncingStatus"]).toBe(syncedHead2); + expect(callOnResynced).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/validator/test/unit/utils/metrics.test.ts b/packages/validator/test/unit/utils/metrics.test.ts index 695e8731b7f6..de4010761001 100644 --- a/packages/validator/test/unit/utils/metrics.test.ts +++ b/packages/validator/test/unit/utils/metrics.test.ts @@ -3,6 +3,6 @@ import {BeaconHealth, renderEnumNumeric} from "../../../src/metrics.js"; describe("renderEnumNumeric", () => { it("BeaconHealth", () => { - expect(renderEnumNumeric(BeaconHealth)).toBe("READY=0, SYNCING=1, NOT_INITIALIZED_OR_ISSUES=2, UNKNOWN=3, ERROR=4"); + expect(renderEnumNumeric(BeaconHealth)).toBe("READY=0, SYNCING=1, ERROR=2"); }); }); diff --git a/packages/validator/test/utils/apiStub.ts b/packages/validator/test/utils/apiStub.ts index ac41c7145128..3c1d80fff75d 100644 --- a/packages/validator/test/utils/apiStub.ts +++ b/packages/validator/test/utils/apiStub.ts @@ -21,6 +21,9 @@ export function getApiClientStub(): ApiClientStub { submitPoolSyncCommitteeSignatures: vi.fn(), submitPoolAttestations: vi.fn(), }, + node: { + getSyncingStatus: vi.fn(), + }, validator: { getProposerDuties: vi.fn(), getAttesterDuties: vi.fn(), diff --git a/packages/validator/test/utils/logger.ts b/packages/validator/test/utils/logger.ts index 8eaed6dfe6c1..44f5190dfe79 100644 --- a/packages/validator/test/utils/logger.ts +++ b/packages/validator/test/utils/logger.ts @@ -1,3 +1,5 @@ +import {vi, Mocked} from "vitest"; +import {Logger} from "@lodestar/logger"; import {getEnvLogger} from "@lodestar/logger/env"; import {getLoggerVc} from "../../src/util/index.js"; import {ClockMock} from "./clock.js"; @@ -13,3 +15,15 @@ import {ClockMock} from "./clock.js"; export const testLogger = getEnvLogger; export const loggerVc = getLoggerVc(getEnvLogger(), new ClockMock()); + +export type MockedLogger = Mocked; + +export function getMockedLogger(): MockedLogger { + return { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + verbose: vi.fn(), + }; +} From 2f86f4aafde02a45c5d82206a0db1f308033182f Mon Sep 17 00:00:00 2001 From: SeaMonkey82 Date: Sun, 4 Aug 2024 05:43:16 -0400 Subject: [PATCH 006/145] chore: update lodestar_validator_monitor.json (#7000) Update lodestar_validator_monitor.json "Percent of attestations having correct head" -> "Percent of attestations having incorrect head" --- dashboards/lodestar_validator_monitor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/lodestar_validator_monitor.json b/dashboards/lodestar_validator_monitor.json index 7579a595b550..54ab6dbe12a5 100644 --- a/dashboards/lodestar_validator_monitor.json +++ b/dashboards/lodestar_validator_monitor.json @@ -500,7 +500,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Percent of attestations having correct head.", + "description": "Percent of attestations having incorrect head.", "fieldConfig": { "defaults": { "color": { From 703249ce3591d4aa09d37e100a46b575eaacd65b Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 5 Aug 2024 20:27:11 +0100 Subject: [PATCH 007/145] chore: update default instance selected as filter on dashboards (#7001) --- dashboards/lodestar_block_processor.json | 2 +- dashboards/lodestar_block_production.json | 2 +- dashboards/lodestar_bls_thread_pool.json | 2 +- dashboards/lodestar_debug_gossipsub.json | 2 +- dashboards/lodestar_discv5.json | 2 +- dashboards/lodestar_execution_engine.json | 2 +- dashboards/lodestar_historical_state_regen.json | 2 +- dashboards/lodestar_libp2p.json | 2 +- dashboards/lodestar_multinode.json | 2 +- dashboards/lodestar_networking.json | 2 +- dashboards/lodestar_rest_api.json | 2 +- dashboards/lodestar_state_cache_regen.json | 2 +- dashboards/lodestar_summary.json | 2 +- dashboards/lodestar_sync.json | 2 +- dashboards/lodestar_validator_client.json | 2 +- dashboards/lodestar_validator_monitor.json | 2 +- dashboards/lodestar_vm_host.json | 2 +- scripts/lint-grafana-dashboard.mjs | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dashboards/lodestar_block_processor.json b/dashboards/lodestar_block_processor.json index 50513613c680..3258efa72fc0 100644 --- a/dashboards/lodestar_block_processor.json +++ b/dashboards/lodestar_block_processor.json @@ -7132,7 +7132,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_block_production.json b/dashboards/lodestar_block_production.json index 5e33a72ef9af..58c7ba3f3684 100644 --- a/dashboards/lodestar_block_production.json +++ b/dashboards/lodestar_block_production.json @@ -2211,7 +2211,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_bls_thread_pool.json b/dashboards/lodestar_bls_thread_pool.json index 867d9fd322c4..0fd2572d5a1d 100644 --- a/dashboards/lodestar_bls_thread_pool.json +++ b/dashboards/lodestar_bls_thread_pool.json @@ -1483,7 +1483,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_debug_gossipsub.json b/dashboards/lodestar_debug_gossipsub.json index 0486bd58fbd3..26cd9b9b791a 100644 --- a/dashboards/lodestar_debug_gossipsub.json +++ b/dashboards/lodestar_debug_gossipsub.json @@ -8876,7 +8876,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_discv5.json b/dashboards/lodestar_discv5.json index 31f115936df2..ca7f300060b4 100644 --- a/dashboards/lodestar_discv5.json +++ b/dashboards/lodestar_discv5.json @@ -2168,7 +2168,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_execution_engine.json b/dashboards/lodestar_execution_engine.json index 2c4cadc131f1..63c1ab636e51 100644 --- a/dashboards/lodestar_execution_engine.json +++ b/dashboards/lodestar_execution_engine.json @@ -3439,7 +3439,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_historical_state_regen.json b/dashboards/lodestar_historical_state_regen.json index 20eddcd1f31e..bc7106bf8de9 100644 --- a/dashboards/lodestar_historical_state_regen.json +++ b/dashboards/lodestar_historical_state_regen.json @@ -2601,7 +2601,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_libp2p.json b/dashboards/lodestar_libp2p.json index 0fe72e0a4032..8d391dee2bbe 100644 --- a/dashboards/lodestar_libp2p.json +++ b/dashboards/lodestar_libp2p.json @@ -1348,7 +1348,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_multinode.json b/dashboards/lodestar_multinode.json index 9a8fecaf0128..6c7f18dc3148 100644 --- a/dashboards/lodestar_multinode.json +++ b/dashboards/lodestar_multinode.json @@ -844,7 +844,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index d1c40e659194..a239d47ac62a 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -6421,7 +6421,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_rest_api.json b/dashboards/lodestar_rest_api.json index 6445873f4dc5..7ef29bb89ec5 100644 --- a/dashboards/lodestar_rest_api.json +++ b/dashboards/lodestar_rest_api.json @@ -649,7 +649,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_state_cache_regen.json b/dashboards/lodestar_state_cache_regen.json index be52d414ea3b..e7f6336eaeb9 100644 --- a/dashboards/lodestar_state_cache_regen.json +++ b/dashboards/lodestar_state_cache_regen.json @@ -3088,7 +3088,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_summary.json b/dashboards/lodestar_summary.json index 5e8773c05d4e..87eaed30bd3e 100644 --- a/dashboards/lodestar_summary.json +++ b/dashboards/lodestar_summary.json @@ -2929,7 +2929,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_sync.json b/dashboards/lodestar_sync.json index 6cc82bedde47..f58ccf308f9b 100644 --- a/dashboards/lodestar_sync.json +++ b/dashboards/lodestar_sync.json @@ -1776,7 +1776,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_validator_client.json b/dashboards/lodestar_validator_client.json index 7ede7ba0bcff..5e4459d1d1b9 100644 --- a/dashboards/lodestar_validator_client.json +++ b/dashboards/lodestar_validator_client.json @@ -2547,7 +2547,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_validator_monitor.json b/dashboards/lodestar_validator_monitor.json index 54ab6dbe12a5..3305e6796115 100644 --- a/dashboards/lodestar_validator_monitor.json +++ b/dashboards/lodestar_validator_monitor.json @@ -1896,7 +1896,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/dashboards/lodestar_vm_host.json b/dashboards/lodestar_vm_host.json index 457d4d120fa4..23e1c0418aba 100644 --- a/dashboards/lodestar_vm_host.json +++ b/dashboards/lodestar_vm_host.json @@ -5924,7 +5924,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "unstable-lg1k-hzax41-dkr" } ], "hide": 0, diff --git a/scripts/lint-grafana-dashboard.mjs b/scripts/lint-grafana-dashboard.mjs index e5e43fc3d9cf..84eb92e94040 100644 --- a/scripts/lint-grafana-dashboard.mjs +++ b/scripts/lint-grafana-dashboard.mjs @@ -244,7 +244,7 @@ export function lintGrafanaDashboard(json) { condition: "", key: "instance", operator: "=", - value: "unstable-lg1k-hzax41", + value: "unstable-lg1k-hzax41-dkr", }, ], hide: 0, From 30a13f91faec36a5220588e14a6aac2bec52abd1 Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 6 Aug 2024 22:08:15 +0700 Subject: [PATCH 008/145] fix: use default node image to build docker (#7004) * fix: use default node image instead of alpine * fix: use apt-get * Apply suggestions from code review --------- Co-authored-by: Cayman --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c65cac28d51d..e4b90c50bd20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-alpine as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-slim as build_src ARG COMMIT WORKDIR /usr/app -RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* +RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/* COPY . . @@ -21,21 +21,21 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:22.4-alpine as build_deps +FROM node:22.4-slim as build_deps WORKDIR /usr/app -RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* +RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=build_src /usr/app . # Do yarn --force to trigger a rebuild of the native packages -# Emmulates `yarn rebuild` which is not available in v1 https://yarnpkg.com/cli/rebuild +# Emmulates `yarn rebuild` which is not available in v1 https://yarnpkg.com/cli/rebuild RUN yarn install --non-interactive --frozen-lockfile --production --force # Rebuild leveldb bindings (required for arm64 build) RUN cd node_modules/classic-level && yarn rebuild # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) -FROM node:22.4-alpine +FROM node:22.4-slim WORKDIR /usr/app COPY --from=build_deps /usr/app . From 360fe779cd07bbe69f47c87d36e3016ea937cdb6 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 6 Aug 2024 16:58:51 -0400 Subject: [PATCH 009/145] chore: bump package versions to 1.21.0 --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 6 +++--- packages/db/package.json | 8 ++++---- packages/flare/package.json | 16 ++++++++-------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 12 ++++++------ packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 8 ++++---- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 101 insertions(+), 101 deletions(-) diff --git a/lerna.json b/lerna.json index ccdcaca872ec..043a5c48569a 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.20.2", + "version": "1.21.0", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 5b91b46c128e..9a250fdc491c 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -72,10 +72,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@lodestar/config": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index cc12fa961923..8d0189c710bf 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -119,18 +119,18 @@ "@libp2p/peer-id-factory": "^4.1.0", "@libp2p/prometheus-metrics": "^3.0.21", "@libp2p/tcp": "9.0.23", - "@lodestar/api": "^1.20.2", - "@lodestar/config": "^1.20.2", - "@lodestar/db": "^1.20.2", - "@lodestar/fork-choice": "^1.20.2", - "@lodestar/light-client": "^1.20.2", - "@lodestar/logger": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/reqresp": "^1.20.2", - "@lodestar/state-transition": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", - "@lodestar/validator": "^1.20.2", + "@lodestar/api": "^1.21.0", + "@lodestar/config": "^1.21.0", + "@lodestar/db": "^1.21.0", + "@lodestar/fork-choice": "^1.21.0", + "@lodestar/light-client": "^1.21.0", + "@lodestar/logger": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/reqresp": "^1.21.0", + "@lodestar/state-transition": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", + "@lodestar/validator": "^1.21.0", "@multiformats/multiaddr": "^12.1.3", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 6b7bf36bacd2..6ce630057922 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.20.2", + "version": "1.21.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -62,17 +62,17 @@ "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", "@libp2p/peer-id-factory": "^4.1.0", - "@lodestar/api": "^1.20.2", - "@lodestar/beacon-node": "^1.20.2", - "@lodestar/config": "^1.20.2", - "@lodestar/db": "^1.20.2", - "@lodestar/light-client": "^1.20.2", - "@lodestar/logger": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/state-transition": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", - "@lodestar/validator": "^1.20.2", + "@lodestar/api": "^1.21.0", + "@lodestar/beacon-node": "^1.21.0", + "@lodestar/config": "^1.21.0", + "@lodestar/db": "^1.21.0", + "@lodestar/light-client": "^1.21.0", + "@lodestar/logger": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/state-transition": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", + "@lodestar/validator": "^1.21.0", "@multiformats/multiaddr": "^12.1.3", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -88,7 +88,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.2", + "@lodestar/test-utils": "^1.21.0", "@types/debug": "^4.1.7", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", diff --git a/packages/config/package.json b/packages/config/package.json index 8d5fd3d80c35..cd4157f9d073 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.20.2", + "version": "1.21.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,7 +65,7 @@ ], "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/params": "^1.20.2", - "@lodestar/types": "^1.20.2" + "@lodestar/params": "^1.21.0", + "@lodestar/types": "^1.21.0" } } diff --git a/packages/db/package.json b/packages/db/package.json index 6cfaf6ecce70..25aa5cdd2eea 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.20.2", + "version": "1.21.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -36,12 +36,12 @@ }, "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@lodestar/config": "^1.21.0", + "@lodestar/utils": "^1.21.0", "classic-level": "^1.4.1", "it-all": "^3.0.4" }, "devDependencies": { - "@lodestar/logger": "^1.20.2" + "@lodestar/logger": "^1.21.0" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index 9e9560d32dd8..461147274591 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.20.2", + "version": "1.21.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -58,14 +58,14 @@ "blockchain" ], "dependencies": { - "@chainsafe/blst": "^2.0.3", "@chainsafe/bls-keygen": "^0.4.0", - "@lodestar/api": "^1.20.2", - "@lodestar/config": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/state-transition": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@chainsafe/blst": "^2.0.3", + "@lodestar/api": "^1.21.0", + "@lodestar/config": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/state-transition": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 2da624af4581..70673b210149 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -37,11 +37,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/state-transition": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2" + "@lodestar/config": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/state-transition": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 6576bddfee6a..d836290ec163 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -77,11 +77,11 @@ "@chainsafe/blst": "^0.2.0", "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/api": "^1.20.2", - "@lodestar/config": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@lodestar/api": "^1.21.0", + "@lodestar/config": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", "mitt": "^3.0.0" }, "devDependencies": { diff --git a/packages/logger/package.json b/packages/logger/package.json index 117040580d60..1ab176dbfd93 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.20.2", + "@lodestar/utils": "^1.21.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.20.2", + "@lodestar/test-utils": "^1.21.0", "@types/triple-beam": "^1.3.2", "triple-beam": "^1.3.0" }, diff --git a/packages/params/package.json b/packages/params/package.json index 20f50f625cfd..c246918959e4 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.20.2", + "version": "1.21.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index f5d8c26105d4..974125e2a17f 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.20.2", - "@lodestar/config": "^1.20.2", - "@lodestar/light-client": "^1.20.2", - "@lodestar/logger": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@lodestar/api": "^1.21.0", + "@lodestar/config": "^1.21.0", + "@lodestar/light-client": "^1.21.0", + "@lodestar/logger": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", "ethereum-cryptography": "^2.0.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.2", + "@lodestar/test-utils": "^1.21.0", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 5f72d47e4efb..1b9d99b000a8 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -54,9 +54,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.3.0", - "@lodestar/config": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@lodestar/config": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/utils": "^1.21.0", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -65,8 +65,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.20.2", - "@lodestar/types": "^1.20.2", + "@lodestar/logger": "^1.21.0", + "@lodestar/types": "^1.21.0", "libp2p": "1.4.3" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index e1240193671b..166df67a7409 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.20.2", + "version": "1.21.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,7 +62,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.20.2", + "@lodestar/utils": "^1.21.0", "axios": "^1.3.4", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 0271f64b684f..50f95edc498b 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -63,10 +63,10 @@ "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@lodestar/config": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", "bigint-buffer": "^1.1.5" }, "keywords": [ diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index f3b0a2da223c..6c8c9e62894f 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.20.2", + "version": "1.21.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -57,10 +57,10 @@ "blockchain" ], "dependencies": { - "@chainsafe/blst": "^2.0.3", "@chainsafe/bls-keystore": "^3.1.0", - "@lodestar/params": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@chainsafe/blst": "^2.0.3", + "@lodestar/params": "^1.21.0", + "@lodestar/utils": "^1.21.0", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index d5512763b197..eb2801c90156 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": { ".": { @@ -71,7 +71,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/params": "^1.20.2", + "@lodestar/params": "^1.21.0", "ethereum-cryptography": "^2.0.0" }, "keywords": [ diff --git a/packages/utils/package.json b/packages/utils/package.json index e3ada3e2901b..df77692de56f 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.2", + "version": "1.21.0", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index abf5f8797d89..134574e761cf 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.20.2", + "version": "1.21.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -47,17 +47,17 @@ "dependencies": { "@chainsafe/blst": "^2.0.3", "@chainsafe/ssz": "^0.15.1", - "@lodestar/api": "^1.20.2", - "@lodestar/config": "^1.20.2", - "@lodestar/db": "^1.20.2", - "@lodestar/params": "^1.20.2", - "@lodestar/state-transition": "^1.20.2", - "@lodestar/types": "^1.20.2", - "@lodestar/utils": "^1.20.2", + "@lodestar/api": "^1.21.0", + "@lodestar/config": "^1.21.0", + "@lodestar/db": "^1.21.0", + "@lodestar/params": "^1.21.0", + "@lodestar/state-transition": "^1.21.0", + "@lodestar/types": "^1.21.0", + "@lodestar/utils": "^1.21.0", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.2", + "@lodestar/test-utils": "^1.21.0", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From a0f16eb0fed836d006fb28728e4b9e493e09ae98 Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 7 Aug 2024 16:21:00 -0400 Subject: [PATCH 010/145] chore: bump ssz dependencies (#7007) --- packages/api/package.json | 4 +- packages/beacon-node/package.json | 6 +-- packages/cli/package.json | 4 +- packages/config/package.json | 2 +- packages/db/package.json | 2 +- packages/fork-choice/package.json | 2 +- packages/light-client/package.json | 6 +-- packages/state-transition/package.json | 6 +-- packages/types/package.json | 2 +- packages/utils/package.json | 2 +- packages/validator/package.json | 2 +- yarn.lock | 58 +++++++++++++++++++------- 12 files changed, 63 insertions(+), 33 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 5b91b46c128e..c2b1618dbbb8 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -70,8 +70,8 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/persistent-merkle-tree": "^0.7.1", - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/persistent-merkle-tree": "^0.8.0", + "@chainsafe/ssz": "^0.17.0", "@lodestar/config": "^1.20.2", "@lodestar/params": "^1.20.2", "@lodestar/types": "^1.20.2", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index cc12fa961923..27da4e153733 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -94,16 +94,16 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/as-sha256": "^0.4.1", + "@chainsafe/as-sha256": "^0.5.0", "@chainsafe/blst": "^2.0.3", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/libp2p-gossipsub": "^13.0.0", "@chainsafe/libp2p-identify": "^1.0.0", "@chainsafe/libp2p-noise": "^15.0.0", - "@chainsafe/persistent-merkle-tree": "^0.7.1", + "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/prometheus-gc-stats": "^1.0.0", - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/ssz": "^0.17.0", "@chainsafe/threads": "^1.11.1", "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^9.0.0", diff --git a/packages/cli/package.json b/packages/cli/package.json index 6b7bf36bacd2..9563c205512d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -56,8 +56,8 @@ "@chainsafe/blst": "^2.0.3", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", - "@chainsafe/persistent-merkle-tree": "^0.7.1", - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/persistent-merkle-tree": "^0.8.0", + "@chainsafe/ssz": "^0.17.0", "@chainsafe/threads": "^1.11.1", "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", diff --git a/packages/config/package.json b/packages/config/package.json index 8d5fd3d80c35..c94debf912aa 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -64,7 +64,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/ssz": "^0.17.0", "@lodestar/params": "^1.20.2", "@lodestar/types": "^1.20.2" } diff --git a/packages/db/package.json b/packages/db/package.json index 6cfaf6ecce70..59f3d9c7d91e 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -35,7 +35,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/ssz": "^0.17.0", "@lodestar/config": "^1.20.2", "@lodestar/utils": "^1.20.2", "classic-level": "^1.4.1", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 2da624af4581..ef9c630399b5 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -36,7 +36,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/ssz": "^0.17.0", "@lodestar/config": "^1.20.2", "@lodestar/params": "^1.20.2", "@lodestar/state-transition": "^1.20.2", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 6576bddfee6a..0283a32cc8c6 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -75,8 +75,8 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/blst": "^0.2.0", - "@chainsafe/persistent-merkle-tree": "^0.7.1", - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/persistent-merkle-tree": "^0.8.0", + "@chainsafe/ssz": "^0.17.0", "@lodestar/api": "^1.20.2", "@lodestar/config": "^1.20.2", "@lodestar/params": "^1.20.2", @@ -85,7 +85,7 @@ "mitt": "^3.0.0" }, "devDependencies": { - "@chainsafe/as-sha256": "^0.4.1", + "@chainsafe/as-sha256": "^0.5.0", "@types/qs": "^6.9.7", "fastify": "^4.27.0", "qs": "^6.11.1", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 0271f64b684f..891cc963c1bb 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -58,11 +58,11 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/as-sha256": "^0.4.1", + "@chainsafe/as-sha256": "^0.5.0", "@chainsafe/blst": "^2.0.3", - "@chainsafe/persistent-merkle-tree": "^0.7.1", + "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/ssz": "^0.17.0", "@lodestar/config": "^1.20.2", "@lodestar/params": "^1.20.2", "@lodestar/types": "^1.20.2", diff --git a/packages/types/package.json b/packages/types/package.json index d5512763b197..c7776c2a7004 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -70,7 +70,7 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/ssz": "^0.17.0", "@lodestar/params": "^1.20.2", "ethereum-cryptography": "^2.0.0" }, diff --git a/packages/utils/package.json b/packages/utils/package.json index e3ada3e2901b..e937dca9fbf8 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -39,7 +39,7 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/as-sha256": "^0.4.1", + "@chainsafe/as-sha256": "^0.5.0", "any-signal": "3.0.1", "bigint-buffer": "^1.1.5", "case": "^1.6.3", diff --git a/packages/validator/package.json b/packages/validator/package.json index abf5f8797d89..6801ce547265 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -46,7 +46,7 @@ ], "dependencies": { "@chainsafe/blst": "^2.0.3", - "@chainsafe/ssz": "^0.15.1", + "@chainsafe/ssz": "^0.17.0", "@lodestar/api": "^1.20.2", "@lodestar/config": "^1.20.2", "@lodestar/db": "^1.20.2", diff --git a/yarn.lock b/yarn.lock index efca10e5aa8c..f561a557351e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,6 +308,11 @@ resolved "https://registry.yarnpkg.com/@chainsafe/as-chacha20poly1305/-/as-chacha20poly1305-0.1.0.tgz#7da6f8796f9b42dac6e830a086d964f1f9189e09" integrity sha512-BpNcL8/lji/GM3+vZ/bgRWqJ1q5kwvTFmGPk7pxm/QQZDbaMI98waOHjEymTjq2JmdD/INdNBFOVSyJofXg7ew== +"@chainsafe/as-sha256@0.5.0", "@chainsafe/as-sha256@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.5.0.tgz#2523fbef2b80b5000f9aa71f4a76e5c2c5c076bb" + integrity sha512-dTIY6oUZNdC5yDTVP5Qc9hAlKAsn0QTQ2DnQvvsbTnKSTbYs3p5RPN0aIUqN0liXei/9h24c7V0dkV44cnWIQA== + "@chainsafe/as-sha256@^0.4.1": version "0.4.1" resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.4.1.tgz#cfc0737e25f8c206767bdb6703e7943e5d44513e" @@ -453,6 +458,30 @@ optionalDependencies: "@node-rs/crc32" "^1.6.0" +"@chainsafe/hashtree-darwin-arm64@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/hashtree-darwin-arm64/-/hashtree-darwin-arm64-1.0.1.tgz#e2c60090c56a1c8dc8bdff329856184ad32e4cd5" + integrity sha512-+KmEgQMpO7FDL3klAcpXbQ4DPZvfCe0qSaBBrtT4vLF8V1JGm3sp+j7oibtxtOsLKz7nJMiK1pZExi7vjXu8og== + +"@chainsafe/hashtree-linux-arm64-gnu@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/hashtree-linux-arm64-gnu/-/hashtree-linux-arm64-gnu-1.0.1.tgz#49d2604a6c9106219448af3eaf76f4da6e44daca" + integrity sha512-p1hnhGq2aFY+Zhdn1Q6L/6yLYNKjqXfn/Pc8jiM0e3+Lf/hB+yCdqYVu1pto26BrZjugCFZfupHaL4DjUTDttw== + +"@chainsafe/hashtree-linux-x64-gnu@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/hashtree-linux-x64-gnu/-/hashtree-linux-x64-gnu-1.0.1.tgz#31c5a2bb196b78f04f2bf4bfb5c1bf1f3331f071" + integrity sha512-uCIGuUWuWV0LiB4KLMy6JFa7Jp6NmPl3hKF5BYWu8TzUBe7vSXMZfqTzGxXPggFYN2/0KymfRdG9iDCOJfGRqg== + +"@chainsafe/hashtree@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/hashtree/-/hashtree-1.0.1.tgz#587666a261e1da6a37904095ce875fddc53c7c89" + integrity sha512-bleu9FjqBeR/l6W1u2Lz+HsS0b0LLJX2eUt3hOPBN7VqOhidx8wzkVh2S7YurS+iTQtfdK4K5QU9tcTGNrGwDg== + optionalDependencies: + "@chainsafe/hashtree-darwin-arm64" "1.0.1" + "@chainsafe/hashtree-linux-arm64-gnu" "1.0.1" + "@chainsafe/hashtree-linux-x64-gnu" "1.0.1" + "@chainsafe/is-ip@^2.0.1": version "2.0.1" resolved "https://registry.yarnpkg.com/@chainsafe/is-ip/-/is-ip-2.0.1.tgz#62cb285669d91f88fd9fa285048dde3882f0993b" @@ -530,6 +559,15 @@ dependencies: "@chainsafe/is-ip" "^2.0.1" +"@chainsafe/persistent-merkle-tree@0.8.0", "@chainsafe/persistent-merkle-tree@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.8.0.tgz#18e2f0a5de3a0b59c6e5be8797a78e0d209dd7dc" + integrity sha512-hh6C1JO6SKlr0QGNTNtTLqgGVMA/Bc20wD6CeMHp+wqbFKCULRJuBUxhF4WDx/7mX8QlqF3nFriF/Eo8oYJ4/A== + dependencies: + "@chainsafe/as-sha256" "0.5.0" + "@chainsafe/hashtree" "1.0.1" + "@noble/hashes" "^1.3.0" + "@chainsafe/persistent-merkle-tree@^0.6.1": version "0.6.1" resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.6.1.tgz#37bde25cf6cbe1660ad84311aa73157dc86ec7f2" @@ -538,14 +576,6 @@ "@chainsafe/as-sha256" "^0.4.1" "@noble/hashes" "^1.3.0" -"@chainsafe/persistent-merkle-tree@^0.7.1": - version "0.7.1" - resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.7.1.tgz#bfe6b3f4296ed3a578bb4fe69f9a7c232822a1dc" - integrity sha512-GUomb8DgkbHyKVBoLY9wMBe67oyAK9HKMjPImIocGOJuwqqxvDbVwh/ihdzyOrhEyhISqD/TxhCBDEXzLM52Vg== - dependencies: - "@chainsafe/as-sha256" "^0.4.1" - "@noble/hashes" "^1.3.0" - "@chainsafe/persistent-ts@^0.19.1": version "0.19.1" resolved "https://registry.npmjs.org/@chainsafe/persistent-ts/-/persistent-ts-0.19.1.tgz" @@ -564,13 +594,13 @@ "@chainsafe/as-sha256" "^0.4.1" "@chainsafe/persistent-merkle-tree" "^0.6.1" -"@chainsafe/ssz@^0.15.1": - version "0.15.1" - resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.15.1.tgz#008a711c3bcdc0d207cd4be15108870b0b1c60c0" - integrity sha512-f09UKTyYwWA1nr1BwrwsFpkXMspDDIZtwWXK1pM5mpPMnexmuPVstnN+P0M4YJ2aHcfqJXG7QOqnOwGj5Z7bUw== +"@chainsafe/ssz@^0.17.0": + version "0.17.0" + resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.17.0.tgz#b154260f6885693fd77bfb2fff43dd8c3fa37edd" + integrity sha512-DEzyH9vF4zz+Zqe2EMZuxXyxV5+7cmmLwljL3VC3ApzmyPORsprGJM7xLaUJu3oMRKMdBpR8UjRNkfB2ROQJzQ== dependencies: - "@chainsafe/as-sha256" "^0.4.1" - "@chainsafe/persistent-merkle-tree" "^0.7.1" + "@chainsafe/as-sha256" "0.5.0" + "@chainsafe/persistent-merkle-tree" "0.8.0" "@chainsafe/threads@^1.11.1": version "1.11.1" From 66636841507143c61212dcb5caf3ee2c3b2af362 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 9 Aug 2024 03:56:37 +0100 Subject: [PATCH 011/145] chore: post release updates (#7013) --- RELEASE.md | 2 +- packages/params/CHANGELOG.md | 19 ------------------- scripts/generate_changelog.mjs | 3 ++- 3 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 packages/params/CHANGELOG.md diff --git a/RELEASE.md b/RELEASE.md index 3379b40f3720..bb40b611ebe5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -191,7 +191,7 @@ Merge `stable` into `unstable`, resolving conflicts: - `git checkout unstable && git merge stable` - Resolve conflicts -- Sanity check locally before pushing by using: `git diff unstable origin/unstable` +- Sanity check locally before pushing by using: `git diff origin/unstable unstable` - Disable `unstable` branch protection - `git push` - Enable `unstable` branch protection diff --git a/packages/params/CHANGELOG.md b/packages/params/CHANGELOG.md deleted file mode 100644 index 20c296a0d5c7..000000000000 --- a/packages/params/CHANGELOG.md +++ /dev/null @@ -1,19 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -### [Unreleased] - -#### Breaking change - -- presets now export {params: IBeaconPreset} instead of {...IBeaconPreset} - -## [0.2.0] - -### Added - -- `RANDOM_SUBNETS_PER_VALIDATOR` and `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` constant params -- TODO: check what else is added diff --git a/scripts/generate_changelog.mjs b/scripts/generate_changelog.mjs index cd809622c5ea..de697e23612b 100644 --- a/scripts/generate_changelog.mjs +++ b/scripts/generate_changelog.mjs @@ -24,7 +24,7 @@ import fs from "node:fs"; const knownAuthors = { "caymannava@gmail.com": "wemeetagain", "develop@g11tech.io": "g11tech", - "tuyen@chainsafe.io": "tuyennhv", + "tuyen@chainsafe.io": "twoeths", "35266934+dapplion@users.noreply.github.com": "dapplion", "41898282+github-actions[bot]@users.noreply.github.com": "github-actions[bot]", "49699333+dependabot[bot]@users.noreply.github.com": "dependabot[bot]", @@ -42,6 +42,7 @@ const knownAuthors = { "nflaig@protonmail.com": "nflaig", "nazarhussain@gmail.com": "nazarhussain", "me@matthewkeil.com": "matthewkeil", + "17676176+ensi321@users.noreply.github.com": "ensi321", }; const fromTag = process.argv[2]; From 64fe1db806050d5d794ad8bdf83554faed84308c Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 9 Aug 2024 21:19:45 +0700 Subject: [PATCH 012/145] fix: improve sync pubkeys (#7010) * fix: improve sync pubkeys * fix: lint --- packages/state-transition/src/cache/epochCache.ts | 12 ++++++------ .../state-transition/src/cache/pubkeyCache.ts | 15 ++++++--------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 8f15dab90535..191aa7f3985c 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -264,12 +264,6 @@ export class EpochCache { {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { - // syncPubkeys here to ensure EpochCacheImmutableData is popualted before computing the rest of caches - // - computeSyncCommitteeCache() needs a fully populated pubkey2index cache - if (!opts?.skipSyncPubkeys) { - syncPubkeys(state, pubkey2index, index2pubkey); - } - const currentEpoch = computeEpochAtSlot(state.slot); const isGenesis = currentEpoch === GENESIS_EPOCH; const previousEpoch = isGenesis ? GENESIS_EPOCH : currentEpoch - 1; @@ -282,6 +276,12 @@ export class EpochCache { const validators = state.validators.getAllReadonlyValues(); const validatorCount = validators.length; + // syncPubkeys here to ensure EpochCacheImmutableData is popualted before computing the rest of caches + // - computeSyncCommitteeCache() needs a fully populated pubkey2index cache + if (!opts?.skipSyncPubkeys) { + syncPubkeys(validators, pubkey2index, index2pubkey); + } + const effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(validatorCount); const totalSlashingsByIncrement = getTotalSlashingsByIncrement(state); const previousActiveIndices: ValidatorIndex[] = []; diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index 0fd7a80fe990..b2b45ca09d25 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,6 +1,5 @@ import {PublicKey} from "@chainsafe/blst"; -import {ValidatorIndex} from "@lodestar/types"; -import {BeaconStateAllForks} from "./types.js"; +import {ValidatorIndex, phase0} from "@lodestar/types"; export type Index2PubkeyCache = PublicKey[]; @@ -53,7 +52,7 @@ export class PubkeyIndexMap { * If pubkey caches are empty: SLOW CODE - 🐢 */ export function syncPubkeys( - state: BeaconStateAllForks, + validators: phase0.Validator[], pubkey2index: PubkeyIndexMap, index2pubkey: Index2PubkeyCache ): void { @@ -61,16 +60,14 @@ export function syncPubkeys( throw new Error(`Pubkey indices have fallen out of sync: ${pubkey2index.size} != ${index2pubkey.length}`); } - // Get the validators sub tree once for all the loop - const validators = state.validators; - - const newCount = state.validators.length; + const newCount = validators.length; + index2pubkey.length = newCount; for (let i = pubkey2index.size; i < newCount; i++) { - const pubkey = validators.getReadonly(i).pubkey; + const pubkey = validators[i].pubkey; pubkey2index.set(pubkey, i); // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. // Afterwards any public key is the state consider validated. // > Do not do any validation here - index2pubkey.push(PublicKey.fromBytes(pubkey)); // Optimize for aggregation + index2pubkey[i] = PublicKey.fromBytes(pubkey); // Optimize for aggregation } } From 9c62011986a2b5d97ba81943e4d2169a1e94e880 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 13 Aug 2024 21:13:50 +0100 Subject: [PATCH 013/145] chore: update docker build actions (#7014) --- .github/workflows/publish-dev.yml | 6 +++--- .github/workflows/publish-rc.yml | 11 ++++++----- .github/workflows/publish-stable.yml | 11 ++++++----- Dockerfile | 4 ++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index fb4197b75da9..4e8c76e0dfdd 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -112,12 +112,12 @@ jobs: - uses: actions/checkout@v4 # https://github.com/docker/setup-qemu-action - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index f8f3b21ff349..936072de42c9 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -114,10 +114,11 @@ jobs: # In case of failure - name: Rollback on failure if: failure() - uses: author/action-rollback@9ec72a6af74774e00343c6de3e946b0901c23013 + uses: author/action-rollback@1.0.4 with: - id: ${{ steps.create_release.outputs.id }} + release_id: ${{ steps.create_release.outputs.id }} tag: ${{ needs.tag.outputs.tag }} + delete_orphan_tag: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -131,12 +132,12 @@ jobs: - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} rc 900 # https://github.com/docker/setup-qemu-action - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 9c41693f26f2..c2909a7e4e24 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -104,10 +104,11 @@ jobs: # In case of failure - name: Rollback on failure if: failure() - uses: author/action-rollback@9ec72a6af74774e00343c6de3e946b0901c23013 + uses: author/action-rollback@1.0.4 with: - id: ${{ steps.create_release.outputs.id }} + release_id: ${{ steps.create_release.outputs.id }} tag: ${{ needs.tag.outputs.tag }} + delete_orphan_tag: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -131,12 +132,12 @@ jobs: - run: scripts/await-release.sh ${{ needs.tag.outputs.tag }} latest 900 # https://github.com/docker/setup-qemu-action - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index e4b90c50bd20..0ee8083c85e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-slim as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-slim AS build_src ARG COMMIT WORKDIR /usr/app RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/* @@ -21,7 +21,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:22.4-slim as build_deps +FROM node:22.4-slim AS build_deps WORKDIR /usr/app RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/* From 44b215667efb60366f28f7b11c97c7c092a3dffc Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 14 Aug 2024 03:39:04 +0700 Subject: [PATCH 014/145] fix: reuse Buffer instance (#7016) * fix: improve message id to string conversion * fix: use shared Buffers for sszBytes util * feat: implement toRootHex() * fix: lint and check-types * Fix type checks * Add comment to vitest config * Update packages/utils/src/bytes.ts --------- Co-authored-by: Nico Flaig Co-authored-by: Cayman --- .../beacon-node/src/chain/balancesCache.ts | 4 +- .../src/chain/blocks/importBlock.ts | 10 ++-- .../chain/blocks/verifyBlocksSanityChecks.ts | 4 +- packages/beacon-node/src/chain/chain.ts | 18 ++++---- .../src/chain/errors/attestationError.ts | 4 +- .../src/chain/errors/blockError.ts | 7 ++- .../beacon-node/src/chain/forkChoice/index.ts | 16 +++---- packages/beacon-node/src/chain/initState.ts | 13 +++--- .../src/chain/lightClient/index.ts | 20 ++++---- .../opPools/aggregatedAttestationPool.ts | 5 +- .../beacon-node/src/chain/opPools/opPool.ts | 3 +- .../chain/opPools/syncCommitteeMessagePool.ts | 8 ++-- .../opPools/syncContributionAndProofPool.ts | 8 ++-- .../beacon-node/src/chain/regen/queued.ts | 7 ++- packages/beacon-node/src/chain/regen/regen.ts | 8 ++-- .../seenCache/seenCommitteeContribution.ts | 5 +- .../chain/seenCache/seenGossipBlockInput.ts | 9 ++-- .../beacon-node/src/chain/shufflingCache.ts | 5 +- .../chain/stateCache/blockStateCacheImpl.ts | 8 ++-- .../chain/stateCache/fifoBlockStateCache.ts | 6 +-- .../stateCache/inMemoryCheckpointsCache.ts | 9 ++-- .../stateCache/persistentCheckpointsCache.ts | 16 +++---- .../src/chain/validation/aggregateAndProof.ts | 4 +- .../src/chain/validation/attestation.ts | 12 ++--- .../beacon-node/src/chain/validation/block.ts | 7 ++- .../src/chain/validation/syncCommittee.ts | 4 +- .../beacon-node/src/eth1/utils/deposits.ts | 6 +-- .../src/network/gossip/encoding.ts | 9 +++- .../src/network/processor/gossipHandlers.ts | 9 ++-- .../reqresp/handlers/beaconBlocksByRoot.ts | 4 +- .../beacon-node/src/sync/backfill/backfill.ts | 25 +++++----- packages/beacon-node/src/sync/range/chain.ts | 5 +- packages/beacon-node/src/sync/unknownBlock.ts | 26 +++++------ packages/beacon-node/src/util/sszBytes.ts | 28 +++++++---- .../test/perf/network/gossip/encoding.test.ts | 46 +++++++++++++++++++ .../fork-choice/src/forkChoice/forkChoice.ts | 24 +++++----- packages/fork-choice/src/forkChoice/store.ts | 4 +- packages/utils/src/bytes.ts | 11 +++++ packages/utils/test/perf/bytes.test.ts | 27 +++++++++++ packages/utils/vitest.config.ts | 5 ++ 40 files changed, 268 insertions(+), 181 deletions(-) create mode 100644 packages/beacon-node/test/perf/network/gossip/encoding.test.ts create mode 100644 packages/utils/test/perf/bytes.test.ts diff --git a/packages/beacon-node/src/chain/balancesCache.ts b/packages/beacon-node/src/chain/balancesCache.ts index 5f4cf218c341..50a86b31b6c8 100644 --- a/packages/beacon-node/src/chain/balancesCache.ts +++ b/packages/beacon-node/src/chain/balancesCache.ts @@ -7,7 +7,7 @@ import { } from "@lodestar/state-transition"; import {CheckpointWithHex} from "@lodestar/fork-choice"; import {Epoch, RootHex} from "@lodestar/types"; -import {toHexString} from "@lodestar/utils"; +import {toRootHex} from "@lodestar/utils"; /** The number of validator balance sets that are cached within `CheckpointBalancesCache`. */ const MAX_BALANCE_CACHE_SIZE = 4; @@ -33,7 +33,7 @@ export class CheckpointBalancesCache { const epoch = state.epochCtx.epoch; const epochBoundarySlot = computeStartSlotAtEpoch(epoch); const epochBoundaryRoot = - epochBoundarySlot === state.slot ? blockRootHex : toHexString(getBlockRootAtSlot(state, epochBoundarySlot)); + epochBoundarySlot === state.slot ? blockRootHex : toRootHex(getBlockRootAtSlot(state, epochBoundarySlot)); const index = this.items.findIndex((item) => item.epoch === epoch && item.rootHex == epochBoundaryRoot); if (index === -1) { diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 9d46410a8638..360a3f8f5db9 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -10,7 +10,7 @@ import { } from "@lodestar/state-transition"; import {routes} from "@lodestar/api"; import {ForkChoiceError, ForkChoiceErrorCode, EpochDifference, AncestorStatus} from "@lodestar/fork-choice"; -import {isErrorAborted} from "@lodestar/utils"; +import {isErrorAborted, toRootHex} from "@lodestar/utils"; import {ZERO_HASH_HEX} from "../../constants/index.js"; import {toCheckpointHex} from "../stateCache/index.js"; import {isOptimisticBlock} from "../../util/forkChoice.js"; @@ -62,7 +62,7 @@ export async function importBlock( const {block, source} = blockInput; const {slot: blockSlot} = block.message; const blockRoot = this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message); - const blockRootHex = toHexString(blockRoot); + const blockRootHex = toRootHex(blockRoot); const currentEpoch = computeEpochAtSlot(this.forkChoice.getTime()); const blockEpoch = computeEpochAtSlot(blockSlot); const parentEpoch = computeEpochAtSlot(parentBlockSlot); @@ -123,7 +123,7 @@ export async function importBlock( const indexedAttestation = postState.epochCtx.getIndexedAttestation(attestation); const {target, beaconBlockRoot} = attestation.data; - const attDataRoot = toHexString(ssz.phase0.AttestationData.hashTreeRoot(indexedAttestation.data)); + const attDataRoot = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(indexedAttestation.data)); this.seenAggregatedAttestations.add( target.epoch, attDataRoot, @@ -371,9 +371,9 @@ export async function importBlock( const preFinalizedEpoch = parentBlockSummary.finalizedEpoch; if (finalizedEpoch > preFinalizedEpoch) { this.emitter.emit(routes.events.EventType.finalizedCheckpoint, { - block: toHexString(finalizedCheckpoint.root), + block: toRootHex(finalizedCheckpoint.root), epoch: finalizedCheckpoint.epoch, - state: toHexString(checkpointState.hashTreeRoot()), + state: toRootHex(checkpointState.hashTreeRoot()), executionOptimistic: false, }); this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint)); diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts index e62355a4889d..fcbcfea05d6e 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts @@ -2,7 +2,7 @@ import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {Slot} from "@lodestar/types"; -import {toHexString} from "@lodestar/utils"; +import {toHexString, toRootHex} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; import {BlockInput, ImportBlockOpts} from "./types.js"; @@ -67,7 +67,7 @@ export function verifyBlocksSanityChecks( parentBlockSlot = relevantBlocks[relevantBlocks.length - 1].block.message.slot; } else { // When importing a block segment, only the first NON-IGNORED block must be known to the fork-choice. - const parentRoot = toHexString(block.message.parentRoot); + const parentRoot = toRootHex(block.message.parentRoot); parentBlock = chain.forkChoice.getBlockHex(parentRoot); if (!parentBlock) { throw new BlockError(block, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot}); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index a12ee4a21f64..43f72d81e882 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import {CompositeTypeAny, fromHexString, TreeView, Type, toHexString} from "@chainsafe/ssz"; +import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz"; import { BeaconStateAllForks, CachedBeaconStateAllForks, @@ -35,7 +35,7 @@ import { } from "@lodestar/types"; import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice"; import {ProcessShutdownCallback} from "@lodestar/validator"; -import {Logger, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toHex} from "@lodestar/utils"; +import {Logger, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toHex, toRootHex} from "@lodestar/utils"; import {ForkSeq, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; @@ -592,7 +592,7 @@ export class BeaconChain implements IBeaconChain { async produceCommonBlockBody(blockAttributes: BlockAttributes): Promise { const {slot, parentBlockRoot} = blockAttributes; const state = await this.regen.getBlockSlotState( - toHexString(parentBlockRoot), + toRootHex(parentBlockRoot), slot, {dontTransferCache: true}, RegenCaller.produceBlock @@ -641,7 +641,7 @@ export class BeaconChain implements IBeaconChain { shouldOverrideBuilder?: boolean; }> { const state = await this.regen.getBlockSlotState( - toHexString(parentBlockRoot), + toRootHex(parentBlockRoot), slot, {dontTransferCache: true}, RegenCaller.produceBlock @@ -673,7 +673,7 @@ export class BeaconChain implements IBeaconChain { : this.config.getExecutionForkTypes(slot).BlindedBeaconBlockBody.hashTreeRoot(body as BlindedBeaconBlockBody); this.logger.debug("Computing block post state from the produced body", { slot, - bodyRoot: toHexString(bodyRoot), + bodyRoot: toRootHex(bodyRoot), blockType, }); @@ -1152,10 +1152,10 @@ export class BeaconChain implements IBeaconChain { const preState = this.regen.getPreStateSync(block); if (preState === null) { - throw Error(`Pre-state is unavailable given block's parent root ${toHexString(block.parentRoot)}`); + throw Error(`Pre-state is unavailable given block's parent root ${toRootHex(block.parentRoot)}`); } - const postState = this.regen.getStateSync(toHexString(block.stateRoot)) ?? undefined; + const postState = this.regen.getStateSync(toRootHex(block.stateRoot)) ?? undefined; return computeBlockRewards(block, preState.clone(), postState?.clone()); } @@ -1173,7 +1173,7 @@ export class BeaconChain implements IBeaconChain { } const {executionOptimistic, finalized} = stateResult; - const stateRoot = toHexString(stateResult.state.hashTreeRoot()); + const stateRoot = toRootHex(stateResult.state.hashTreeRoot()); const cachedState = this.regen.getStateSync(stateRoot); @@ -1193,7 +1193,7 @@ export class BeaconChain implements IBeaconChain { const preState = this.regen.getPreStateSync(block); if (preState === null) { - throw Error(`Pre-state is unavailable given block's parent root ${toHexString(block.parentRoot)}`); + throw Error(`Pre-state is unavailable given block's parent root ${toRootHex(block.parentRoot)}`); } return computeSyncCommitteeRewards(block, preState.clone(), validatorIds); diff --git a/packages/beacon-node/src/chain/errors/attestationError.ts b/packages/beacon-node/src/chain/errors/attestationError.ts index 8e0dc925f32e..fa0635bc0115 100644 --- a/packages/beacon-node/src/chain/errors/attestationError.ts +++ b/packages/beacon-node/src/chain/errors/attestationError.ts @@ -1,5 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; +import {toRootHex} from "@lodestar/utils"; import {GossipActionError} from "./gossipValidation.js"; export enum AttestationErrorCode { @@ -167,7 +167,7 @@ export class AttestationError extends GossipActionError { const type = this.type; switch (type.code) { case AttestationErrorCode.UNKNOWN_TARGET_ROOT: - return {code: type.code, root: toHexString(type.root)}; + return {code: type.code, root: toRootHex(type.root)}; case AttestationErrorCode.MISSING_STATE_TO_VERIFY_ATTESTATION: // TODO: The stack trace gets lost here return {code: type.code, error: type.error.message}; diff --git a/packages/beacon-node/src/chain/errors/blockError.ts b/packages/beacon-node/src/chain/errors/blockError.ts index 5f12bd939342..6280533c7a68 100644 --- a/packages/beacon-node/src/chain/errors/blockError.ts +++ b/packages/beacon-node/src/chain/errors/blockError.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {RootHex, SignedBeaconBlock, Slot, ValidatorIndex} from "@lodestar/types"; -import {LodestarError} from "@lodestar/utils"; +import {LodestarError, toRootHex} from "@lodestar/utils"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {ExecutionPayloadStatus} from "../../execution/engine/interface.js"; import {QueueErrorCode} from "../../util/queue/index.js"; @@ -151,8 +150,8 @@ export function renderBlockErrorType(type: BlockErrorType): Record { this.metrics?.lightclientServer.onSyncAggregate.inc({event: "processed"}); - const signedBlockRootHex = toHexString(signedBlockRoot); + const signedBlockRootHex = toRootHex(signedBlockRoot); const attestedData = this.prevHeadData.get(signedBlockRootHex); if (!attestedData) { // Log cacheSize since at start this.prevHeadData will be empty @@ -574,7 +574,7 @@ export class LightClientServer { } catch (e) { this.logger.error( "Error updating best LightClientUpdate", - {syncPeriod, slot: attestedHeader.beacon.slot, blockRoot: toHexString(attestedData.blockRoot)}, + {syncPeriod, slot: attestedHeader.beacon.slot, blockRoot: toRootHex(attestedData.blockRoot)}, e as Error ); } @@ -619,7 +619,7 @@ export class LightClientServer { const syncCommitteeWitness = await this.db.syncCommitteeWitness.get(attestedData.blockRoot); if (!syncCommitteeWitness) { - throw Error(`syncCommitteeWitness not available at ${toHexString(attestedData.blockRoot)}`); + throw Error(`syncCommitteeWitness not available at ${toRootHex(attestedData.blockRoot)}`); } const nextSyncCommittee = await this.db.syncCommittee.get(syncCommitteeWitness.nextSyncCommitteeRoot); if (!nextSyncCommittee) { @@ -697,7 +697,7 @@ export class LightClientServer { * Get finalized header from db. Keeps a small in-memory cache to speed up most of the lookups */ private async getFinalizedHeader(finalizedBlockRoot: Uint8Array): Promise { - const finalizedBlockRootHex = toHexString(finalizedBlockRoot); + const finalizedBlockRootHex = toRootHex(finalizedBlockRoot); const cachedFinalizedHeader = this.checkpointHeaders.get(finalizedBlockRootHex); if (cachedFinalizedHeader) { return cachedFinalizedHeader; diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 556e1f397d60..d896c9d3cc35 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {aggregateSignatures} from "@chainsafe/blst"; import {ForkName, ForkSeq, MAX_ATTESTATIONS, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0, Epoch, Slot, ssz, ValidatorIndex, RootHex} from "@lodestar/types"; @@ -11,7 +10,7 @@ import { getBlockRootAtSlot, } from "@lodestar/state-transition"; import {IForkChoice, EpochDifference} from "@lodestar/fork-choice"; -import {toHex, MapDef} from "@lodestar/utils"; +import {toHex, MapDef, toRootHex} from "@lodestar/utils"; import {intersectUint8Arrays, IntersectResult} from "../../util/bitArray.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; import {InsertOutcome} from "./types.js"; @@ -573,7 +572,7 @@ function isValidShuffling( // Otherwise the shuffling is determined by the block at the end of the target epoch // minus the shuffling lookahead (usually 2). We call this the "pivot". const pivotSlot = computeStartSlotAtEpoch(targetEpoch - 1) - 1; - const stateDependentRoot = toHexString(getBlockRootAtSlot(state, pivotSlot)); + const stateDependentRoot = toRootHex(getBlockRootAtSlot(state, pivotSlot)); // Use fork choice's view of the block DAG to quickly evaluate whether the attestation's // pivot block is the same as the current state's pivot block. If it is, then the diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index 69c331f6fd39..a991a6e8630a 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -16,6 +16,7 @@ import { ForkSeq, } from "@lodestar/params"; import {Epoch, phase0, capella, ssz, ValidatorIndex, SignedBeaconBlock} from "@lodestar/types"; +import {toRootHex} from "@lodestar/utils"; import {IBeaconDb} from "../../db/index.js"; import {SignedBLSToExecutionChangeVersioned} from "../../util/types.js"; import {BlockType} from "../interface.js"; @@ -135,7 +136,7 @@ export class OpPool { if (!rootHash) rootHash = ssz.phase0.AttesterSlashing.hashTreeRoot(attesterSlashing); // TODO: Do once and cache attached to the AttesterSlashing object const intersectingIndices = getAttesterSlashableIndices(attesterSlashing); - this.attesterSlashings.set(toHexString(rootHash), { + this.attesterSlashings.set(toRootHex(rootHash), { attesterSlashing, intersectingIndices, }); diff --git a/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts b/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts index 90a310841f01..bbaba1835dce 100644 --- a/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts +++ b/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts @@ -1,8 +1,8 @@ -import {BitArray, toHexString} from "@chainsafe/ssz"; +import {BitArray} from "@chainsafe/ssz"; import {Signature, aggregateSignatures} from "@chainsafe/blst"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {altair, Root, Slot, SubcommitteeIndex} from "@lodestar/types"; -import {MapDef} from "@lodestar/utils"; +import {MapDef, toRootHex} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; @@ -64,7 +64,7 @@ export class SyncCommitteeMessagePool { // TODO: indexInSubcommittee: number should be indicesInSyncCommittee add(subnet: Subnet, signature: altair.SyncCommitteeMessage, indexInSubcommittee: number): InsertOutcome { const {slot, beaconBlockRoot} = signature; - const rootHex = toHexString(beaconBlockRoot); + const rootHex = toRootHex(beaconBlockRoot); const lowestPermissibleSlot = this.lowestPermissibleSlot; // Reject if too old. @@ -99,7 +99,7 @@ export class SyncCommitteeMessagePool { * This is for the aggregator to produce ContributionAndProof. */ getContribution(subnet: SubcommitteeIndex, slot: Slot, prevBlockRoot: Root): altair.SyncCommitteeContribution | null { - const contribution = this.contributionsByRootBySubnetBySlot.get(slot)?.get(subnet)?.get(toHexString(prevBlockRoot)); + const contribution = this.contributionsByRootBySubnetBySlot.get(slot)?.get(subnet)?.get(toRootHex(prevBlockRoot)); if (!contribution) { return null; } diff --git a/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts b/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts index 7834ae534501..ff0feea891e1 100644 --- a/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts +++ b/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts @@ -1,9 +1,9 @@ -import {BitArray, toHexString} from "@chainsafe/ssz"; +import {BitArray} from "@chainsafe/ssz"; import {Signature, aggregateSignatures} from "@chainsafe/blst"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; import {altair, Slot, Root, ssz} from "@lodestar/types"; import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition"; -import {MapDef} from "@lodestar/utils"; +import {MapDef, toRootHex} from "@lodestar/utils"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; @@ -72,7 +72,7 @@ export class SyncContributionAndProofPool { add(contributionAndProof: altair.ContributionAndProof, syncCommitteeParticipants: number): InsertOutcome { const {contribution} = contributionAndProof; const {slot, beaconBlockRoot} = contribution; - const rootHex = toHexString(beaconBlockRoot); + const rootHex = toRootHex(beaconBlockRoot); // Reject if too old. if (slot < this.lowestPermissibleSlot) { @@ -100,7 +100,7 @@ export class SyncContributionAndProofPool { * This is for the block factory, the same to process_sync_committee_contributions in the spec. */ getAggregate(slot: Slot, prevBlockRoot: Root): altair.SyncAggregate { - const bestContributionBySubnet = this.bestContributionBySubnetRootBySlot.get(slot)?.get(toHexString(prevBlockRoot)); + const bestContributionBySubnet = this.bestContributionBySubnetRootBySlot.get(slot)?.get(toRootHex(prevBlockRoot)); if (!bestContributionBySubnet || bestContributionBySubnet.size === 0) { // TODO: Add metric for missing SyncAggregate // Must return signature as G2_POINT_AT_INFINITY when participating bits are empty diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index ad673b334bd1..082ffbe271e4 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -1,8 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {phase0, Slot, RootHex, Epoch, BeaconBlock} from "@lodestar/types"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; -import {Logger} from "@lodestar/utils"; +import {Logger, toRootHex} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {CheckpointHex, toCheckpointHex} from "../stateCache/index.js"; import {Metrics} from "../../metrics/index.js"; @@ -89,7 +88,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { block: BeaconBlock, opts: StateCloneOpts = {dontTransferCache: true} ): CachedBeaconStateAllForks | null { - const parentRoot = toHexString(block.parentRoot); + const parentRoot = toRootHex(block.parentRoot); const parentBlock = this.forkChoice.getBlockHex(parentRoot); if (!parentBlock) { throw new RegenError({ @@ -167,7 +166,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void { const {stateRoot: newHeadStateRoot, blockRoot: newHeadBlockRoot, slot: newHeadSlot} = newHead; - const maybeHeadStateRoot = toHexString(maybeHeadState.hashTreeRoot()); + const maybeHeadStateRoot = toRootHex(maybeHeadState.hashTreeRoot()); const logCtx = { newHeadSlot, newHeadBlockRoot, diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 2b6fc835cf7c..409c12c77b21 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -1,4 +1,4 @@ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {phase0, Slot, RootHex, BeaconBlock} from "@lodestar/types"; import { CachedBeaconStateAllForks, @@ -10,7 +10,7 @@ import { stateTransition, } from "@lodestar/state-transition"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {Logger} from "@lodestar/utils"; +import {Logger, toRootHex} from "@lodestar/utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; import {Metrics} from "../../metrics/index.js"; @@ -89,7 +89,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { allowDiskReload = false ): Promise { const checkpointStartSlot = computeStartSlotAtEpoch(cp.epoch); - return this.getBlockSlotState(toHexString(cp.root), checkpointStartSlot, opts, regenCaller, allowDiskReload); + return this.getBlockSlotState(toRootHex(cp.root), checkpointStartSlot, opts, regenCaller, allowDiskReload); } /** @@ -224,7 +224,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { this.modules.metrics ); - const stateRoot = toHexString(state.hashTreeRoot()); + const stateRoot = toRootHex(state.hashTreeRoot()); if (b.stateRoot !== stateRoot) { throw new RegenError({ slot: b.slot, diff --git a/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts b/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts index fedaff8225d6..86ade618b1d1 100644 --- a/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts +++ b/packages/beacon-node/src/chain/seenCache/seenCommitteeContribution.ts @@ -1,7 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {Slot, ValidatorIndex} from "@lodestar/types"; import {ContributionAndProof, SyncCommitteeContribution} from "@lodestar/types/altair"; -import {MapDef} from "@lodestar/utils"; +import {MapDef, toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {isSuperSetOrEqual} from "../../util/bitArray.js"; import {AggregationInfo, insertDesc} from "./seenAggregateAndProof.js"; @@ -101,5 +100,5 @@ function seenAggregatorKey(subcommitteeIndex: number, aggregatorIndex: Validator function toContributionDataKey(contribution: SyncCommitteeContribution): ContributionDataKey { const {slot, beaconBlockRoot, subcommitteeIndex} = contribution; - return `${slot} - ${toHexString(beaconBlockRoot)} - ${subcommitteeIndex}`; + return `${slot} - ${toRootHex(beaconBlockRoot)} - ${subcommitteeIndex}`; } diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 6b51332353f2..3806668436d8 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -1,7 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {deneb, RootHex, SignedBeaconBlock, ssz} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; -import {pruneSetToMax} from "@lodestar/utils"; +import {pruneSetToMax, toRootHex} from "@lodestar/utils"; import {BLOBSIDECAR_FIXED_SIZE, isForkBlobs, ForkName} from "@lodestar/params"; import { @@ -81,9 +80,7 @@ export class SeenGossipBlockInput { const {signedBlock, blockBytes} = gossipedInput; fork = config.getForkName(signedBlock.message.slot); - blockHex = toHexString( - config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message) - ); + blockHex = toRootHex(config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message)); blockCache = this.blockInputCache.get(blockHex) ?? getEmptyBlockInputCacheEntry(fork); blockCache.block = signedBlock; @@ -93,7 +90,7 @@ export class SeenGossipBlockInput { const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blobSidecar.signedBlockHeader.message); fork = config.getForkName(blobSidecar.signedBlockHeader.message.slot); - blockHex = toHexString(blockRoot); + blockHex = toRootHex(blockRoot); blockCache = this.blockInputCache.get(blockHex) ?? getEmptyBlockInputCacheEntry(fork); // TODO: freetheblobs check if its the same blob or a duplicate and throw/take actions diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 23177142d846..12dd1bf3e9ae 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -1,7 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks, EpochShuffling, getShufflingDecisionBlock} from "@lodestar/state-transition"; import {Epoch, RootHex, ssz} from "@lodestar/types"; -import {MapDef, pruneSetToMax} from "@lodestar/utils"; +import {MapDef, pruneSetToMax, toRootHex} from "@lodestar/utils"; import {GENESIS_SLOT} from "@lodestar/params"; import {Metrics} from "../metrics/metrics.js"; import {computeAnchorCheckpoint} from "./initState.js"; @@ -206,5 +205,5 @@ function isPromiseCacheItem(item: CacheItem): item is PromiseCacheItem { function getDecisionBlock(state: CachedBeaconStateAllForks, epoch: Epoch): RootHex { return state.slot > GENESIS_SLOT ? getShufflingDecisionBlock(state, epoch) - : toHexString(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(state.config, state).blockHeader)); + : toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(state.config, state).blockHeader)); } diff --git a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts index fdeb3ed5a659..05073d0e515f 100644 --- a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts +++ b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts @@ -1,7 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {Epoch, RootHex} from "@lodestar/types"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {routes} from "@lodestar/api"; +import {toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {StateCloneOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; @@ -53,7 +53,7 @@ export class BlockStateCacheImpl implements BlockStateCache { } add(item: CachedBeaconStateAllForks): void { - const key = toHexString(item.hashTreeRoot()); + const key = toRootHex(item.hashTreeRoot()); if (this.cache.get(key)) { return; } @@ -70,7 +70,7 @@ export class BlockStateCacheImpl implements BlockStateCache { setHeadState(item: CachedBeaconStateAllForks | null): void { if (item) { - const key = toHexString(item.hashTreeRoot()); + const key = toRootHex(item.hashTreeRoot()); this.head = {state: item, stateRoot: key}; } else { this.head = null; @@ -130,7 +130,7 @@ export class BlockStateCacheImpl implements BlockStateCache { dumpSummary(): routes.lodestar.StateCacheItem[] { return Array.from(this.cache.entries()).map(([key, state]) => ({ slot: state.slot, - root: toHexString(state.hashTreeRoot()), + root: toRootHex(state.hashTreeRoot()), reads: this.cache.readCount.get(key) ?? 0, lastRead: this.cache.lastRead.get(key) ?? 0, checkpointState: false, diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 93b581633c05..602b0abaee8d 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -1,7 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {RootHex} from "@lodestar/types"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {routes} from "@lodestar/api"; +import {toRootHex} from "@lodestar/utils"; import {Metrics} from "../../metrics/index.js"; import {LinkedList} from "../../util/array.js"; import {StateCloneOpts} from "../regen/interface.js"; @@ -107,7 +107,7 @@ export class FIFOBlockStateCache implements BlockStateCache { * In importBlock() steps, normally it'll call add() with isHead = false first. Then call setHeadState() to set the head. */ add(item: CachedBeaconStateAllForks, isHead = false): void { - const key = toHexString(item.hashTreeRoot()); + const key = toRootHex(item.hashTreeRoot()); if (this.cache.get(key) != null) { if (!this.keyOrder.has(key)) { throw Error(`State exists but key not found in keyOrder: ${key}`); @@ -183,7 +183,7 @@ export class FIFOBlockStateCache implements BlockStateCache { dumpSummary(): routes.lodestar.StateCacheItem[] { return Array.from(this.cache.entries()).map(([key, state]) => ({ slot: state.slot, - root: toHexString(state.hashTreeRoot()), + root: toRootHex(state.hashTreeRoot()), reads: this.cache.readCount.get(key) ?? 0, lastRead: this.cache.lastRead.get(key) ?? 0, checkpointState: false, diff --git a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts index 37c67c0d86b4..03cdc84de166 100644 --- a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts @@ -1,7 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {phase0, Epoch, RootHex} from "@lodestar/types"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; -import {MapDef} from "@lodestar/utils"; +import {MapDef, toRootHex} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {Metrics} from "../../metrics/index.js"; import {StateCloneOpts} from "../regen/interface.js"; @@ -144,7 +143,7 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { delete(cp: phase0.Checkpoint): void { this.cache.delete(toCheckpointKey(toCheckpointHex(cp))); - const epochKey = toHexString(cp.root); + const epochKey = toRootHex(cp.root); const value = this.epochIndex.get(cp.epoch); if (value) { value.delete(epochKey); @@ -170,7 +169,7 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { dumpSummary(): routes.lodestar.StateCacheItem[] { return Array.from(this.cache.entries()).map(([key, state]) => ({ slot: state.slot, - root: toHexString(state.hashTreeRoot()), + root: toRootHex(state.hashTreeRoot()), reads: this.cache.readCount.get(key) ?? 0, lastRead: this.cache.lastRead.get(key) ?? 0, checkpointState: true, @@ -186,7 +185,7 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex { return { epoch: checkpoint.epoch, - rootHex: toHexString(checkpoint.root), + rootHex: toRootHex(checkpoint.root), }; } diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 58aeca061bc0..5c5901583ad8 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -1,7 +1,7 @@ import {fromHexString, toHexString} from "@chainsafe/ssz"; import {phase0, Epoch, RootHex} from "@lodestar/types"; import {CachedBeaconStateAllForks, computeStartSlotAtEpoch, getBlockRootAtSlot} from "@lodestar/state-transition"; -import {Logger, MapDef, sleep} from "@lodestar/utils"; +import {Logger, MapDef, sleep, toRootHex} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {loadCachedBeaconState} from "@lodestar/state-transition"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; @@ -171,7 +171,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { for (const persistedKey of persistedKeys) { const cp = datastoreKeyToCheckpoint(persistedKey); this.cache.set(toCacheKey(cp), {type: CacheItemType.persisted, value: persistedKey}); - this.epochIndex.getOrDefault(cp.epoch).add(toHexString(cp.root)); + this.epochIndex.getOrDefault(cp.epoch).add(toRootHex(cp.root)); } this.logger.info("Loaded persisted checkpoint states from the last run", { count: persistedKeys.length, @@ -227,7 +227,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { validatorsBytes ); newCachedState.commit(); - const stateRoot = toHexString(newCachedState.hashTreeRoot()); + const stateRoot = toRootHex(newCachedState.hashTreeRoot()); timer?.(); this.logger.debug("Reload: cached state load successful", { ...logMeta, @@ -561,7 +561,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // amongst states of the same epoch, choose the one with the same view of reloadedCp if ( reloadedCpSlot < state.slot && - toHexString(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex + toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex ) { return state; } @@ -645,8 +645,8 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const epochBoundarySlot = computeStartSlotAtEpoch(epoch); const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHexString(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot); - const epochBoundaryHex = toHexString(epochBoundaryRoot); - const prevEpochRoot = toHexString(getBlockRootAtSlot(state, epochBoundarySlot - 1)); + const epochBoundaryHex = toRootHex(epochBoundaryRoot); + const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1)); // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State const cpRootHexes = this.epochIndex.get(epoch) ?? []; @@ -804,7 +804,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex { return { epoch: checkpoint.epoch, - rootHex: toHexString(checkpoint.root), + rootHex: toRootHex(checkpoint.root), }; } @@ -812,7 +812,7 @@ function toCacheKey(cp: CheckpointHex | phase0.Checkpoint): CacheKey { if (isCheckpointHex(cp)) { return `${cp.rootHex}_${cp.epoch}`; } - return `${toHexString(cp.root)}_${cp.epoch}`; + return `${toRootHex(cp.root)}_${cp.epoch}`; } function fromCacheKey(key: CacheKey): CheckpointHex { diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 430464683493..3c32ffe1a9ec 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {ForkName} from "@lodestar/params"; import {phase0, RootHex, ssz} from "@lodestar/types"; import { @@ -6,6 +5,7 @@ import { isAggregatorFromCommitteeLength, createAggregateSignatureSetFromComponents, } from "@lodestar/state-transition"; +import {toRootHex} from "@lodestar/utils"; import {IBeaconChain} from ".."; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {RegenCaller} from "../regen/index.js"; @@ -111,7 +111,7 @@ async function validateAggregateAndProof( // is a non-strict superset has _not_ already been seen. const attDataRootHex = cachedAttData ? cachedAttData.attDataRootHex - : toHexString(ssz.phase0.AttestationData.hashTreeRoot(attData)); + : toRootHex(ssz.phase0.AttestationData.hashTreeRoot(attData)); if ( !skipValidationKnownAttesters && chain.seenAggregatedAttestations.isKnown(targetEpoch, attDataRootHex, aggregationBits) diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index fc39534b45e6..1116e87e1d25 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {phase0, Epoch, Root, Slot, RootHex, ssz} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq, DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; @@ -13,6 +12,7 @@ import { computeSigningRoot, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; +import {toRootHex} from "@lodestar/utils"; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC} from "../../constants/index.js"; import {RegenCaller} from "../regen/index.js"; @@ -437,7 +437,7 @@ async function validateGossipAttestationNoSignatureCheck( ); // add cached attestation data before verifying signature - attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attData)); + attDataRootHex = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(attData)); if (attDataBase64) { chain.seenAttestationDatas.add(attSlot, attDataBase64, { committeeIndices, @@ -643,7 +643,7 @@ function verifyHeadBlockIsKnown(chain: IBeaconChain, beaconBlockRoot: Root): Pro if (headBlock === null) { throw new AttestationError(GossipAction.IGNORE, { code: AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT, - root: toHexString(beaconBlockRoot), + root: toRootHex(beaconBlockRoot), }); } @@ -671,7 +671,7 @@ function verifyAttestationTargetRoot(headBlock: ProtoBlock, targetRoot: Root, at // https://github.com/ethereum/consensus-specs/pull/2001#issuecomment-699246659 throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_TARGET_ROOT, - targetRoot: toHexString(targetRoot), + targetRoot: toRootHex(targetRoot), expected: null, }); } else { @@ -687,11 +687,11 @@ function verifyAttestationTargetRoot(headBlock: ProtoBlock, targetRoot: Root, at headBlock.blockRoot; // TODO: Do a fast comparision to convert and compare byte by byte - if (expectedTargetRoot !== toHexString(targetRoot)) { + if (expectedTargetRoot !== toRootHex(targetRoot)) { // Reject any attestation with an invalid target root. throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_TARGET_ROOT, - targetRoot: toHexString(targetRoot), + targetRoot: toRootHex(targetRoot), expected: expectedTargetRoot, }); } diff --git a/packages/beacon-node/src/chain/validation/block.ts b/packages/beacon-node/src/chain/validation/block.ts index 214eeaf0ab4e..aabc1b14958a 100644 --- a/packages/beacon-node/src/chain/validation/block.ts +++ b/packages/beacon-node/src/chain/validation/block.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import { computeStartSlotAtEpoch, @@ -8,7 +7,7 @@ import { isExecutionEnabled, getBlockProposerSignatureSet, } from "@lodestar/state-transition"; -import {sleep} from "@lodestar/utils"; +import {sleep, toRootHex} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; import {SignedBeaconBlock} from "@lodestar/types"; import {MAXIMUM_GOSSIP_CLOCK_DISPARITY} from "../../constants/index.js"; @@ -55,7 +54,7 @@ export async function validateGossipBlock( // reboot if the `observed_block_producers` cache is empty. In that case, without this // check, we will load the parent and state from disk only to find out later that we // already know this block. - const blockRoot = toHexString(config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block)); + const blockRoot = toRootHex(config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block)); if (chain.forkChoice.getBlockHex(blockRoot) !== null) { throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot}); } @@ -71,7 +70,7 @@ export async function validateGossipBlock( // [REJECT] The current finalized_checkpoint is an ancestor of block -- i.e. // get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root - const parentRoot = toHexString(block.parentRoot); + const parentRoot = toRootHex(block.parentRoot); const parentBlock = chain.forkChoice.getBlockHex(parentRoot); if (parentBlock === null) { // If fork choice does *not* consider the parent to be a descendant of the finalized block, diff --git a/packages/beacon-node/src/chain/validation/syncCommittee.ts b/packages/beacon-node/src/chain/validation/syncCommittee.ts index 43a4c95c59da..f47aa53a314e 100644 --- a/packages/beacon-node/src/chain/validation/syncCommittee.ts +++ b/packages/beacon-node/src/chain/validation/syncCommittee.ts @@ -1,7 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {SYNC_COMMITTEE_SUBNET_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {altair} from "@lodestar/types"; +import {toRootHex} from "@lodestar/utils"; import {GossipAction, SyncCommitteeError, SyncCommitteeErrorCode} from "../errors/index.js"; import {IBeaconChain} from "../interface.js"; import {getSyncCommitteeSignatureSet} from "./signatureSets/index.js"; @@ -17,7 +17,7 @@ export async function validateGossipSyncCommittee( subnet: number ): Promise<{indexInSubcommittee: IndexInSubcommittee}> { const {slot, validatorIndex, beaconBlockRoot} = syncCommittee; - const messageRoot = toHexString(beaconBlockRoot); + const messageRoot = toRootHex(beaconBlockRoot); const headState = chain.getHeadState(); const indexInSubcommittee = validateGossipSyncCommitteeExceptSig(chain, headState, subnet, syncCommittee); diff --git a/packages/beacon-node/src/eth1/utils/deposits.ts b/packages/beacon-node/src/eth1/utils/deposits.ts index 19544917ffdc..24916264e6d8 100644 --- a/packages/beacon-node/src/eth1/utils/deposits.ts +++ b/packages/beacon-node/src/eth1/utils/deposits.ts @@ -1,9 +1,9 @@ import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; -import {toHexString} from "@chainsafe/ssz"; import {MAX_DEPOSITS} from "@lodestar/params"; import {BeaconStateAllForks} from "@lodestar/state-transition"; import {phase0, ssz} from "@lodestar/types"; import {FilterOptions} from "@lodestar/db"; +import {toRootHex} from "@lodestar/utils"; import {Eth1Error, Eth1ErrorCode} from "../errors.js"; import {DepositTree} from "../../db/repositories/depositDataRoot.js"; @@ -51,8 +51,8 @@ export function getDepositsWithProofs( if (!ssz.Root.equals(depositRoot, eth1Data.depositRoot)) { throw new Eth1Error({ code: Eth1ErrorCode.WRONG_DEPOSIT_ROOT, - root: toHexString(depositRoot), - expectedRoot: toHexString(eth1Data.depositRoot), + root: toRootHex(depositRoot), + expectedRoot: toRootHex(eth1Data.depositRoot), }); } diff --git a/packages/beacon-node/src/network/gossip/encoding.ts b/packages/beacon-node/src/network/gossip/encoding.ts index 53847f56df7d..02c0df07b2f1 100644 --- a/packages/beacon-node/src/network/gossip/encoding.ts +++ b/packages/beacon-node/src/network/gossip/encoding.ts @@ -4,7 +4,7 @@ import {Message} from "@libp2p/interface"; import {digest} from "@chainsafe/as-sha256"; import {RPC} from "@chainsafe/libp2p-gossipsub/message"; import {DataTransform} from "@chainsafe/libp2p-gossipsub/types"; -import {intToBytes, toHex} from "@lodestar/utils"; +import {intToBytes} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js"; import {getGossipSSZType, GossipTopicCache} from "./topic.js"; @@ -15,6 +15,9 @@ const xxhash = await xxhashFactory(); // Use salt to prevent msgId from being mined for collisions const h64Seed = BigInt(Math.floor(Math.random() * 1e9)); +// Shared buffer to convert msgId to string +const sharedMsgIdBuf = Buffer.alloc(20); + /** * The function used to generate a gossipsub message id * We use the first 8 bytes of SHA256(data) for content addressing @@ -28,7 +31,9 @@ export function fastMsgIdFn(rpcMsg: RPC.Message): string { } export function msgIdToStrFn(msgId: Uint8Array): string { - return toHex(msgId); + // this is the same logic to `toHex(msgId)` with better performance + sharedMsgIdBuf.set(msgId); + return `0x${sharedMsgIdBuf.toString("hex")}`; } /** diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 82fe7d8db358..6f64148b87e2 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; -import {LogLevel, Logger, prettyBytes} from "@lodestar/utils"; +import {LogLevel, Logger, prettyBytes, toRootHex} from "@lodestar/utils"; import {Root, Slot, ssz, deneb, UintNum64, SignedBeaconBlock} from "@lodestar/types"; import {ForkName, ForkSeq} from "@lodestar/params"; import {routes} from "@lodestar/api"; @@ -298,7 +297,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler case BlockErrorCode.DATA_UNAVAILABLE: { const slot = signedBlock.message.slot; const forkTypes = config.getForkTypes(slot); - const rootHex = toHexString(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); + const rootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); events.emit(NetworkEvent.unknownBlock, {rootHex, peer: peerIdStr}); @@ -747,12 +746,12 @@ export async function validateGossipFnRetryUnknownRoot( ) { if (unknownBlockRootRetries === 0) { // Trigger unknown block root search here - const rootHex = toHexString(blockRoot); + const rootHex = toRootHex(blockRoot); network.searchUnknownSlotRoot({slot, root: rootHex}); } if (unknownBlockRootRetries++ < MAX_UNKNOWN_BLOCK_ROOT_RETRIES) { - const foundBlock = await chain.waitForBlock(slot, toHexString(blockRoot)); + const foundBlock = await chain.waitForBlock(slot, toRootHex(blockRoot)); // Returns true if the block was found on time. In that case, try to get it from the fork-choice again. // Otherwise, throw the error below. if (foundBlock) { diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRoot.ts index 0ed0e6a2d185..36d90256276e 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRoot.ts @@ -1,6 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {ResponseOutgoing} from "@lodestar/reqresp"; import {Slot, phase0} from "@lodestar/types"; +import {toRootHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; import {getSlotFromSignedBeaconBlockSerialized} from "../../../util/sszBytes.js"; @@ -33,7 +33,7 @@ export async function* onBeaconBlocksByRoot( if (slot === undefined) { const slotFromBytes = getSlotFromSignedBeaconBlockSerialized(blockBytes); if (slotFromBytes === null) { - throw Error(`Invalid block bytes for block root ${toHexString(root)}`); + throw Error(`Invalid block bytes for block root ${toRootHex(root)}`); } slot = slotFromBytes; } diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index 6d9716a37329..15ba34f4e583 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -1,10 +1,9 @@ import {EventEmitter} from "events"; import {StrictEventEmitter} from "strict-event-emitter-types"; -import {toHexString} from "@chainsafe/ssz"; import {BeaconStateAllForks, blockToHeader} from "@lodestar/state-transition"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {phase0, Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; -import {ErrorAborted, Logger, sleep, toHex} from "@lodestar/utils"; +import {ErrorAborted, Logger, sleep, toHex, toRootHex} from "@lodestar/utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {IBeaconChain} from "../../chain/index.js"; @@ -251,7 +250,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} lte: backfillStartFromSlot, }); modules.logger.info("Initializing from Checkpoint", { - root: toHexString(anchorCp.root), + root: toRootHex(anchorCp.root), epoch: anchorCp.epoch, backfillStartFromSlot, previousBackfilledRanges: JSON.stringify(previousBackfilledRanges), @@ -327,7 +326,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} this.logger.error( `Backfilled till ${ this.syncAnchor.lastBackSyncedBlock.slot - } but not found previous saved finalized or wsCheckpoint with root=${toHexString( + } but not found previous saved finalized or wsCheckpoint with root=${toRootHex( this.prevFinalizedCheckpointBlock.root )}, slot=${this.prevFinalizedCheckpointBlock.slot}` ); @@ -341,7 +340,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} this.logger.error( `Invalid root synced at a previous finalized or wsCheckpoint, slot=${ this.prevFinalizedCheckpointBlock.slot - }: expected=${toHexString(this.prevFinalizedCheckpointBlock.root)}, actual=${toHexString( + }: expected=${toRootHex(this.prevFinalizedCheckpointBlock.root)}, actual=${toRootHex( this.syncAnchor.lastBackSyncedBlock.root )}` ); @@ -349,7 +348,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} break; } this.logger.verbose("Validated current prevFinalizedCheckpointBlock", { - root: toHexString(this.prevFinalizedCheckpointBlock.root), + root: toRootHex(this.prevFinalizedCheckpointBlock.root), slot: this.prevFinalizedCheckpointBlock.slot, }); @@ -379,7 +378,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} if (this.syncAnchor.lastBackSyncedBlock.slot === GENESIS_SLOT) { if (!byteArrayEquals(this.syncAnchor.lastBackSyncedBlock.block.message.parentRoot, ZERO_HASH)) { Error( - `Invalid Gensis Block with non zero parentRoot=${toHexString( + `Invalid Gensis Block with non zero parentRoot=${toRootHex( this.syncAnchor.lastBackSyncedBlock.block.message.parentRoot )}` ); @@ -537,7 +536,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} }` ); this.logger.info("wsCheckpoint validated!", { - root: toHexString(this.wsCheckpointHeader.root), + root: toRootHex(this.wsCheckpointHeader.root), epoch: this.wsCheckpointHeader.slot / SLOTS_PER_EPOCH, }); this.wsValidated = true; @@ -584,7 +583,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} this.prevFinalizedCheckpointBlock.slot === prevBackfillCpBlock.message.slot ) { this.logger.verbose("Validated current prevFinalizedCheckpointBlock", { - root: toHexString(this.prevFinalizedCheckpointBlock.root), + root: toRootHex(this.prevFinalizedCheckpointBlock.root), slot: prevBackfillCpBlock.message.slot, }); } else { @@ -689,7 +688,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} throw Error( `Skipped a prevFinalizedCheckpointBlock with slot=${toHex( this.prevFinalizedCheckpointBlock.root - )}, root=${toHexString(this.prevFinalizedCheckpointBlock.root)}` + )}, root=${toRootHex(this.prevFinalizedCheckpointBlock.root)}` ); } if (anchorBlock.message.slot === this.prevFinalizedCheckpointBlock.slot) { @@ -700,7 +699,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} throw Error( `Invalid root for prevFinalizedCheckpointBlock at slot=${ this.prevFinalizedCheckpointBlock.slot - }, expected=${toHexString(this.prevFinalizedCheckpointBlock.root)}, found=${toHex(anchorBlockRoot)}` + }, expected=${toRootHex(this.prevFinalizedCheckpointBlock.root)}, found=${toHex(anchorBlockRoot)}` ); } @@ -770,7 +769,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.blockbyroot}); this.logger.verbose("Fetched new anchorBlock", { - root: toHexString(anchorBlockRoot), + root: toRootHex(anchorBlockRoot), slot: anchorBlock.data.message.slot, }); @@ -883,7 +882,7 @@ async function extractPreviousFinOrWsCheckpoint( const root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(header); prevFinalizedCheckpointBlock = {root, slot: nextPrevFinOrWsBlock.message.slot}; logger?.debug("Extracted new prevFinalizedCheckpointBlock as potential previous finalized or wsCheckpoint", { - root: toHexString(prevFinalizedCheckpointBlock.root), + root: toRootHex(prevFinalizedCheckpointBlock.root), slot: prevFinalizedCheckpointBlock.slot, }); } else { diff --git a/packages/beacon-node/src/sync/range/chain.ts b/packages/beacon-node/src/sync/range/chain.ts index 41bbce3da820..0a2fc1a3f7c3 100644 --- a/packages/beacon-node/src/sync/range/chain.ts +++ b/packages/beacon-node/src/sync/range/chain.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {Epoch, Root, Slot, phase0} from "@lodestar/types"; -import {ErrorAborted, Logger} from "@lodestar/utils"; +import {ErrorAborted, Logger, toRootHex} from "@lodestar/utils"; import {ChainForkConfig} from "@lodestar/config"; import {BlockInput, BlockInputType} from "../../chain/blocks/types.js"; import {PeerAction} from "../../network/index.js"; @@ -234,7 +233,7 @@ export class SyncChain { /** Full debug state for lodestar API */ getDebugState(): SyncChainDebugState { return { - targetRoot: toHexString(this.target.root), + targetRoot: toRootHex(this.target.root), targetSlot: this.target.slot, syncType: this.syncType, status: this.status, diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index 3c15b32eb8d8..d985847d450f 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -1,6 +1,6 @@ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {Logger, pruneSetToMax} from "@lodestar/utils"; +import {Logger, pruneSetToMax, toRootHex} from "@lodestar/utils"; import {Root, RootHex, deneb} from "@lodestar/types"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; @@ -139,8 +139,8 @@ export class UnknownBlockSync { private addUnknownParent(blockInput: BlockInput, peerIdStr: string): void { const block = blockInput.block.message; const blockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block); - const blockRootHex = toHexString(blockRoot); - const parentBlockRootHex = toHexString(block.parentRoot); + const blockRootHex = toRootHex(blockRoot); + const parentBlockRootHex = toRootHex(block.parentRoot); // add 1 pending block with status downloaded let pendingBlock = this.pendingBlocks.get(blockRootHex); @@ -180,9 +180,7 @@ export class UnknownBlockSync { } else { if (blockInputOrRootHex.block !== null) { const {block} = blockInputOrRootHex; - blockRootHex = toHexString( - this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message) - ); + blockRootHex = toRootHex(this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message)); unknownBlockType = PendingBlockType.UNKNOWN_BLOBS; } else { unknownBlockType = PendingBlockType.UNKNOWN_BLOCKINPUT; @@ -304,7 +302,7 @@ export class UnknownBlockSync { ...block, status: PendingBlockStatus.downloaded, blockInput, - parentBlockRootHex: toHexString(blockInput.block.message.parentRoot), + parentBlockRootHex: toRootHex(blockInput.block.message.parentRoot), }; this.pendingBlocks.set(block.blockRootHex, block); const blockSlot = blockInput.block.message.slot; @@ -336,7 +334,7 @@ export class UnknownBlockSync { this.logger.debug("Downloaded block is before finalized slot", { finalizedSlot, blockSlot, - parentRoot: toHexString(blockRoot), + parentRoot: toRootHex(blockRoot), unknownBlockType, }); this.removeAndDownscoreAllDescendants(block); @@ -384,7 +382,7 @@ export class UnknownBlockSync { .BeaconBlock.hashTreeRoot(pendingBlock.blockInput.block.message); this.logger.verbose("Avoid proposer boost for this block of known proposer", { blockSlot, - blockRoot: toHexString(blockRoot), + blockRoot: toRootHex(blockRoot), proposerIndex, }); await sleep(this.proposerBoostSecWindow * 1000); @@ -466,7 +464,7 @@ export class UnknownBlockSync { connectedPeers: PeerIdStr[] ): Promise<{blockInput: BlockInput; peerIdStr: string}> { const shuffledPeers = shuffle(connectedPeers); - const blockRootHex = toHexString(blockRoot); + const blockRootHex = toRootHex(blockRoot); let lastError: Error | null = null; for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) { @@ -483,7 +481,7 @@ export class UnknownBlockSync { const block = blockInput.block.message; const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block); if (!byteArrayEquals(receivedBlockRoot, blockRoot)) { - throw Error(`Wrong block received by peer, got ${toHexString(receivedBlockRoot)} expected ${blockRootHex}`); + throw Error(`Wrong block received by peer, got ${toRootHex(receivedBlockRoot)} expected ${blockRootHex}`); } return {blockInput, peerIdStr: peer}; @@ -527,7 +525,7 @@ export class UnknownBlockSync { blockRoot = this.config .getForkTypes(unavailableBlock.message.slot) .BeaconBlock.hashTreeRoot(unavailableBlock.message); - blockRootHex = toHexString(blockRoot); + blockRootHex = toRootHex(blockRoot); blobKzgCommitmentsLen = (unavailableBlock.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; pendingBlobs = blobKzgCommitmentsLen - unavailableBlockInput.cachedData.blobsCache.size; } @@ -554,7 +552,7 @@ export class UnknownBlockSync { const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block); if (!byteArrayEquals(receivedBlockRoot, blockRoot)) { - throw Error(`Wrong block received by peer, got ${toHexString(receivedBlockRoot)} expected ${blockRootHex}`); + throw Error(`Wrong block received by peer, got ${toRootHex(receivedBlockRoot)} expected ${blockRootHex}`); } if (unavailableBlockInput.block === null) { this.logger.debug("Fetched NullBlockInput", {attempts: i, blockRootHex}); diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index f87b899e9591..ce4125140075 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -1,6 +1,5 @@ import {BitArray, deserializeUint8ArrayBitListFromBytes} from "@chainsafe/ssz"; import {BLSSignature, RootHex, Slot} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; import {BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB} from "@lodestar/params"; export type BlockRootHex = RootHex; @@ -25,6 +24,10 @@ const SLOT_SIZE = 8; const ATTESTATION_DATA_SIZE = 128; const SIGNATURE_SIZE = 96; +// shared Buffers to convert bytes to hex/base64 +const blockRootBuf = Buffer.alloc(ROOT_SIZE); +const attDataBuf = Buffer.alloc(ATTESTATION_DATA_SIZE); + /** * Extract slot from attestation serialized bytes. * Return null if data is not long enough to extract slot. @@ -46,7 +49,10 @@ export function getBlockRootFromAttestationSerialized(data: Uint8Array): BlockRo return null; } - return toHex(data.subarray(ATTESTATION_BEACON_BLOCK_ROOT_OFFSET, ATTESTATION_BEACON_BLOCK_ROOT_OFFSET + ROOT_SIZE)); + blockRootBuf.set( + data.subarray(ATTESTATION_BEACON_BLOCK_ROOT_OFFSET, ATTESTATION_BEACON_BLOCK_ROOT_OFFSET + ROOT_SIZE) + ); + return "0x" + blockRootBuf.toString("hex"); } /** @@ -59,7 +65,8 @@ export function getAttDataBase64FromAttestationSerialized(data: Uint8Array): Att } // base64 is a bit efficient than hex - return toBase64(data.slice(VARIABLE_FIELD_OFFSET, VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE)); + attDataBuf.set(data.subarray(VARIABLE_FIELD_OFFSET, VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE)); + return attDataBuf.toString("base64"); } /** @@ -130,12 +137,13 @@ export function getBlockRootFromSignedAggregateAndProofSerialized(data: Uint8Arr return null; } - return toHex( + blockRootBuf.set( data.subarray( SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET, SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET + ROOT_SIZE ) ); + return "0x" + blockRootBuf.toString("hex"); } /** @@ -148,9 +156,13 @@ export function getAttDataBase64FromSignedAggregateAndProofSerialized(data: Uint } // base64 is a bit efficient than hex - return toBase64( - data.slice(SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET, SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE) + attDataBuf.set( + data.subarray( + SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET, + SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE + ) ); + return attDataBuf.toString("base64"); } /** @@ -217,7 +229,3 @@ function getSlotFromOffsetTrusted(data: Uint8Array, offset: number): Slot { function checkSlotHighBytes(data: Uint8Array, offset: number): boolean { return (data[offset + 4] | data[offset + 5] | data[offset + 6] | data[offset + 7]) === 0; } - -function toBase64(data: Uint8Array): string { - return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("base64"); -} diff --git a/packages/beacon-node/test/perf/network/gossip/encoding.test.ts b/packages/beacon-node/test/perf/network/gossip/encoding.test.ts new file mode 100644 index 000000000000..693c91f59249 --- /dev/null +++ b/packages/beacon-node/test/perf/network/gossip/encoding.test.ts @@ -0,0 +1,46 @@ +import {itBench} from "@dapplion/benchmark"; +import {toHex} from "@lodestar/utils"; + +/** + * This is a benchmark for different ways of converting a gossipsub message id to a hex string using Mac M1 + * encoding + ✔ toHex 6463330 ops/s 154.7190 ns/op - 7170 runs 1.26 s + ✔ Buffer.from 6696982 ops/s 149.3210 ns/op - 2023 runs 0.454 s + ✔ shared Buffer 1.013911e+7 ops/s 98.62800 ns/op - 3083 runs 0.404 s + */ +describe("encoding", function () { + const msgId = Uint8Array.from(Array.from({length: 20}, (_, i) => i)); + + const runsFactor = 1000; + itBench({ + id: "toHex", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + toHex(msgId); + } + }, + runsFactor, + }); + + itBench({ + id: "Buffer.from", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + "0x" + Buffer.from(msgId.buffer, msgId.byteOffset, msgId.byteLength).toString("hex"); + } + }, + runsFactor, + }); + + const sharedBuf = Buffer.from(msgId); + itBench({ + id: "shared Buffer", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + sharedBuf.set(msgId); + "0x" + sharedBuf.toString("hex"); + } + }, + runsFactor, + }); +}); diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 2e2abed95eb7..2405a442c8cd 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -1,5 +1,5 @@ import {toHexString} from "@chainsafe/ssz"; -import {Logger, fromHex} from "@lodestar/utils"; +import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {SLOTS_PER_HISTORICAL_ROOT, SLOTS_PER_EPOCH, INTERVALS_PER_SLOT} from "@lodestar/params"; import {bellatrix, Slot, ValidatorIndex, phase0, ssz, RootHex, Epoch, Root, BeaconBlock} from "@lodestar/types"; import { @@ -472,7 +472,7 @@ export class ForkChoice implements IForkChoice { dataAvailabilityStatus: DataAvailabilityStatus ): ProtoBlock { const {parentRoot, slot} = block; - const parentRootHex = toHexString(parentRoot); + const parentRootHex = toRootHex(parentRoot); // Parent block must be known const parentBlock = this.protoArray.getBlock(parentRootHex); if (!parentBlock) { @@ -529,7 +529,7 @@ export class ForkChoice implements IForkChoice { } const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block); - const blockRootHex = toHexString(blockRoot); + const blockRootHex = toRootHex(blockRoot); // Assign proposer score boost if the block is timely // before attesting interval = before 1st interval @@ -628,14 +628,14 @@ export class ForkChoice implements IForkChoice { slot: slot, blockRoot: blockRootHex, parentRoot: parentRootHex, - targetRoot: toHexString(targetRoot), - stateRoot: toHexString(block.stateRoot), + targetRoot: toRootHex(targetRoot), + stateRoot: toRootHex(block.stateRoot), timeliness: isTimely, justifiedEpoch: stateJustifiedEpoch, - justifiedRoot: toHexString(state.currentJustifiedCheckpoint.root), + justifiedRoot: toRootHex(state.currentJustifiedCheckpoint.root), finalizedEpoch: finalizedCheckpoint.epoch, - finalizedRoot: toHexString(state.finalizedCheckpoint.root), + finalizedRoot: toRootHex(state.finalizedCheckpoint.root), unrealizedJustifiedEpoch: unrealizedJustifiedCheckpoint.epoch, unrealizedJustifiedRoot: unrealizedJustifiedCheckpoint.rootHex, unrealizedFinalizedEpoch: unrealizedFinalizedCheckpoint.epoch, @@ -694,7 +694,7 @@ export class ForkChoice implements IForkChoice { // to genesis just by being present in the chain. const attestationData = attestation.data; const {slot, beaconBlockRoot} = attestationData; - const blockRootHex = toHexString(beaconBlockRoot); + const blockRootHex = toRootHex(beaconBlockRoot); const targetEpoch = attestationData.target.epoch; if (ssz.Root.equals(beaconBlockRoot, ZERO_HASH)) { return; @@ -770,11 +770,11 @@ export class ForkChoice implements IForkChoice { /** Returns `true` if the block is known **and** a descendant of the finalized root. */ hasBlock(blockRoot: Root): boolean { - return this.hasBlockHex(toHexString(blockRoot)); + return this.hasBlockHex(toRootHex(blockRoot)); } /** Returns a `ProtoBlock` if the block is known **and** a descendant of the finalized root. */ getBlock(blockRoot: Root): ProtoBlock | null { - return this.getBlockHex(toHexString(blockRoot)); + return this.getBlockHex(toRootHex(blockRoot)); } /** @@ -793,7 +793,7 @@ export class ForkChoice implements IForkChoice { * Same to hasBlock but without checking if the block is a descendant of the finalized root. */ hasBlockUnsafe(blockRoot: Root): boolean { - return this.hasBlockHexUnsafe(toHexString(blockRoot)); + return this.hasBlockHexUnsafe(toRootHex(blockRoot)); } /** @@ -1221,7 +1221,7 @@ export class ForkChoice implements IForkChoice { forceImport?: boolean ): void { const epochNow = computeEpochAtSlot(this.fcStore.currentSlot); - const targetRootHex = toHexString(attestationData.target.root); + const targetRootHex = toRootHex(attestationData.target.root); // Attestation must be from the current of previous epoch. if (targetEpoch > epochNow) { diff --git a/packages/fork-choice/src/forkChoice/store.ts b/packages/fork-choice/src/forkChoice/store.ts index faf700241fa8..d16e021529db 100644 --- a/packages/fork-choice/src/forkChoice/store.ts +++ b/packages/fork-choice/src/forkChoice/store.ts @@ -1,4 +1,4 @@ -import {toHexString} from "@chainsafe/ssz"; +import {toRootHex} from "@lodestar/utils"; import {EffectiveBalanceIncrements, CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {phase0, Slot, RootHex, ValidatorIndex} from "@lodestar/types"; import {CheckpointHexWithTotalBalance, CheckpointHexWithBalance} from "./interface.js"; @@ -103,7 +103,7 @@ export function toCheckpointWithHex(checkpoint: phase0.Checkpoint): CheckpointWi return { epoch: checkpoint.epoch, root, - rootHex: toHexString(root), + rootHex: toRootHex(root), }; } diff --git a/packages/utils/src/bytes.ts b/packages/utils/src/bytes.ts index 5395c4315d22..f4a622da021e 100644 --- a/packages/utils/src/bytes.ts +++ b/packages/utils/src/bytes.ts @@ -56,6 +56,17 @@ export function toHex(buffer: Uint8Array | Parameters[0]): s } } +// Shared buffer to convert root to hex +const rootBuf = Buffer.alloc(32); + +/** + * Convert a Uint8Array, length 32, to 0x-prefixed hex string + */ +export function toRootHex(root: Uint8Array): string { + rootBuf.set(root); + return `0x${rootBuf.toString("hex")}`; +} + export function fromHex(hex: string): Uint8Array { const b = Buffer.from(hex.replace("0x", ""), "hex"); return new Uint8Array(b.buffer, b.byteOffset, b.length); diff --git a/packages/utils/test/perf/bytes.test.ts b/packages/utils/test/perf/bytes.test.ts new file mode 100644 index 000000000000..368758d12efe --- /dev/null +++ b/packages/utils/test/perf/bytes.test.ts @@ -0,0 +1,27 @@ +import {itBench} from "@dapplion/benchmark"; +import {toHex, toRootHex} from "../../src/bytes.js"; + +describe("bytes utils", function () { + const runsFactor = 1000; + const blockRoot = new Uint8Array(Array.from({length: 32}, (_, i) => i)); + + itBench({ + id: "block root to RootHex using toHex", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + toHex(blockRoot); + } + }, + runsFactor, + }); + + itBench({ + id: "block root to RootHex using toRootHex", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + toRootHex(blockRoot); + } + }, + runsFactor, + }); +}); diff --git a/packages/utils/vitest.config.ts b/packages/utils/vitest.config.ts index 7a6069341168..b2b7cc82b0e2 100644 --- a/packages/utils/vitest.config.ts +++ b/packages/utils/vitest.config.ts @@ -6,6 +6,11 @@ export default mergeConfig( defineConfig({ test: { globalSetup: ["./test/globalSetup.ts"], + typecheck: { + // For some reason Vitest tries to run perf test files which causes an error + // as we use Mocha for those. This ignores all errors outside of test files. + ignoreSourceErrors: true, + }, }, }) ); From 20c18ad85d6cb5e37d4980446d4f608ce9169c24 Mon Sep 17 00:00:00 2001 From: twoeths Date: Thu, 15 Aug 2024 08:02:25 +0700 Subject: [PATCH 015/145] fix: use toRootHex where applicable (#7021) * fix: use toRootHex where applicable * fix: toRootHex() to handle different lengths * fix: throw error if root is not 32 bytes * Fix eth1MergeBlockTracker unit test --------- Co-authored-by: Nico Flaig --- .../src/api/impl/beacon/blocks/index.ts | 16 +++++++-------- .../src/api/impl/lodestar/index.ts | 4 ++-- .../src/api/impl/validator/index.ts | 20 +++++++++---------- .../src/chain/archiver/archiveBlocks.ts | 6 +++--- .../beacon-node/src/chain/blocks/index.ts | 4 ++-- .../src/chain/blocks/verifyBlock.ts | 9 ++++----- .../blocks/verifyBlocksExecutionPayloads.ts | 13 +++++------- .../chain/blocks/verifyBlocksSanityChecks.ts | 6 ++---- .../src/chain/blocks/writeBlockInputToDb.ts | 6 +++--- packages/beacon-node/src/chain/chain.ts | 10 +++++----- .../beacon-node/src/chain/forkChoice/index.ts | 3 +-- .../opPools/aggregatedAttestationPool.ts | 6 +++--- .../chain/produceBlock/produceBlockBody.ts | 10 +++++----- .../src/chain/validation/blobSidecar.ts | 10 +++++----- .../src/eth1/eth1MergeBlockTracker.ts | 5 ++--- .../beacon-node/src/eth1/utils/eth1Vote.ts | 4 ++-- .../src/metrics/validatorMonitor.ts | 18 ++++++++--------- .../peers/utils/assertPeerRelevance.ts | 3 ++- .../reqresp/handlers/blobSidecarsByRoot.ts | 4 ++-- .../beacon-node/src/sync/backfill/backfill.ts | 14 ++++++------- packages/beacon-node/src/sync/range/range.ts | 8 ++++---- .../src/sync/range/utils/chainTarget.ts | 4 ++-- .../src/sync/range/utils/hashBlocks.ts | 8 ++++---- .../unit/eth1/eth1MergeBlockTracker.test.ts | 2 +- .../src/block/processAttestationPhase0.ts | 4 ++-- .../src/block/processBlockHeader.ts | 5 +++-- .../src/block/processWithdrawals.ts | 7 ++++--- .../state-transition/src/stateTransition.ts | 4 ++-- .../src/util/epochShuffling.ts | 5 ++--- .../src/util/weakSubjectivity.ts | 6 ++---- packages/utils/src/bytes.ts | 4 ++++ 31 files changed, 112 insertions(+), 116 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 1b8a59cc8967..4d0f96d1ea08 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -7,7 +7,7 @@ import { signedBeaconBlockToBlinded, } from "@lodestar/state-transition"; import {ForkExecution, SLOTS_PER_HISTORICAL_ROOT, isForkExecution} from "@lodestar/params"; -import {sleep, fromHex, toHex} from "@lodestar/utils"; +import {sleep, fromHex, toRootHex} from "@lodestar/utils"; import { deneb, isSignedBlockContents, @@ -97,9 +97,9 @@ export function getBeaconBlockApi({ // state transition to produce the stateRoot const slot = signedBlock.message.slot; const fork = config.getForkName(slot); - const blockRoot = toHex(chain.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); + const blockRoot = toRootHex(chain.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); // bodyRoot should be the same to produced block - const bodyRoot = toHex(chain.config.getForkTypes(slot).BeaconBlockBody.hashTreeRoot(signedBlock.message.body)); + const bodyRoot = toRootHex(chain.config.getForkTypes(slot).BeaconBlockBody.hashTreeRoot(signedBlock.message.body)); const blockLocallyProduced = chain.producedBlockRoot.has(blockRoot) || chain.producedBlindedBlockRoot.has(blockRoot); const valLogMeta = {slot, blockRoot, bodyRoot, broadcastValidation, blockLocallyProduced}; @@ -147,7 +147,7 @@ export function getBeaconBlockApi({ ); throw new BlockError(signedBlock, { code: BlockErrorCode.PARENT_UNKNOWN, - parentRoot: toHex(signedBlock.message.parentRoot), + parentRoot: toRootHex(signedBlock.message.parentRoot), }); } @@ -243,7 +243,7 @@ export function getBeaconBlockApi({ opts: PublishBlockOpts = {} ) => { const slot = signedBlindedBlock.message.slot; - const blockRoot = toHex( + const blockRoot = toRootHex( chain.config .getExecutionForkTypes(signedBlindedBlock.message.slot) .BlindedBeaconBlock.hashTreeRoot(signedBlindedBlock.message) @@ -258,7 +258,7 @@ export function getBeaconBlockApi({ chain.logger.debug("Reconstructing signedBlockOrContents", {slot, blockRoot, source}); const contents = executionPayload - ? chain.producedContentsCache.get(toHex(executionPayload.blockHash)) ?? null + ? chain.producedContentsCache.get(toRootHex(executionPayload.blockHash)) ?? null : null; const signedBlockOrContents = reconstructFullBlockOrContents(signedBlindedBlock, {executionPayload, contents}); @@ -354,7 +354,7 @@ export function getBeaconBlockApi({ } finalized = false; - if (summary.blockRoot !== toHex(canonicalRoot)) { + if (summary.blockRoot !== toRootHex(canonicalRoot)) { const block = await db.block.get(fromHex(summary.blockRoot)); if (block) { result.push(toBeaconHeaderResponse(config, block)); @@ -473,7 +473,7 @@ export function getBeaconBlockApi({ } if (!blobSidecars) { - throw Error(`blobSidecars not found in db for slot=${block.message.slot} root=${toHex(blockRoot)}`); + throw Error(`blobSidecars not found in db for slot=${block.message.slot} root=${toRootHex(blockRoot)}`); } return { diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index d3083dda8e9c..0e8d2b1fa94b 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -3,7 +3,7 @@ import path from "node:path"; import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {Repository} from "@lodestar/db"; -import {toHex} from "@lodestar/utils"; +import {toHex, toRootHex} from "@lodestar/utils"; import {getLatestWeakSubjectivityCheckpointEpoch} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {ssz} from "@lodestar/types"; @@ -202,7 +202,7 @@ function regenRequestToJson(config: ChainForkConfig, regenRequest: RegenRequest) case "getPreState": { const slot = regenRequest.args[0].slot; return { - root: toHex(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(regenRequest.args[0])), + root: toRootHex(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(regenRequest.args[0])), slot, }; } diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index b2a0b8575f5c..9d692a4cc405 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -41,7 +41,7 @@ import { BlindedBeaconBlock, } from "@lodestar/types"; import {ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; -import {fromHex, toHex, resolveOrRacePromises, prettyWeiToEth} from "@lodestar/utils"; +import {fromHex, toHex, resolveOrRacePromises, prettyWeiToEth, toRootHex} from "@lodestar/utils"; import { AttestationError, AttestationErrorCode, @@ -324,7 +324,7 @@ export function getValidatorApi( function notOnOptimisticBlockRoot(beaconBlockRoot: Root): void { const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot); if (!protoBeaconBlock) { - throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toHex(beaconBlockRoot)}`); + throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); } if (protoBeaconBlock.executionStatus === ExecutionStatus.Syncing) @@ -336,7 +336,7 @@ export function getValidatorApi( function notOnOutOfRangeData(beaconBlockRoot: Root): void { const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot); if (!protoBeaconBlock) { - throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toHex(beaconBlockRoot)}`); + throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); } if (protoBeaconBlock.dataAvailabilityStatus === DataAvailabilityStatus.OutOfRange) @@ -416,7 +416,7 @@ export function getValidatorApi( slot, executionPayloadValue, consensusBlockValue, - root: toHex(config.getExecutionForkTypes(slot).BlindedBeaconBlock.hashTreeRoot(block)), + root: toRootHex(config.getExecutionForkTypes(slot).BlindedBeaconBlock.hashTreeRoot(block)), }); if (chain.opts.persistProducedBlocks) { @@ -494,13 +494,13 @@ export function getValidatorApi( slot, executionPayloadValue, consensusBlockValue, - root: toHex(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block)), + root: toRootHex(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block)), }); if (chain.opts.persistProducedBlocks) { void chain.persistBlock(block, "produced_engine_block"); } if (isForkBlobs(version)) { - const blockHash = toHex((block as bellatrix.BeaconBlock).body.executionPayload.blockHash); + const blockHash = toRootHex((block as bellatrix.BeaconBlock).body.executionPayload.blockHash); const contents = chain.producedContentsCache.get(blockHash); if (contents === undefined) { throw Error("contents missing in cache"); @@ -869,7 +869,7 @@ export function getValidatorApi( // and it hasn't been in our forkchoice since we haven't seen / processing that block // see https://github.com/ChainSafe/lodestar/issues/5063 if (!chain.forkChoice.hasBlock(beaconBlockRoot)) { - const rootHex = toHex(beaconBlockRoot); + const rootHex = toRootHex(beaconBlockRoot); network.searchUnknownSlotRoot({slot, root: rootHex}); // if result of this call is false, i.e. block hasn't seen after 1 slot then the below notOnOptimisticBlockRoot call will throw error await chain.waitForBlock(slot, rootHex); @@ -954,7 +954,7 @@ export function getValidatorApi( return { data: duties, meta: { - dependentRoot: toHex(dependentRoot), + dependentRoot: toRootHex(dependentRoot), executionOptimistic: isOptimisticBlock(head), }, }; @@ -1006,7 +1006,7 @@ export function getValidatorApi( return { data: duties, meta: { - dependentRoot: toHex(dependentRoot), + dependentRoot: toRootHex(dependentRoot), executionOptimistic: isOptimisticBlock(head), }, }; @@ -1071,7 +1071,7 @@ export function getValidatorApi( await waitForSlot(slot); // Must never request for a future slot > currentSlot - const dataRootHex = toHex(attestationDataRoot); + const dataRootHex = toRootHex(attestationDataRoot); const aggregate = chain.attestationPool.getAggregate(slot, dataRootHex); if (!aggregate) { diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts index 76bfe651ad77..ed6dd5bfc091 100644 --- a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts @@ -1,7 +1,7 @@ import {fromHexString} from "@chainsafe/ssz"; import {Epoch, Slot, RootHex} from "@lodestar/types"; import {IForkChoice} from "@lodestar/fork-choice"; -import {Logger, toHex} from "@lodestar/utils"; +import {Logger, toRootHex} from "@lodestar/utils"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {KeyValue} from "@lodestar/db"; @@ -137,7 +137,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot canonicalBlocks.map(async (block) => { const blockBuffer = await db.block.getBinary(block.root); if (!blockBuffer) { - throw Error(`No block found for slot ${block.slot} root ${toHex(block.root)}`); + throw Error(`No block found for slot ${block.slot} root ${toRootHex(block.root)}`); } return { key: block.slot, @@ -177,7 +177,7 @@ async function migrateBlobSidecarsFromHotToColdDb( .map(async (block) => { const bytes = await db.blobSidecars.getBinary(block.root); if (!bytes) { - throw Error(`No blobSidecars found for slot ${block.slot} root ${toHex(block.root)}`); + throw Error(`No blobSidecars found for slot ${block.slot} root ${toRootHex(block.root)}`); } return {key: block.slot, value: bytes}; }) diff --git a/packages/beacon-node/src/chain/blocks/index.ts b/packages/beacon-node/src/chain/blocks/index.ts index eb8c2663c9b6..a7a2ced2ad7a 100644 --- a/packages/beacon-node/src/chain/blocks/index.ts +++ b/packages/beacon-node/src/chain/blocks/index.ts @@ -1,4 +1,4 @@ -import {toHex, isErrorAborted} from "@lodestar/utils"; +import {isErrorAborted, toRootHex} from "@lodestar/utils"; import {SignedBeaconBlock} from "@lodestar/types"; import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js"; import {Metrics} from "../../metrics/metrics.js"; @@ -127,7 +127,7 @@ export async function processBlocks( const blockSlot = signedBlock.message.slot; const {preState, postState} = err.type; const forkTypes = this.config.getForkTypes(blockSlot); - const invalidRoot = toHex(postState.hashTreeRoot()); + const invalidRoot = toRootHex(postState.hashTreeRoot()); const suffix = `slot_${blockSlot}_invalid_state_root_${invalidRoot}`; this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, suffix); diff --git a/packages/beacon-node/src/chain/blocks/verifyBlock.ts b/packages/beacon-node/src/chain/blocks/verifyBlock.ts index bf4cc7e60dcd..4d21342bd8cc 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlock.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlock.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAllForks, computeEpochAtSlot, @@ -9,7 +8,7 @@ import {bellatrix, deneb} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; import {ProtoBlock, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {ChainForkConfig} from "@lodestar/config"; -import {Logger} from "@lodestar/utils"; +import {Logger, toRootHex} from "@lodestar/utils"; import {BlockError, BlockErrorCode} from "../errors/index.js"; import {BlockProcessOpts} from "../options.js"; import {RegenCaller} from "../regen/index.js"; @@ -198,9 +197,9 @@ export async function verifyBlocksInEpoch( } function logOnPowBlock(logger: Logger, config: ChainForkConfig, mergeBlock: bellatrix.BeaconBlock): void { - const mergeBlockHash = toHexString(config.getForkTypes(mergeBlock.slot).BeaconBlock.hashTreeRoot(mergeBlock)); - const mergeExecutionHash = toHexString(mergeBlock.body.executionPayload.blockHash); - const mergePowHash = toHexString(mergeBlock.body.executionPayload.parentHash); + const mergeBlockHash = toRootHex(config.getForkTypes(mergeBlock.slot).BeaconBlock.hashTreeRoot(mergeBlock)); + const mergeExecutionHash = toRootHex(mergeBlock.body.executionPayload.blockHash); + const mergePowHash = toRootHex(mergeBlock.body.executionPayload.parentHash); logger.info(POS_PANDA_MERGE_TRANSITION_BANNER); logger.info("Execution transitioning from PoW to PoS!!!"); logger.info("Importing block referencing terminal PoW block", { diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index d08a747259b1..284d4c0976ee 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAllForks, isExecutionStateType, @@ -17,7 +16,7 @@ import { LVHInvalidResponse, } from "@lodestar/fork-choice"; import {ChainForkConfig} from "@lodestar/config"; -import {ErrorAborted, Logger} from "@lodestar/utils"; +import {ErrorAborted, Logger, toRootHex} from "@lodestar/utils"; import {ForkSeq, SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY} from "@lodestar/params"; import {IExecutionEngine} from "../../execution/engine/interface.js"; @@ -205,10 +204,8 @@ export async function verifyBlocksExecutionPayload( // in import block if (isMergeTransitionBlock) { const mergeBlock = block.message as bellatrix.BeaconBlock; - const mergeBlockHash = toHexString( - chain.config.getForkTypes(mergeBlock.slot).BeaconBlock.hashTreeRoot(mergeBlock) - ); - const powBlockRootHex = toHexString(mergeBlock.body.executionPayload.parentHash); + const mergeBlockHash = toRootHex(chain.config.getForkTypes(mergeBlock.slot).BeaconBlock.hashTreeRoot(mergeBlock)); + const powBlockRootHex = toRootHex(mergeBlock.body.executionPayload.parentHash); const powBlock = await chain.eth1.getPowBlock(powBlockRootHex).catch((error) => { // Lets just warn the user here, errors if any will be reported on // `assertValidTerminalPowBlock` checks @@ -330,7 +327,7 @@ export async function verifyBlockExecutionPayload( const lvhResponse = { executionStatus, latestValidExecHash: execResult.latestValidHash, - invalidateFromParentBlockRoot: toHexString(block.message.parentRoot), + invalidateFromParentBlockRoot: toRootHex(block.message.parentRoot), }; const execError = new BlockError(block, { code: BlockErrorCode.EXECUTION_ENGINE_ERROR, @@ -407,7 +404,7 @@ function getSegmentErrorResponse( for (let mayBeLVHIndex = blockIndex - 1; mayBeLVHIndex >= 0; mayBeLVHIndex--) { const block = blocks[mayBeLVHIndex]; if ( - toHexString((block.message.body as bellatrix.BeaconBlockBody).executionPayload.blockHash) === + toRootHex((block.message.body as bellatrix.BeaconBlockBody).executionPayload.blockHash) === lvhResponse.latestValidExecHash ) { lvhFound = true; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts index fcbcfea05d6e..573dfa52ce8b 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts @@ -2,7 +2,7 @@ import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {Slot} from "@lodestar/types"; -import {toHexString, toRootHex} from "@lodestar/utils"; +import {toRootHex} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; import {BlockInput, ImportBlockOpts} from "./types.js"; @@ -85,9 +85,7 @@ export function verifyBlocksSanityChecks( // Not already known // IGNORE if `partiallyVerifiedBlock.ignoreIfKnown` - const blockHash = toHexString( - chain.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message) - ); + const blockHash = toRootHex(chain.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message)); if (chain.forkChoice.hasBlockHex(blockHash)) { if (opts.ignoreIfKnown) { continue; diff --git a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts index b0f5ab159591..89cf7ddc7556 100644 --- a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts +++ b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts @@ -1,4 +1,4 @@ -import {toHex} from "@lodestar/utils"; +import {toRootHex} from "@lodestar/utils"; import {BeaconChain} from "../chain.js"; import {BlockInput, BlockInputType} from "./types.js"; @@ -15,7 +15,7 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockI for (const blockInput of blocksInput) { const {block, blockBytes} = blockInput; const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); - const blockRootHex = toHex(blockRoot); + const blockRootHex = toRootHex(blockRoot); if (blockBytes) { // skip serializing data if we already have it this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc(); @@ -59,7 +59,7 @@ export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, block for (const blockInput of blockInputs) { const {block, type} = blockInput; const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); - const blockRootHex = toHex(blockRoot); + const blockRootHex = toRootHex(blockRoot); if (!this.forkChoice.hasBlockHex(blockRootHex)) { blockToRemove.push(block); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 43f72d81e882..75608ab33b2e 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -35,7 +35,7 @@ import { } from "@lodestar/types"; import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice"; import {ProcessShutdownCallback} from "@lodestar/validator"; -import {Logger, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toHex, toRootHex} from "@lodestar/utils"; +import {Logger, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils"; import {ForkSeq, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; @@ -691,7 +691,7 @@ export class BeaconChain implements IBeaconChain { blockType === BlockType.Full ? this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block) : this.config.getExecutionForkTypes(slot).BlindedBeaconBlock.hashTreeRoot(block as BlindedBeaconBlock); - const blockRootHex = toHex(blockRoot); + const blockRootHex = toRootHex(blockRoot); // track the produced block for consensus broadcast validations if (blockType === BlockType.Full) { @@ -729,7 +729,7 @@ export class BeaconChain implements IBeaconChain { * ) */ getContents(beaconBlock: deneb.BeaconBlock): deneb.Contents { - const blockHash = toHex(beaconBlock.body.executionPayload.blockHash); + const blockHash = toRootHex(beaconBlock.body.executionPayload.blockHash); const contents = this.producedContentsCache.get(blockHash); if (!contents) { throw Error(`No contents for executionPayload.blockHash ${blockHash}`); @@ -926,7 +926,7 @@ export class BeaconChain implements IBeaconChain { checkpointRoot: checkpoint.rootHex, stateId, stateSlot: state.slot, - stateRoot: toHex(state.hashTreeRoot()), + stateRoot: toRootHex(state.hashTreeRoot()), }); } @@ -997,7 +997,7 @@ export class BeaconChain implements IBeaconChain { // by default store to lodestar_archive of current dir const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr); - const filepath = path.join(dirpath, `${typeName}_${toHex(root)}.ssz`); + const filepath = path.join(dirpath, `${typeName}_${toRootHex(root)}.ssz`); await ensureDir(dirpath); diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index 385b79dcf713..d57b7f86cb98 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {Slot} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import { @@ -91,7 +90,7 @@ export function initializeForkChoice( ...(isExecutionStateType(state) && isMergeTransitionComplete(state) ? { - executionPayloadBlockHash: toHexString(state.latestExecutionPayloadHeader.blockHash), + executionPayloadBlockHash: toRootHex(state.latestExecutionPayloadHeader.blockHash), executionPayloadNumber: state.latestExecutionPayloadHeader.blockNumber, executionStatus: blockHeader.slot === GENESIS_SLOT ? ExecutionStatus.Valid : ExecutionStatus.Syncing, } diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index d896c9d3cc35..8277732ea1d5 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -10,7 +10,7 @@ import { getBlockRootAtSlot, } from "@lodestar/state-transition"; import {IForkChoice, EpochDifference} from "@lodestar/fork-choice"; -import {toHex, MapDef, toRootHex} from "@lodestar/utils"; +import {MapDef, toRootHex} from "@lodestar/utils"; import {intersectUint8Arrays, IntersectResult} from "../../util/bitArray.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; import {InsertOutcome} from "./types.js"; @@ -514,7 +514,7 @@ export function getValidateAttestationDataFn( } // the isValidAttestationData does not depend on slot and index - const beaconBlockRootHex = toHex(attData.beaconBlockRoot); + const beaconBlockRootHex = toRootHex(attData.beaconBlockRoot); const cacheKey = beaconBlockRootHex + targetEpoch; let isValid = cachedValidatedAttestationData.get(cacheKey); if (isValid === undefined) { @@ -559,7 +559,7 @@ export function isValidAttestationData( if (stateEpoch < 2 || targetEpoch < 2) { return true; } - const beaconBlockRootHex = toHex(data.beaconBlockRoot); + const beaconBlockRootHex = toRootHex(data.beaconBlockRoot); return isValidShuffling(forkChoice, state, beaconBlockRootHex, targetEpoch); } diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 214fbdc890ec..d1b410610a43 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -32,7 +32,7 @@ import { } from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {ForkSeq, ForkExecution, isForkExecution} from "@lodestar/params"; -import {toHex, sleep, Logger} from "@lodestar/utils"; +import {toHex, sleep, Logger, toRootHex} from "@lodestar/utils"; import type {BeaconChain} from "../chain.js"; import {PayloadId, IExecutionEngine, IExecutionBuilder, PayloadAttributes} from "../../execution/index.js"; import {ZERO_HASH, ZERO_HASH_HEX} from "../../constants/index.js"; @@ -273,7 +273,7 @@ export async function produceBlockBody( prepType, payloadId, fetchedTime, - executionHeadBlockHash: toHex(engineRes.executionPayload.blockHash), + executionHeadBlockHash: toRootHex(engineRes.executionPayload.blockHash), }); if (executionPayload.transactions.length === 0) { this.metrics?.blockPayload.emptyPayloads.inc({prepType}); @@ -290,7 +290,7 @@ export async function produceBlockBody( } (blockBody as deneb.BeaconBlockBody).blobKzgCommitments = blobsBundle.commitments; - const blockHash = toHex(executionPayload.blockHash); + const blockHash = toRootHex(executionPayload.blockHash); const contents = {kzgProofs: blobsBundle.proofs, blobs: blobsBundle.blobs}; blobsResult = {type: BlobsResultType.produced, contents, blockHash}; @@ -380,7 +380,7 @@ export async function prepareExecutionPayload( const prevRandao = getRandaoMix(state, state.epochCtx.epoch); const payloadIdCached = chain.executionEngine.payloadIdCache.get({ - headBlockHash: toHex(parentHash), + headBlockHash: toRootHex(parentHash), finalizedBlockHash, timestamp: numToQuantity(timestamp), prevRandao: toHex(prevRandao), @@ -414,7 +414,7 @@ export async function prepareExecutionPayload( payloadId = await chain.executionEngine.notifyForkchoiceUpdate( fork, - toHex(parentHash), + toRootHex(parentHash), safeBlockHash, finalizedBlockHash, attributes diff --git a/packages/beacon-node/src/chain/validation/blobSidecar.ts b/packages/beacon-node/src/chain/validation/blobSidecar.ts index f1ea7bfa95c8..4c82d7be153d 100644 --- a/packages/beacon-node/src/chain/validation/blobSidecar.ts +++ b/packages/beacon-node/src/chain/validation/blobSidecar.ts @@ -1,5 +1,5 @@ import {deneb, Root, Slot, ssz} from "@lodestar/types"; -import {toHex, verifyMerkleBranch} from "@lodestar/utils"; +import {toRootHex, verifyMerkleBranch} from "@lodestar/utils"; import {computeStartSlotAtEpoch, getBlockHeaderProposerSignatureSet} from "@lodestar/state-transition"; import {KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, KZG_COMMITMENT_SUBTREE_INDEX0} from "@lodestar/params"; @@ -57,7 +57,7 @@ export async function validateGossipBlobSidecar( // check, we will load the parent and state from disk only to find out later that we // already know this block. const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blobSidecar.signedBlockHeader.message); - const blockHex = toHex(blockRoot); + const blockHex = toRootHex(blockRoot); if (chain.forkChoice.getBlockHex(blockHex) !== null) { throw new BlobSidecarGossipError(GossipAction.IGNORE, {code: BlobSidecarErrorCode.ALREADY_KNOWN, root: blockHex}); } @@ -68,7 +68,7 @@ export async function validateGossipBlobSidecar( // _[IGNORE]_ The blob's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both // gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is // retrieved). - const parentRoot = toHex(blobSidecar.signedBlockHeader.message.parentRoot); + const parentRoot = toRootHex(blobSidecar.signedBlockHeader.message.parentRoot); const parentBlock = chain.forkChoice.getBlockHex(parentRoot); if (parentBlock === null) { // If fork choice does *not* consider the parent to be a descendant of the finalized block, @@ -183,9 +183,9 @@ export function validateBlobSidecars( !byteArrayEquals(expectedKzgCommitments[index], blobSidecar.kzgCommitment) ) { throw new Error( - `Invalid blob with slot=${blobBlockHeader.slot} blobBlockRoot=${toHex(blobBlockRoot)} index=${ + `Invalid blob with slot=${blobBlockHeader.slot} blobBlockRoot=${toRootHex(blobBlockRoot)} index=${ blobSidecar.index - } for the block blockRoot=${toHex(blockRoot)} slot=${blockSlot} index=${index}` + } for the block blockRoot=${toRootHex(blockRoot)} slot=${blockSlot} index=${index}` ); } blobs.push(blobSidecar.blob); diff --git a/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts b/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts index 8602b3058180..9ba4a09a5549 100644 --- a/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts +++ b/packages/beacon-node/src/eth1/eth1MergeBlockTracker.ts @@ -1,7 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {RootHex} from "@lodestar/types"; -import {Logger, pruneSetToMax} from "@lodestar/utils"; +import {Logger, pruneSetToMax, toRootHex} from "@lodestar/utils"; import {Metrics} from "../metrics/index.js"; import {ZERO_HASH_HEX} from "../constants/index.js"; import {enumToIndexMap} from "../util/enum.js"; @@ -239,7 +238,7 @@ export class Eth1MergeBlockTracker { private async internalGetTerminalPowBlockFromEth1(): Promise { // Search merge block by hash // Terminal block hash override takes precedence over terminal total difficulty - const terminalBlockHash = toHexString(this.config.TERMINAL_BLOCK_HASH); + const terminalBlockHash = toRootHex(this.config.TERMINAL_BLOCK_HASH); if (terminalBlockHash !== ZERO_HASH_HEX) { const block = await this.getPowBlock(terminalBlockHash); if (block) { diff --git a/packages/beacon-node/src/eth1/utils/eth1Vote.ts b/packages/beacon-node/src/eth1/utils/eth1Vote.ts index 3940ccb27bae..cdc8fa04163e 100644 --- a/packages/beacon-node/src/eth1/utils/eth1Vote.ts +++ b/packages/beacon-node/src/eth1/utils/eth1Vote.ts @@ -2,7 +2,7 @@ import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; import {phase0, RootHex} from "@lodestar/types"; import {BeaconStateAllForks, computeTimeAtSlot} from "@lodestar/state-transition"; -import {toHex} from "@lodestar/utils"; +import {toRootHex} from "@lodestar/utils"; export type Eth1DataGetter = ({ timestampRange, @@ -128,7 +128,7 @@ function getEth1DataKey(eth1Data: phase0.Eth1Data): string { * Serialize eth1Data types to a unique string ID. It is only used for comparison. */ export function fastSerializeEth1Data(eth1Data: phase0.Eth1Data): string { - return toHex(eth1Data.blockHash) + eth1Data.depositCount.toString(16) + toHex(eth1Data.depositRoot); + return toRootHex(eth1Data.blockHash) + eth1Data.depositCount.toString(16) + toRootHex(eth1Data.depositRoot); } export function votingPeriodStartTime(config: ChainForkConfig, state: BeaconStateAllForks): number { diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index e5c8eef83679..5492f57edc02 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -8,7 +8,7 @@ import { getBlockRootAtSlot, ParticipationFlags, } from "@lodestar/state-transition"; -import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, toHex} from "@lodestar/utils"; +import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, toRootHex} from "@lodestar/utils"; import {BeaconBlock, RootHex, altair, deneb} from "@lodestar/types"; import {ChainConfig, ChainForkConfig} from "@lodestar/config"; import {ForkSeq, INTERVALS_PER_SLOT, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; @@ -392,7 +392,7 @@ export function createValidatorMonitor( const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot)); summary.blockProposals.push({ - blockRoot: toHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)), + blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)), blockSlot: block.slot, poolSubmitDelaySec: delaySec, successfullyImported: false, @@ -416,7 +416,7 @@ export function createValidatorMonitor( proposal.successfullyImported = true; } else { summary.blockProposals.push({ - blockRoot: toHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)), + blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)), blockSlot: block.slot, poolSubmitDelaySec: null, successfullyImported: true, @@ -445,7 +445,7 @@ export function createValidatorMonitor( const attestationSummary = validator.attestations .getOrDefault(indexedAttestation.data.target.epoch) - .getOrDefault(toHex(indexedAttestation.data.target.root)); + .getOrDefault(toRootHex(indexedAttestation.data.target.root)); if ( attestationSummary.poolSubmitDelayMinSec === null || attestationSummary.poolSubmitDelayMinSec > delaySec @@ -494,7 +494,7 @@ export function createValidatorMonitor( validator.attestations .getOrDefault(indexedAttestation.data.target.epoch) - .getOrDefault(toHex(indexedAttestation.data.target.root)) + .getOrDefault(toRootHex(indexedAttestation.data.target.root)) .aggregateInclusionDelaysSec.push(delaySec); } } @@ -533,7 +533,7 @@ export function createValidatorMonitor( validator.attestations .getOrDefault(indexedAttestation.data.target.epoch) - .getOrDefault(toHex(indexedAttestation.data.target.root)) + .getOrDefault(toRootHex(indexedAttestation.data.target.root)) .aggregateInclusionDelaysSec.push(delaySec); } } @@ -577,7 +577,7 @@ export function createValidatorMonitor( validator.attestations .getOrDefault(indexedAttestation.data.target.epoch) - .getOrDefault(toHex(indexedAttestation.data.target.root)) + .getOrDefault(toRootHex(indexedAttestation.data.target.root)) .blockInclusions.push({ blockRoot: inclusionBlockRoot, blockSlot: inclusionBlockSlot, @@ -650,7 +650,7 @@ export function createValidatorMonitor( if (config.getForkSeq(headState.slot) >= ForkSeq.altair) { const {previousEpochParticipation} = headState as CachedBeaconStateAltair; const prevEpochStartSlot = computeStartSlotAtEpoch(prevEpoch); - const prevEpochTargetRoot = toHex(getBlockRootAtSlot(headState, prevEpochStartSlot)); + const prevEpochTargetRoot = toRootHex(getBlockRootAtSlot(headState, prevEpochStartSlot)); // Check attestation performance for (const [index, validator] of validators.entries()) { @@ -1041,7 +1041,7 @@ export class RootHexCache { getBlockRootAtSlot(slot: Slot): RootHex { let root = this.blockRootSlotCache.get(slot); if (!root) { - root = toHex(getBlockRootAtSlot(this.state, slot)); + root = toRootHex(getBlockRootAtSlot(this.state, slot)); this.blockRootSlotCache.set(slot, root); } return root; diff --git a/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts b/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts index 95743331ab2a..96e35e9d317d 100644 --- a/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts +++ b/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts @@ -1,5 +1,6 @@ import {toHexString} from "@chainsafe/ssz"; import {ForkDigest, Root, Slot, phase0, ssz} from "@lodestar/types"; +import {toRootHex} from "@lodestar/utils"; // TODO: Why this value? (From Lighthouse) const FUTURE_SLOT_TOLERANCE = 1; @@ -82,6 +83,6 @@ export function renderIrrelevantPeerType(type: IrrelevantPeerType): string { case IrrelevantPeerCode.DIFFERENT_CLOCKS: return `DIFFERENT_CLOCKS slotDiff: ${type.slotDiff}`; case IrrelevantPeerCode.DIFFERENT_FINALIZED: - return `DIFFERENT_FINALIZED root: ${toHexString(type.remoteRoot)} expected: ${toHexString(type.expectedRoot)}`; + return `DIFFERENT_FINALIZED root: ${toRootHex(type.remoteRoot)} expected: ${toRootHex(type.expectedRoot)}`; } } diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts index 6aa16a0c2629..144d470d7199 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts @@ -1,7 +1,7 @@ import {ResponseError, ResponseOutgoing, RespStatus} from "@lodestar/reqresp"; import {BLOBSIDECAR_FIXED_SIZE} from "@lodestar/params"; import {deneb, RootHex} from "@lodestar/types"; -import {toHex, fromHex} from "@lodestar/utils"; +import {fromHex, toRootHex} from "@lodestar/utils"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../../../db/repositories/blobSidecars.js"; @@ -20,7 +20,7 @@ export async function* onBlobSidecarsByRoot( for (const blobIdentifier of requestBody) { const {blockRoot, index} = blobIdentifier; - const blockRootHex = toHex(blockRoot); + const blockRootHex = toRootHex(blockRoot); const block = chain.forkChoice.getBlockHex(blockRootHex); // NOTE: Only support non-finalized blocks. diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index 15ba34f4e583..38258fde07fd 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -3,7 +3,7 @@ import {StrictEventEmitter} from "strict-event-emitter-types"; import {BeaconStateAllForks, blockToHeader} from "@lodestar/state-transition"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {phase0, Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; -import {ErrorAborted, Logger, sleep, toHex, toRootHex} from "@lodestar/utils"; +import {ErrorAborted, Logger, sleep, toRootHex} from "@lodestar/utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {IBeaconChain} from "../../chain/index.js"; @@ -527,7 +527,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} ) // TODO: explode and stop the entire node throw new Error( - `InvalidWsCheckpoint root=${toHex(this.wsCheckpointHeader.root)}, epoch=${ + `InvalidWsCheckpoint root=${toRootHex(this.wsCheckpointHeader.root)}, epoch=${ this.wsCheckpointHeader.slot / SLOTS_PER_EPOCH }, ${ wsDbCheckpointBlock @@ -589,7 +589,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} } else { validSequence = false; this.logger.warn( - `Invalid backfill sequence: previous finalized or checkpoint block root=${toHex( + `Invalid backfill sequence: previous finalized or checkpoint block root=${toRootHex( this.prevFinalizedCheckpointBlock.root )}, slot=${this.prevFinalizedCheckpointBlock.slot} ${ prevBackfillCpBlock ? "found at slot=" + prevBackfillCpBlock.message.slot : "not found" @@ -643,7 +643,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} if (cleanupSeqs.length > 0) { await this.db.backfilledRanges.batchDelete(cleanupSeqs.map((entry) => entry.key)); this.logger.debug( - `Cleaned up the old sequences between ${this.backfillStartFromSlot},${toHex( + `Cleaned up the old sequences between ${this.backfillStartFromSlot},${toRootHex( this.syncAnchor.lastBackSyncedBlock.root )}`, {cleanupSeqs: JSON.stringify(cleanupSeqs)} @@ -669,7 +669,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} if (expectedSlot !== null && anchorBlock.message.slot !== expectedSlot) throw Error( - `Invalid slot of anchorBlock read from DB with root=${toHex( + `Invalid slot of anchorBlock read from DB with root=${toRootHex( anchorBlockRoot )}, expected=${expectedSlot}, actual=${anchorBlock.message.slot}` ); @@ -686,7 +686,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} // Before moving anchorBlock back, we need check for prevFinalizedCheckpointBlock if (anchorBlock.message.slot < this.prevFinalizedCheckpointBlock.slot) { throw Error( - `Skipped a prevFinalizedCheckpointBlock with slot=${toHex( + `Skipped a prevFinalizedCheckpointBlock with slot=${toRootHex( this.prevFinalizedCheckpointBlock.root )}, root=${toRootHex(this.prevFinalizedCheckpointBlock.root)}` ); @@ -699,7 +699,7 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter} throw Error( `Invalid root for prevFinalizedCheckpointBlock at slot=${ this.prevFinalizedCheckpointBlock.slot - }, expected=${toRootHex(this.prevFinalizedCheckpointBlock.root)}, found=${toHex(anchorBlockRoot)}` + }, expected=${toRootHex(this.prevFinalizedCheckpointBlock.root)}, found=${toRootHex(anchorBlockRoot)}` ); } diff --git a/packages/beacon-node/src/sync/range/range.ts b/packages/beacon-node/src/sync/range/range.ts index d20e0c3690cd..887a86a3aaf2 100644 --- a/packages/beacon-node/src/sync/range/range.ts +++ b/packages/beacon-node/src/sync/range/range.ts @@ -3,7 +3,7 @@ import {StrictEventEmitter} from "strict-event-emitter-types"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import {Epoch, phase0} from "@lodestar/types"; -import {Logger, toHex} from "@lodestar/utils"; +import {Logger, toRootHex} from "@lodestar/utils"; import {IBeaconChain} from "../../chain/index.js"; import {INetwork} from "../../network/index.js"; import {Metrics} from "../../metrics/index.js"; @@ -119,7 +119,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) { syncType, startEpoch, targetSlot: target.slot, - targetRoot: toHex(target.root), + targetRoot: toRootHex(target.root), }); // If the peer existed in any other chain, remove it. @@ -242,7 +242,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) { syncType, firstEpoch: syncChain.firstBatchEpoch, targetSlot: syncChain.target.slot, - targetRoot: toHex(syncChain.target.root), + targetRoot: toRootHex(syncChain.target.root), }); } @@ -274,7 +274,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) { lastValidatedSlot: syncChain.lastValidatedSlot, firstEpoch: syncChain.firstBatchEpoch, targetSlot: syncChain.target.slot, - targetRoot: toHex(syncChain.target.root), + targetRoot: toRootHex(syncChain.target.root), validatedEpochs: syncChain.validatedEpochs, }); diff --git a/packages/beacon-node/src/sync/range/utils/chainTarget.ts b/packages/beacon-node/src/sync/range/utils/chainTarget.ts index 221b6e1be68f..15076a028818 100644 --- a/packages/beacon-node/src/sync/range/utils/chainTarget.ts +++ b/packages/beacon-node/src/sync/range/utils/chainTarget.ts @@ -1,5 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {Root, Slot} from "@lodestar/types"; +import {toRootHex} from "@lodestar/utils"; /** * Sync this up to this target. Uses slot instead of epoch to re-use logic for finalized sync @@ -21,7 +21,7 @@ export function computeMostCommonTarget(targets: ChainTarget[]): ChainTarget { let mostCommonCount = 0; for (const target of targets) { - const targetId = `${target.slot}-${toHexString(target.root)}`; + const targetId = `${target.slot}-${toRootHex(target.root)}`; const count = 1 + (countById.get(targetId) ?? 0); countById.set(targetId, count); if (count > mostCommonCount) { diff --git a/packages/beacon-node/src/sync/range/utils/hashBlocks.ts b/packages/beacon-node/src/sync/range/utils/hashBlocks.ts index 522242924a3c..986d023d9ca1 100644 --- a/packages/beacon-node/src/sync/range/utils/hashBlocks.ts +++ b/packages/beacon-node/src/sync/range/utils/hashBlocks.ts @@ -1,6 +1,6 @@ import {RootHex} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; -import {toHex} from "@lodestar/utils"; +import {toRootHex} from "@lodestar/utils"; import {BlockInput} from "../../../chain/blocks/types.js"; /** @@ -12,14 +12,14 @@ export function hashBlocks(blocks: BlockInput[], config: ChainForkConfig): RootH return "0x"; case 1: { const block0 = blocks[0].block; - return toHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)); + return toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)); } default: { const block0 = blocks[0].block; const blockN = blocks[blocks.length - 1].block; return ( - toHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)) + - toHex(config.getForkTypes(blockN.message.slot).SignedBeaconBlock.hashTreeRoot(blockN)) + toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)) + + toRootHex(config.getForkTypes(blockN.message.slot).SignedBeaconBlock.hashTreeRoot(blockN)) ); } } diff --git a/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts b/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts index 938c272b316a..151f3931cfde 100644 --- a/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts +++ b/packages/beacon-node/test/unit/eth1/eth1MergeBlockTracker.test.ts @@ -32,7 +32,7 @@ describe("eth1 / Eth1MergeBlockTracker", () => { }); it("Should find terminal pow block through TERMINAL_BLOCK_HASH", async () => { - config.TERMINAL_BLOCK_HASH = Buffer.alloc(1, 32); + config.TERMINAL_BLOCK_HASH = Buffer.alloc(32, 1); const block: EthJsonRpcBlockRaw = { number: toHex(10), hash: toRootHex(11), diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index 248ba83b4ed2..95fc575003ee 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -1,7 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {Slot, phase0, ssz} from "@lodestar/types"; import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, ForkSeq} from "@lodestar/params"; +import {toRootHex} from "@lodestar/utils"; import {computeEpochAtSlot} from "../util/index.js"; import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../types.js"; import {isValidIndexedAttestation} from "./index.js"; @@ -113,5 +113,5 @@ export function isTimelyTarget(fork: ForkSeq, inclusionDistance: Slot): boolean } export function checkpointToStr(checkpoint: phase0.Checkpoint): string { - return `${toHexString(checkpoint.root)}:${checkpoint.epoch}`; + return `${toRootHex(checkpoint.root)}:${checkpoint.epoch}`; } diff --git a/packages/state-transition/src/block/processBlockHeader.ts b/packages/state-transition/src/block/processBlockHeader.ts index 755850969b4f..da3e389fb507 100644 --- a/packages/state-transition/src/block/processBlockHeader.ts +++ b/packages/state-transition/src/block/processBlockHeader.ts @@ -1,5 +1,6 @@ -import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {BeaconBlock, BlindedBeaconBlock, ssz} from "@lodestar/types"; +import {toRootHex} from "@lodestar/utils"; import {CachedBeaconStateAllForks} from "../types.js"; import {ZERO_HASH} from "../constants/index.js"; import {blindedOrFullBlockToHeader} from "../util/index.js"; @@ -32,7 +33,7 @@ export function processBlockHeader(state: CachedBeaconStateAllForks, block: Beac // verify that the parent matches if (!byteArrayEquals(block.parentRoot, ssz.phase0.BeaconBlockHeader.hashTreeRoot(state.latestBlockHeader))) { throw new Error( - `Block parent root ${toHexString(block.parentRoot)} does not match state latest block, block slot=${slot}` + `Block parent root ${toRootHex(block.parentRoot)} does not match state latest block, block slot=${slot}` ); } diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index ddea73c27a26..9ae4c570e013 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -1,4 +1,4 @@ -import {byteArrayEquals, toHexString} from "@chainsafe/ssz"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {ssz, capella} from "@lodestar/types"; import { MAX_EFFECTIVE_BALANCE, @@ -6,6 +6,7 @@ import { MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP, } from "@lodestar/params"; +import {toRootHex} from "@lodestar/utils"; import {CachedBeaconStateCapella} from "../types.js"; import {decreaseBalance, hasEth1WithdrawalCredential, isCapellaPayloadHeader} from "../util/index.js"; @@ -21,9 +22,9 @@ export function processWithdrawals( const actualWithdrawalsRoot = payload.withdrawalsRoot; if (!byteArrayEquals(expectedWithdrawalsRoot, actualWithdrawalsRoot)) { throw Error( - `Invalid withdrawalsRoot of executionPayloadHeader, expected=${toHexString( + `Invalid withdrawalsRoot of executionPayloadHeader, expected=${toRootHex( expectedWithdrawalsRoot - )}, actual=${toHexString(actualWithdrawalsRoot)}` + )}, actual=${toRootHex(actualWithdrawalsRoot)}` ); } } else { diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 78bcaa140c62..a454bf4b34d0 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -1,6 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {SignedBeaconBlock, SignedBlindedBeaconBlock, Slot, ssz} from "@lodestar/types"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {toRootHex} from "@lodestar/utils"; import {BeaconStateTransitionMetrics, onPostStateMetrics, onStateCloneMetrics} from "./metrics.js"; import {beforeProcessEpoch, EpochTransitionCache, EpochTransitionCacheOpts} from "./cache/epochTransitionCache.js"; import { @@ -123,7 +123,7 @@ export function stateTransition( if (!ssz.Root.equals(block.stateRoot, stateRoot)) { throw new Error( - `Invalid state root at slot ${block.slot}, expected=${toHexString(block.stateRoot)}, actual=${toHexString( + `Invalid state root at slot ${block.slot}, expected=${toRootHex(block.stateRoot)}, actual=${toRootHex( stateRoot )}` ); diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index e101da38f297..856721d8d083 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; -import {intDiv} from "@lodestar/utils"; +import {intDiv, toRootHex} from "@lodestar/utils"; import { DOMAIN_BEACON_ATTESTER, MAX_COMMITTEES_PER_SLOT, @@ -108,5 +107,5 @@ export function computeEpochShuffling( export function getShufflingDecisionBlock(state: BeaconStateAllForks, epoch: Epoch): RootHex { const pivotSlot = computeStartSlotAtEpoch(epoch - 1) - 1; - return toHexString(getBlockRootAtSlot(state, pivotSlot)); + return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } diff --git a/packages/state-transition/src/util/weakSubjectivity.ts b/packages/state-transition/src/util/weakSubjectivity.ts index 6bd6636c3e70..0e606e66340f 100644 --- a/packages/state-transition/src/util/weakSubjectivity.ts +++ b/packages/state-transition/src/util/weakSubjectivity.ts @@ -1,9 +1,9 @@ -import {toHexString} from "@chainsafe/ssz"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {EFFECTIVE_BALANCE_INCREMENT, MAX_DEPOSITS, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, Root} from "@lodestar/types"; import {ssz} from "@lodestar/types"; import {Checkpoint} from "@lodestar/types/phase0"; +import {toRootHex} from "@lodestar/utils"; import {ZERO_HASH} from "../constants/constants.js"; import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../types.js"; import {computeEpochAtSlot, getCurrentEpoch, computeCheckpointEpochAtStateSlot} from "./epoch.js"; @@ -127,9 +127,7 @@ export function ensureWithinWeakSubjectivityPeriod( const wsStateEpoch = computeCheckpointEpochAtStateSlot(wsState.slot); const blockRoot = getLatestBlockRoot(wsState); if (!ssz.Root.equals(blockRoot, wsCheckpoint.root)) { - throw new Error( - `Roots do not match. expected=${toHexString(wsCheckpoint.root)}, actual=${toHexString(blockRoot)}` - ); + throw new Error(`Roots do not match. expected=${toRootHex(wsCheckpoint.root)}, actual=${toRootHex(blockRoot)}`); } if (!ssz.Epoch.equals(wsStateEpoch, wsCheckpoint.epoch)) { throw new Error(`Epochs do not match. expected=${wsCheckpoint.epoch}, actual=${wsStateEpoch}`); diff --git a/packages/utils/src/bytes.ts b/packages/utils/src/bytes.ts index f4a622da021e..820cf7d63300 100644 --- a/packages/utils/src/bytes.ts +++ b/packages/utils/src/bytes.ts @@ -63,6 +63,10 @@ const rootBuf = Buffer.alloc(32); * Convert a Uint8Array, length 32, to 0x-prefixed hex string */ export function toRootHex(root: Uint8Array): string { + if (root.length !== 32) { + throw Error(`Expect root to be 32 bytes, got ${root.length}`); + } + rootBuf.set(root); return `0x${rootBuf.toString("hex")}`; } From 550c349259cb458cf58926cc042d23fe08fd9c1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 18:38:46 +0000 Subject: [PATCH 016/145] chore(deps): bump axios from 1.6.0 to 1.7.4 (#7020) * chore(deps): bump axios from 1.6.0 to 1.7.4 Bumps [axios](https://github.com/axios/axios) from 1.6.0 to 1.7.4. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.6.0...v1.7.4) --- updated-dependencies: - dependency-name: axios dependency-type: direct:production ... Signed-off-by: dependabot[bot] * chore: dedupe dependency versions * chore: dedupe dependency versions --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cayman --- yarn.lock | 105 ++++++++++++++++++++++-------------------------------- 1 file changed, 43 insertions(+), 62 deletions(-) diff --git a/yarn.lock b/yarn.lock index f561a557351e..d47feb4514f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2636,18 +2636,18 @@ integrity sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ== "@puppeteer/browsers@1.4.6", "@puppeteer/browsers@^1.6.0", "@puppeteer/browsers@^2.1.0": - version "2.2.2" - resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.2.2.tgz#c43b00a9808370fec3e548779d81d1e0b972e8bb" - integrity sha512-hZ/JhxPIceWaGSEzUZp83/8M49CoxlkuThfTR7t4AoCu5+ZvJ3vktLm60Otww2TXeROB5igiZ8D9oPQh6ckBVg== + version "2.3.0" + resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.3.0.tgz#791ea7d80450fea24eb19fb1d70c367ad4e08cae" + integrity sha512-ioXoq9gPxkss4MYhD+SFaU9p1IHFUX0ILAWFPyjGaBdjLsYAlZw6j1iLA0N/m12uVHLFDfSYNF7EQccjinIMDA== dependencies: - debug "4.3.4" - extract-zip "2.0.1" - progress "2.0.3" - proxy-agent "6.4.0" - semver "7.6.0" - tar-fs "3.0.5" - unbzip2-stream "1.4.3" - yargs "17.7.2" + debug "^4.3.5" + extract-zip "^2.0.1" + progress "^2.0.3" + proxy-agent "^6.4.0" + semver "^7.6.3" + tar-fs "^3.0.6" + unbzip2-stream "^1.4.3" + yargs "^17.7.2" "@rollup/plugin-inject@^5.0.5": version "5.0.5" @@ -4313,11 +4313,11 @@ aws-sdk@^2.932.0: xml2js "0.4.19" axios@^1.0.0, axios@^1.3.4: - version "1.6.0" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.0.tgz#f1e5292f26b2fd5c2e66876adc5b06cdbd7d2102" - integrity sha512-EZ1DYihju9pwVB+jg67ogm+Tmqc6JmhamRN6I4Zt8DfZu5lbcQGw3ozH9lFejSJgs/ibaef3A9PMXPLeefFGJg== + version "1.7.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.4.tgz#4c8ded1b43683c8dd362973c393f3ede24052aa2" + integrity sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw== dependencies: - follow-redirects "^1.15.0" + follow-redirects "^1.15.6" form-data "^4.0.0" proxy-from-env "^1.1.0" @@ -6550,7 +6550,7 @@ external-editor@^3.0.3: iconv-lite "^0.4.24" tmp "^0.0.33" -extract-zip@2.0.1, extract-zip@^2.0.1: +extract-zip@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== @@ -6831,7 +6831,7 @@ fn.name@1.x.x: resolved "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.0.0, follow-redirects@^1.15.0: +follow-redirects@^1.0.0, follow-redirects@^1.15.6: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== @@ -7057,7 +7057,7 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-func-name@^2.0.0: +get-func-name@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41" integrity sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ== @@ -7642,18 +7642,10 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" -https-proxy-agent@^7.0.2: - version "7.0.2" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz#e2645b846b90e96c6e6f347fb5b2e41f1590b09b" - integrity sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA== - dependencies: - agent-base "^7.0.2" - debug "4" - -https-proxy-agent@^7.0.3, https-proxy-agent@^7.0.4: - version "7.0.4" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" - integrity sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg== +https-proxy-agent@^7.0.2, https-proxy-agent@^7.0.3, https-proxy-agent@^7.0.4: + version "7.0.5" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz#9e8b5013873299e11fab6fd548405da2d6c602b2" + integrity sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw== dependencies: agent-base "^7.0.2" debug "4" @@ -9029,11 +9021,11 @@ loglevel@^1.6.0: integrity sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg== loupe@^2.3.6, loupe@^3.1.0, loupe@^3.1.1: - version "2.3.6" - resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.6.tgz#76e4af498103c532d1ecc9be102036a21f787b53" - integrity sha512-RaPMZKiMy8/JruncMU5Bt6na1eftNoo++R4Y+N2FrxkDVTrGvcyzFTsaGif4QTeKESheMGegbhw6iUAq+5A8zA== + version "2.3.7" + resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.7.tgz#6e69b7d4db7d3ab436328013d37d1c8c3540c697" + integrity sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA== dependencies: - get-func-name "^2.0.0" + get-func-name "^2.0.1" lowercase-keys@^2.0.0: version "2.0.0" @@ -9703,9 +9695,9 @@ n12@0.4.0: integrity sha512-p/hj4zQ8d3pbbFLQuN1K9honUxiDDhueOWyFLw/XgBv+wZCE44bcLH4CIcsolOceJQduh4Jf7m/LfaTxyGmGtQ== nan@^2.16.0, nan@^2.17.0, nan@^2.19.0: - version "2.19.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.19.0.tgz#bb58122ad55a6c5bc973303908d5b16cfdd5a8c0" - integrity sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw== + version "2.20.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.20.0.tgz#08c5ea813dd54ed16e5bd6505bf42af4f7838ca3" + integrity sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw== nanoid@3.3.3: version "3.3.3" @@ -10845,7 +10837,7 @@ progress-events@^1.0.0: resolved "https://registry.yarnpkg.com/progress-events/-/progress-events-1.0.0.tgz#34f5e8fdb5dae3561837b22672d1e02277bb2109" integrity sha512-zIB6QDrSbPfRg+33FZalluFIowkbV5Xh1xSuetjG+rlC5he6u2dc6VQJ0TbMdlN3R1RHdpOqxEFMKTnQ+itUwA== -progress@2.0.3, progress@^2.0.3: +progress@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== @@ -10916,7 +10908,7 @@ proxy-addr@^2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy-agent@6.4.0: +proxy-agent@^6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.4.0.tgz#b4e2dd51dee2b377748aef8d45604c2d7608652d" integrity sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ== @@ -11600,19 +11592,17 @@ semver@7.5.3: dependencies: lru-cache "^6.0.0" -semver@7.6.0, semver@^7.6.0: - version "7.6.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" - integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== - dependencies: - lru-cache "^6.0.0" - semver@^6.1.0, semver@^6.2.0, semver@^6.3.1: version "6.3.1" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@~7.5.4: +semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + +semver@~7.5.4: version "7.5.4" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== @@ -12309,10 +12299,10 @@ tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-fs@3.0.5, tar-fs@^3.0.5: - version "3.0.5" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.5.tgz#f954d77767e4e6edf973384e1eb95f8f81d64ed9" - integrity sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg== +tar-fs@^3.0.4, tar-fs@^3.0.5, tar-fs@^3.0.6: + version "3.0.6" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.6.tgz#eaccd3a67d5672f09ca8e8f9c3d2b89fa173f217" + integrity sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w== dependencies: pump "^3.0.0" tar-stream "^3.1.5" @@ -12320,15 +12310,6 @@ tar-fs@3.0.5, tar-fs@^3.0.5: bare-fs "^2.1.1" bare-path "^2.1.0" -tar-fs@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" - integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== - dependencies: - mkdirp-classic "^0.5.2" - pump "^3.0.0" - tar-stream "^3.1.5" - tar-fs@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" @@ -12877,7 +12858,7 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" -unbzip2-stream@1.4.3: +unbzip2-stream@^1.4.3: version "1.4.3" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" integrity sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg== @@ -13886,7 +13867,7 @@ yargs@16.2.0, yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@17.7.2, yargs@^17.1.1, yargs@^17.5.1, yargs@^17.6.2, yargs@^17.7.1, yargs@^17.7.2: +yargs@^17.1.1, yargs@^17.5.1, yargs@^17.6.2, yargs@^17.7.1, yargs@^17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== From 6f470f8a35dcd5f39be8cdc5f59c513e1ab00fec Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sun, 18 Aug 2024 03:00:02 +0100 Subject: [PATCH 017/145] fix: return 404 error if no sync committee contribution is available (#6649) * fix: return 404 error if no sync committee contribution is available * Return 404 error if block not in fork choice --- packages/beacon-node/src/api/impl/validator/index.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 9d692a4cc405..ca0fe1ac95a2 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -324,7 +324,7 @@ export function getValidatorApi( function notOnOptimisticBlockRoot(beaconBlockRoot: Root): void { const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot); if (!protoBeaconBlock) { - throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); + throw new ApiError(404, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); } if (protoBeaconBlock.executionStatus === ExecutionStatus.Syncing) @@ -336,7 +336,7 @@ export function getValidatorApi( function notOnOutOfRangeData(beaconBlockRoot: Root): void { const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot); if (!protoBeaconBlock) { - throw new ApiError(400, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); + throw new ApiError(404, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`); } if (protoBeaconBlock.dataAvailabilityStatus === DataAvailabilityStatus.OutOfRange) @@ -880,7 +880,12 @@ export function getValidatorApi( notOnOutOfRangeData(beaconBlockRoot); const contribution = chain.syncCommitteeMessagePool.getContribution(subcommitteeIndex, slot, beaconBlockRoot); - if (!contribution) throw new ApiError(500, "No contribution available"); + if (!contribution) { + throw new ApiError( + 404, + `No sync committee contribution for slot=${slot}, subnet=${subcommitteeIndex}, beaconBlockRoot=${toRootHex(beaconBlockRoot)}` + ); + } metrics?.production.producedSyncContributionParticipants.observe( contribution.aggregationBits.getTrueBitIndexes().length From 4ea7edd6a7482f3131a03f9cde931bf835ce0e03 Mon Sep 17 00:00:00 2001 From: Phil Ngo <58080811+philknows@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:33:33 -0400 Subject: [PATCH 018/145] chore: update license on state-transition (#7037) Update LICENSE --- packages/state-transition/LICENSE | 366 ++++++++++++++++-------------- 1 file changed, 201 insertions(+), 165 deletions(-) diff --git a/packages/state-transition/LICENSE b/packages/state-transition/LICENSE index 153d416dc8d2..261eeb9e9f8b 100644 --- a/packages/state-transition/LICENSE +++ b/packages/state-transition/LICENSE @@ -1,165 +1,201 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - - This version of the GNU Lesser General Public License incorporates -the terms and conditions of version 3 of the GNU General Public -License, supplemented by the additional permissions listed below. - - 0. Additional Definitions. - - As used herein, "this License" refers to version 3 of the GNU Lesser -General Public License, and the "GNU GPL" refers to version 3 of the GNU -General Public License. - - "The Library" refers to a covered work governed by this License, -other than an Application or a Combined Work as defined below. - - An "Application" is any work that makes use of an interface provided -by the Library, but which is not otherwise based on the Library. -Defining a subclass of a class defined by the Library is deemed a mode -of using an interface provided by the Library. - - A "Combined Work" is a work produced by combining or linking an -Application with the Library. The particular version of the Library -with which the Combined Work was made is also called the "Linked -Version". - - The "Minimal Corresponding Source" for a Combined Work means the -Corresponding Source for the Combined Work, excluding any source code -for portions of the Combined Work that, considered in isolation, are -based on the Application, and not on the Linked Version. - - The "Corresponding Application Code" for a Combined Work means the -object code and/or source code for the Application, including any data -and utility programs needed for reproducing the Combined Work from the -Application, but excluding the System Libraries of the Combined Work. - - 1. Exception to Section 3 of the GNU GPL. - - You may convey a covered work under sections 3 and 4 of this License -without being bound by section 3 of the GNU GPL. - - 2. Conveying Modified Versions. - - If you modify a copy of the Library, and, in your modifications, a -facility refers to a function or data to be supplied by an Application -that uses the facility (other than as an argument passed when the -facility is invoked), then you may convey a copy of the modified -version: - - a) under this License, provided that you make a good faith effort to - ensure that, in the event an Application does not supply the - function or data, the facility still operates, and performs - whatever part of its purpose remains meaningful, or - - b) under the GNU GPL, with none of the additional permissions of - this License applicable to that copy. - - 3. Object Code Incorporating Material from Library Header Files. - - The object code form of an Application may incorporate material from -a header file that is part of the Library. You may convey such object -code under terms of your choice, provided that, if the incorporated -material is not limited to numerical parameters, data structure -layouts and accessors, or small macros, inline functions and templates -(ten or fewer lines in length), you do both of the following: - - a) Give prominent notice with each copy of the object code that the - Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the object code with a copy of the GNU GPL and this license - document. - - 4. Combined Works. - - You may convey a Combined Work under terms of your choice that, -taken together, effectively do not restrict modification of the -portions of the Library contained in the Combined Work and reverse -engineering for debugging such modifications, if you also do each of -the following: - - a) Give prominent notice with each copy of the Combined Work that - the Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the Combined Work with a copy of the GNU GPL and this license - document. - - c) For a Combined Work that displays copyright notices during - execution, include the copyright notice for the Library among - these notices, as well as a reference directing the user to the - copies of the GNU GPL and this license document. - - d) Do one of the following: - - 0) Convey the Minimal Corresponding Source under the terms of this - License, and the Corresponding Application Code in a form - suitable for, and under terms that permit, the user to - recombine or relink the Application with a modified version of - the Linked Version to produce a modified Combined Work, in the - manner specified by section 6 of the GNU GPL for conveying - Corresponding Source. - - 1) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (a) uses at run time - a copy of the Library already present on the user's computer - system, and (b) will operate properly with a modified version - of the Library that is interface-compatible with the Linked - Version. - - e) Provide Installation Information, but only if you would otherwise - be required to provide such information under section 6 of the - GNU GPL, and only to the extent that such information is - necessary to install and execute a modified version of the - Combined Work produced by recombining or relinking the - Application with a modified version of the Linked Version. (If - you use option 4d0, the Installation Information must accompany - the Minimal Corresponding Source and Corresponding Application - Code. If you use option 4d1, you must provide the Installation - Information in the manner specified by section 6 of the GNU GPL - for conveying Corresponding Source.) - - 5. Combined Libraries. - - You may place library facilities that are a work based on the -Library side by side in a single library together with other library -facilities that are not Applications and are not covered by this -License, and convey such a combined library under terms of your -choice, if you do both of the following: - - a) Accompany the combined library with a copy of the same work based - on the Library, uncombined with any other library facilities, - conveyed under the terms of this License. - - b) Give prominent notice with the combined library that part of it - is a work based on the Library, and explaining where to find the - accompanying uncombined form of the same work. - - 6. Revised Versions of the GNU Lesser General Public License. - - The Free Software Foundation may publish revised and/or new versions -of the GNU Lesser General Public License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. - - Each version is given a distinguishing version number. If the -Library as you received it specifies that a certain numbered version -of the GNU Lesser General Public License "or any later version" -applies to it, you have the option of following the terms and -conditions either of that published version or of any later version -published by the Free Software Foundation. If the Library as you -received it does not specify a version number of the GNU Lesser -General Public License, you may choose any version of the GNU Lesser -General Public License ever published by the Free Software Foundation. - - If the Library as you received it specifies that a proxy can decide -whether future versions of the GNU Lesser General Public License shall -apply, that proxy's public statement of acceptance of any version is -permanent authorization for you to choose that version for the -Library. \ No newline at end of file + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 6d015933230fdb4d3a5c512c570caea3b3ff7e13 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Aug 2024 11:47:55 +0100 Subject: [PATCH 019/145] chore: update keymanager spec tests to run against v1.1.0 (#7044) --- packages/api/test/unit/keymanager/oapiSpec.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/test/unit/keymanager/oapiSpec.test.ts b/packages/api/test/unit/keymanager/oapiSpec.test.ts index 10c29f0cea55..97011f5d3660 100644 --- a/packages/api/test/unit/keymanager/oapiSpec.test.ts +++ b/packages/api/test/unit/keymanager/oapiSpec.test.ts @@ -12,7 +12,7 @@ import {testData} from "./testData.js"; // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const version = "v1.0.0"; +const version = "v1.1.0"; const openApiFile: OpenApiFile = { url: `https://github.com/ethereum/keymanager-APIs/releases/download/${version}/keymanager-oapi.json`, filepath: path.join(__dirname, "../../../oapi-schemas/keymanager-oapi.json"), From 5d2e1a7228483af92930a32ce76277eee9977b01 Mon Sep 17 00:00:00 2001 From: Phil Ngo <58080811+philknows@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:04:16 -0400 Subject: [PATCH 020/145] chore: generate funding.json for OP RPGF (#7051) generate funding.json for OP RPGF --- funding.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 funding.json diff --git a/funding.json b/funding.json new file mode 100644 index 000000000000..872321cdca5b --- /dev/null +++ b/funding.json @@ -0,0 +1,5 @@ +{ + "opRetro": { + "projectId": "0x8ec88058175ef4c1c9b1f26910c4d4f2cfa733d6fcd1dbd9385476a313d9e12d" + } +} From d87e63729104bfb9d0925d936046ec4a057822e4 Mon Sep 17 00:00:00 2001 From: harkamal Date: Wed, 24 Jan 2024 17:38:11 +0530 Subject: [PATCH 021/145] feat: placeholder PR for electra add types stub and epoch config fix types --- .../beacon-node/src/chain/blocks/types.ts | 2 +- .../test/spec/presets/fork.test.ts | 2 + .../test/spec/presets/transition.test.ts | 8 + .../upgradeLightClientHeader.test.ts | 33 +++- .../test/unit/network/fork.test.ts | 13 +- packages/beacon-node/test/utils/config.ts | 8 + .../config/src/chainConfig/configs/mainnet.ts | 4 + .../config/src/chainConfig/configs/minimal.ts | 4 + packages/config/src/chainConfig/types.ts | 6 + packages/config/src/forkConfig/index.ts | 10 +- packages/light-client/src/spec/utils.ts | 4 + packages/params/src/forkName.ts | 2 + .../test/unit/upgradeState.test.ts | 8 + packages/types/src/electra/index.ts | 3 + packages/types/src/electra/sszTypes.ts | 148 ++++++++++++++++++ packages/types/src/electra/types.ts | 29 ++++ packages/types/src/sszTypes.ts | 4 +- packages/types/src/types.ts | 36 +++++ packages/validator/src/util/params.ts | 5 + 19 files changed, 323 insertions(+), 6 deletions(-) create mode 100644 packages/types/src/electra/index.ts create mode 100644 packages/types/src/electra/sszTypes.ts create mode 100644 packages/types/src/electra/types.ts diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index da573bb76334..5eb5eebc7840 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -36,7 +36,7 @@ export enum GossipedInputType { type BlobsCacheMap = Map; -type ForkBlobsInfo = {fork: ForkName.deneb}; +type ForkBlobsInfo = {fork: ForkName.deneb | ForkName.electra}; type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource}; export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData; export type BlockInputData = BlockInputDataBlobs; diff --git a/packages/beacon-node/test/spec/presets/fork.test.ts b/packages/beacon-node/test/spec/presets/fork.test.ts index 228ab6a38935..c880d24bbbe3 100644 --- a/packages/beacon-node/test/spec/presets/fork.test.ts +++ b/packages/beacon-node/test/spec/presets/fork.test.ts @@ -35,6 +35,8 @@ const fork: TestRunnerFn = (forkNext) => { return slotFns.upgradeStateToCapella(preState as CachedBeaconStateBellatrix); case ForkName.deneb: return slotFns.upgradeStateToDeneb(preState as CachedBeaconStateCapella); + case ForkName.electra: + throw Error("not Implemented"); } }, options: { diff --git a/packages/beacon-node/test/spec/presets/transition.test.ts b/packages/beacon-node/test/spec/presets/transition.test.ts index d9925f292677..cae7c667b590 100644 --- a/packages/beacon-node/test/spec/presets/transition.test.ts +++ b/packages/beacon-node/test/spec/presets/transition.test.ts @@ -102,6 +102,14 @@ function getTransitionConfig(fork: ForkName, forkEpoch: number): Partial${toFork}`, function () { + lcHeaderByFork[fromFork].beacon.slot = testSlots[fromFork]; + lcHeaderByFork[toFork].beacon.slot = testSlots[fromFork]; + + expect(() => { + upgradeLightClientHeader(config, toFork, lcHeaderByFork[fromFork]); + }).toThrow("Not Implemented"); + }); + } + + // Since electra is not implemented for loop is till deneb (Object.values(ForkName).length-1) + // Once electra is implemnted run for loop till Object.values(ForkName).length + + // for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { + + for (let i = ForkSeq.altair; i < Object.values(ForkName).length - 1; i++) { for (let j = i; j > 0; j--) { const fromFork = ForkName[ForkSeq[i] as ForkName]; const toFork = ForkName[ForkSeq[j] as ForkName]; diff --git a/packages/beacon-node/test/unit/network/fork.test.ts b/packages/beacon-node/test/unit/network/fork.test.ts index be748d2e8185..bbe1c0870d30 100644 --- a/packages/beacon-node/test/unit/network/fork.test.ts +++ b/packages/beacon-node/test/unit/network/fork.test.ts @@ -9,12 +9,14 @@ function getForkConfig({ bellatrix, capella, deneb, + electra, }: { phase0: number; altair: number; bellatrix: number; capella: number; deneb: number; + electra: number; }): BeaconConfig { const forks: Record = { phase0: { @@ -57,6 +59,14 @@ function getForkConfig({ prevVersion: Buffer.from([0, 0, 0, 3]), prevForkName: ForkName.capella, }, + electra: { + name: ForkName.electra, + seq: ForkSeq.electra, + epoch: electra, + version: Buffer.from([0, 0, 0, 5]), + prevVersion: Buffer.from([0, 0, 0, 4]), + prevForkName: ForkName.deneb, + }, }; const forksAscendingEpochOrder = Object.values(forks); const forksDescendingEpochOrder = Object.values(forks).reverse(); @@ -133,9 +143,10 @@ const testScenarios = [ for (const testScenario of testScenarios) { const {phase0, altair, bellatrix, capella, testCases} = testScenario; const deneb = Infinity; + const electra = Infinity; describe(`network / fork: phase0: ${phase0}, altair: ${altair}, bellatrix: ${bellatrix} capella: ${capella}`, () => { - const forkConfig = getForkConfig({phase0, altair, bellatrix, capella, deneb}); + const forkConfig = getForkConfig({phase0, altair, bellatrix, capella, deneb, electra}); const forks = forkConfig.forks; for (const testCase of testCases) { const {epoch, currentFork, nextFork, activeForks} = testCase; diff --git a/packages/beacon-node/test/utils/config.ts b/packages/beacon-node/test/utils/config.ts index 54c058d30722..2aad1c14c03e 100644 --- a/packages/beacon-node/test/utils/config.ts +++ b/packages/beacon-node/test/utils/config.ts @@ -31,5 +31,13 @@ export function getConfig(fork: ForkName, forkEpoch = 0): ChainForkConfig { CAPELLA_FORK_EPOCH: 0, DENEB_FORK_EPOCH: forkEpoch, }); + case ForkName.electra: + return createChainForkConfig({ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: forkEpoch, + }); } } diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 883688ca821b..0de1bee666ec 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -49,6 +49,10 @@ export const chainConfig: ChainConfig = { DENEB_FORK_VERSION: b("0x04000000"), DENEB_FORK_EPOCH: 269568, // March 13, 2024, 01:55:35pm UTC + // Electra + ELECTRA_FORK_VERSION: b("0x05000000"), + ELECTRA_FORK_EPOCH: Infinity, + // Time parameters // --------------------------------------------------------------- // 12 seconds diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index 23cd14e763ec..c99a76d1ee40 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -46,6 +46,10 @@ export const chainConfig: ChainConfig = { DENEB_FORK_VERSION: b("0x04000001"), DENEB_FORK_EPOCH: Infinity, + // Electra + ELECTRA_FORK_VERSION: b("0x05000001"), + ELECTRA_FORK_EPOCH: Infinity, + // Time parameters // --------------------------------------------------------------- // [customized] Faster for testing purposes diff --git a/packages/config/src/chainConfig/types.ts b/packages/config/src/chainConfig/types.ts index 45f05bfaa724..234a08558be5 100644 --- a/packages/config/src/chainConfig/types.ts +++ b/packages/config/src/chainConfig/types.ts @@ -40,6 +40,9 @@ export type ChainConfig = { // DENEB DENEB_FORK_VERSION: Uint8Array; DENEB_FORK_EPOCH: number; + // ELECTRA + ELECTRA_FORK_VERSION: Uint8Array; + ELECTRA_FORK_EPOCH: number; // Time parameters SECONDS_PER_SLOT: number; @@ -99,6 +102,9 @@ export const chainConfigTypes: SpecTypes = { // DENEB DENEB_FORK_VERSION: "bytes", DENEB_FORK_EPOCH: "number", + // ELECTRA + ELECTRA_FORK_VERSION: "bytes", + ELECTRA_FORK_EPOCH: "number", // Time parameters SECONDS_PER_SLOT: "number", diff --git a/packages/config/src/forkConfig/index.ts b/packages/config/src/forkConfig/index.ts index 16d8952548b8..54edc8b618e0 100644 --- a/packages/config/src/forkConfig/index.ts +++ b/packages/config/src/forkConfig/index.ts @@ -59,10 +59,18 @@ export function createForkConfig(config: ChainConfig): ForkConfig { prevVersion: config.CAPELLA_FORK_VERSION, prevForkName: ForkName.capella, }; + const electra: ForkInfo = { + name: ForkName.electra, + seq: ForkSeq.electra, + epoch: config.ELECTRA_FORK_EPOCH, + version: config.ELECTRA_FORK_VERSION, + prevVersion: config.DENEB_FORK_VERSION, + prevForkName: ForkName.deneb, + }; /** Forks in order order of occurence, `phase0` first */ // Note: Downstream code relies on proper ordering. - const forks = {phase0, altair, bellatrix, capella, deneb}; + const forks = {phase0, altair, bellatrix, capella, deneb, electra}; // Prevents allocating an array on every getForkInfo() call const forksAscendingEpochOrder = Object.values(forks); diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 65d6f3e84c59..aafd81c9250a 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -112,6 +112,10 @@ export function upgradeLightClientHeader( // Break if no further upgradation is required else fall through if (ForkSeq[targetFork] <= ForkSeq.deneb) break; + + // eslint-disable-next-line no-fallthrough + case ForkName.electra: + throw Error("Not Implemented"); } return upgradedHeader; } diff --git a/packages/params/src/forkName.ts b/packages/params/src/forkName.ts index a5f6d49d1cef..e1f1b2eda73a 100644 --- a/packages/params/src/forkName.ts +++ b/packages/params/src/forkName.ts @@ -7,6 +7,7 @@ export enum ForkName { bellatrix = "bellatrix", capella = "capella", deneb = "deneb", + electra = "electra", } /** @@ -18,6 +19,7 @@ export enum ForkSeq { bellatrix = 2, capella = 3, deneb = 4, + electra = 5, } function exclude(coll: T[], val: U[]): Exclude[] { diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index 2ea8eef182ac..75ba415c1bea 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -55,5 +55,13 @@ function getConfig(fork: ForkName, forkEpoch = 0): ChainForkConfig { CAPELLA_FORK_EPOCH: 0, DENEB_FORK_EPOCH: forkEpoch, }); + case ForkName.electra: + return createChainForkConfig({ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: forkEpoch, + }); } } diff --git a/packages/types/src/electra/index.ts b/packages/types/src/electra/index.ts new file mode 100644 index 000000000000..7856cd729620 --- /dev/null +++ b/packages/types/src/electra/index.ts @@ -0,0 +1,3 @@ +export * from "./types.js"; +export * as ts from "./types.js"; +export * as ssz from "./sszTypes.js"; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts new file mode 100644 index 000000000000..30690a499845 --- /dev/null +++ b/packages/types/src/electra/sszTypes.ts @@ -0,0 +1,148 @@ +import {ContainerType} from "@chainsafe/ssz"; +import {ssz as primitiveSsz} from "../primitive/index.js"; +import {ssz as denebSsz} from "../deneb/index.js"; + +const {BLSSignature} = primitiveSsz; + +export const ExecutionPayload = new ContainerType( + { + ...denebSsz.ExecutionPayload.fields, + }, + {typeName: "ExecutionPayload", jsonCase: "eth2"} +); + +export const ExecutionPayloadHeader = new ContainerType( + { + ...denebSsz.ExecutionPayloadHeader.fields, + }, + {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} +); + +export const BeaconBlockBody = new ContainerType( + { + ...denebSsz.BeaconBlockBody.fields, + }, + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const BeaconBlock = new ContainerType( + { + ...denebSsz.BeaconBlock.fields, + }, + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const SignedBeaconBlock = new ContainerType( + { + message: BeaconBlock, + signature: BLSSignature, + }, + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); + +export const BlobSidecar = new ContainerType( + { + ...denebSsz.BlobSidecar.fields, + }, + {typeName: "BlobSidecar", jsonCase: "eth2"} +); + +export const BlindedBeaconBlockBody = new ContainerType( + { + ...denebSsz.BlindedBeaconBlockBody.fields, + }, + {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const BlindedBeaconBlock = new ContainerType( + { + ...denebSsz.BlindedBeaconBlock.fields, + }, + {typeName: "BlindedBeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const SignedBlindedBeaconBlock = new ContainerType( + { + message: BlindedBeaconBlock, + signature: BLSSignature, + }, + {typeName: "SignedBlindedBeaconBlock", jsonCase: "eth2"} +); + +export const BuilderBid = new ContainerType( + { + ...denebSsz.BuilderBid.fields, + }, + {typeName: "BuilderBid", jsonCase: "eth2"} +); + +export const SignedBuilderBid = new ContainerType( + { + message: BuilderBid, + signature: BLSSignature, + }, + {typeName: "SignedBuilderBid", jsonCase: "eth2"} +); + +export const ExecutionPayloadAndBlobsBundle = new ContainerType( + { + ...denebSsz.ExecutionPayloadAndBlobsBundle.fields, + }, + {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} +); + +export const BeaconState = new ContainerType( + { + ...denebSsz.BeaconState.fields, + }, + {typeName: "BeaconState", jsonCase: "eth2"} +); + +export const LightClientHeader = new ContainerType( + { + ...denebSsz.LightClientHeader.fields, + }, + {typeName: "LightClientHeader", jsonCase: "eth2"} +); + +export const LightClientBootstrap = new ContainerType( + { + ...denebSsz.LightClientBootstrap.fields, + }, + {typeName: "LightClientBootstrap", jsonCase: "eth2"} +); + +export const LightClientUpdate = new ContainerType( + { + ...denebSsz.LightClientUpdate.fields, + }, + {typeName: "LightClientUpdate", jsonCase: "eth2"} +); + +export const LightClientFinalityUpdate = new ContainerType( + { + ...denebSsz.LightClientFinalityUpdate.fields, + }, + {typeName: "LightClientFinalityUpdate", jsonCase: "eth2"} +); + +export const LightClientOptimisticUpdate = new ContainerType( + { + ...denebSsz.LightClientOptimisticUpdate.fields, + }, + {typeName: "LightClientOptimisticUpdate", jsonCase: "eth2"} +); + +export const LightClientStore = new ContainerType( + { + ...denebSsz.LightClientStore.fields, + }, + {typeName: "LightClientStore", jsonCase: "eth2"} +); + +export const SSEPayloadAttributes = new ContainerType( + { + ...denebSsz.SSEPayloadAttributes.fields, + }, + {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} +); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts new file mode 100644 index 000000000000..198259eed1dd --- /dev/null +++ b/packages/types/src/electra/types.ts @@ -0,0 +1,29 @@ +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes.js"; + +export type BlobSidecar = ValueOf; +export type ExecutionPayloadAndBlobsBundle = ValueOf; + +export type ExecutionPayload = ValueOf; +export type ExecutionPayloadHeader = ValueOf; + +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; + +export type BeaconState = ValueOf; + +export type BlindedBeaconBlockBody = ValueOf; +export type BlindedBeaconBlock = ValueOf; +export type SignedBlindedBeaconBlock = ValueOf; + +export type BuilderBid = ValueOf; +export type SignedBuilderBid = ValueOf; +export type SSEPayloadAttributes = ValueOf; + +export type LightClientHeader = ValueOf; +export type LightClientBootstrap = ValueOf; +export type LightClientUpdate = ValueOf; +export type LightClientFinalityUpdate = ValueOf; +export type LightClientOptimisticUpdate = ValueOf; +export type LightClientStore = ValueOf; diff --git a/packages/types/src/sszTypes.ts b/packages/types/src/sszTypes.ts index 60980fa0822a..c2044edb5e98 100644 --- a/packages/types/src/sszTypes.ts +++ b/packages/types/src/sszTypes.ts @@ -5,9 +5,10 @@ import {ssz as altair} from "./altair/index.js"; import {ssz as bellatrix} from "./bellatrix/index.js"; import {ssz as capella} from "./capella/index.js"; import {ssz as deneb} from "./deneb/index.js"; +import {ssz as electra} from "./electra/index.js"; export * from "./primitive/sszTypes.js"; -export {phase0, altair, bellatrix, capella, deneb}; +export {phase0, altair, bellatrix, capella, deneb, electra}; /** * Index the ssz types that differ by fork @@ -19,6 +20,7 @@ const typesByFork = { [ForkName.bellatrix]: {...phase0, ...altair, ...bellatrix}, [ForkName.capella]: {...phase0, ...altair, ...bellatrix, ...capella}, [ForkName.deneb]: {...phase0, ...altair, ...bellatrix, ...capella, ...deneb}, + [ForkName.deneb]: {...phase0, ...altair, ...bellatrix, ...capella, ...deneb, ...electra}, }; /** diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index 3602299ae5e1..c0cc6591dd56 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -4,6 +4,7 @@ import {ts as altair} from "./altair/index.js"; import {ts as bellatrix} from "./bellatrix/index.js"; import {ts as capella} from "./capella/index.js"; import {ts as deneb} from "./deneb/index.js"; +import {ts as electra} from "./electra/index.js"; import {Slot} from "./primitive/types.js"; export * from "./primitive/types.js"; @@ -12,6 +13,7 @@ export {ts as altair} from "./altair/index.js"; export {ts as bellatrix} from "./bellatrix/index.js"; export {ts as capella} from "./capella/index.js"; export {ts as deneb} from "./deneb/index.js"; +export {ts as electra} from "./electra/index.js"; /** Common non-spec type to represent roots as strings */ export type RootHex = string; @@ -132,6 +134,40 @@ type TypesByFork = { SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; }; + [ForkName.electra]: { + BeaconBlockHeader: phase0.BeaconBlockHeader; + SignedBeaconBlockHeader: phase0.SignedBeaconBlockHeader; + BeaconBlock: electra.BeaconBlock; + BeaconBlockBody: electra.BeaconBlockBody; + BeaconState: electra.BeaconState; + SignedBeaconBlock: electra.SignedBeaconBlock; + Metadata: altair.Metadata; + LightClientHeader: electra.LightClientHeader; + LightClientBootstrap: electra.LightClientBootstrap; + LightClientUpdate: electra.LightClientUpdate; + LightClientFinalityUpdate: electra.LightClientFinalityUpdate; + LightClientOptimisticUpdate: electra.LightClientOptimisticUpdate; + LightClientStore: electra.LightClientStore; + BlindedBeaconBlock: electra.BlindedBeaconBlock; + BlindedBeaconBlockBody: electra.BlindedBeaconBlockBody; + SignedBlindedBeaconBlock: electra.SignedBlindedBeaconBlock; + ExecutionPayload: electra.ExecutionPayload; + ExecutionPayloadHeader: electra.ExecutionPayloadHeader; + BuilderBid: electra.BuilderBid; + SignedBuilderBid: electra.SignedBuilderBid; + SSEPayloadAttributes: electra.SSEPayloadAttributes; + BlockContents: {block: BeaconBlock; kzgProofs: deneb.KZGProofs; blobs: deneb.Blobs}; + SignedBlockContents: { + signedBlock: SignedBeaconBlock; + kzgProofs: deneb.KZGProofs; + blobs: deneb.Blobs; + }; + ExecutionPayloadAndBlobsBundle: deneb.ExecutionPayloadAndBlobsBundle; + BlobsBundle: deneb.BlobsBundle; + Contents: deneb.Contents; + SyncCommittee: altair.SyncCommittee; + SyncAggregate: altair.SyncAggregate; + }; }; export type TypesFor = K extends void diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 8ccaf9fe75ba..0afede39b951 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -73,6 +73,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Tue, 20 Feb 2024 02:51:21 +0800 Subject: [PATCH 022/145] feat: implement EIP-6110 (#6042) * Add immutable in the dependencies * Initial change to pubkeyCache * Added todos * Moved unfinalized cache to epochCache * Move populating finalized cache to afterProcessEpoch * Specify unfinalized cache during state cloning * Move from unfinalized to finalized cache in afterProcessEpoch * Confused myself * Clean up * Change logic * Fix cloning issue * Clean up redundant code * Add CarryoverData in epochCtx.createFromState * Fix typo * Update usage of pubkeyCache * Update pubkeyCache usage * Fix lint * Fix lint * Add 6110 to ChainConfig * Add 6110 to BeaconPreset * Define 6110 fork and container * Add V6110 api to execution engine * Update test * Add depositReceiptsRoot to process_execution_payload * State transitioning to EIP6110 * State transitioning to EIP6110 * Light client change in EIP-6110 * Update tests * produceBlock * Refactor processDeposit to match the spec * Implement processDepositReceipt * Implement 6110 fork guard for pubkeyCache * Handle changes in eth1 deposit * Update eth1 deposit test * Fix typo * Lint * Remove embarassing comments * Address comments * Modify applyDeposit signature * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/pubkeyCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Remove old code * Rename fields in epochCache and immutableData * Remove CarryoverData * Move isAfter6110 from var to method * Fix cyclic import * Fix operations spec runner * Fix for spec test * Fix spec test * state.depositReceiptsStartIndex to BigInt * getDeposit requires cached state * default depositReceiptsStartIndex value in genesis * Fix pubkeyCache bug * newUnfinalizedPubkeyIndexMap in createCachedBeaconState * Lint * Pass epochCache instead of pubkey2IndexFn in apis * Address comments * Add unit test on pubkey cache cloning * Add unfinalizedPubkeyCacheSize to metrics * Add unfinalizedPubkeyCacheSize to metrics * Clean up code * Add besu to el-interop * Add 6110 genesis file * Template for sim test * Add unit test for getEth1DepositCount * Update sim test * Update besudocker * Finish beacon api calls in sim test * Update epochCache.createFromState() * Fix bug unfinalized validators are not finalized * Add sim test to run a few blocks * Lint * Merge branch 'unstable' into 611 * Add more check to sim test * Update besu docker image instruction * Update sim test with correct tx * Address comment + cleanup * Clean up code * Properly handle promise rejection * Lint * Update packages/beacon-node/src/execution/engine/types.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update comments * Accept type undefined in ExecutionPayloadBodyRpc * Update comment and semantic * Remove if statement when adding finalized validator * Comment on repeated insert on finalized cache * rename createFromState * Add comment on getPubkey() * Stash change to reduce diffs * Stash change to reduce diffs * Lint * addFinalizedPubkey on finalized checkpoint * Update comment * Use OrderedMap for unfinalized cache * Pull out logic of deleting pubkeys for batch op * Add updateUnfinalizedPubkeys in regen * Update updateUnfinalizedPubkeys logic * Add comment * Add metrics for state context caches * Address comment * Address comment * Deprecate eth1Data polling when condition is reached * Fix conflicts * Fix sim test * Lint * Fix type * Fix test * Fix test * Lint * Update packages/light-client/src/spec/utils.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Fix spec test * Address comments * Improve cache logic on checkpoint finalized * Update sim test according to new cache logic * Update comment * Lint * Finalized pubkey cache only update once per checkpoint * Add perf test for updateUnfinalizedPubkeys * Add perf test for updateUnfinalizedPubkeys * Tweak params for perf test * Freeze besu docker image version for 6110 * Add benchmark result * Use Map instead of OrderedMap. Update benchmark * Minor optimization * Minor optimization * Add memory test for immutable.js * Update test * Reduce code duplication * Lint * Remove try/catch in updateUnfinalizedPubkeys * Introduce EpochCache metric * Add historicalValidatorLengths * Polish code * Migrate state-transition unit tests to vitest * Fix calculation of pivot index * `historicalValidatorLengths` only activate post 6110 * Update sim test * Lint * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Improve readability on historicalValidatorLengths * Update types * Fix calculation * Add eth1data poll todo * Add epochCache.getValidatorCountAtEpoch * Add todo * Add getStateIterator for state cache * Partial commit * Update perf test * updateUnfinalizedPubkeys directly modify states from regen * Update sim test. Lint * Add todo * some improvements and a fix for effectiveBalanceIncrements fork safeness * rename eip6110 to elctra * fix electra-interop.test.ts --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Co-authored-by: gajinder lint and tsc small cleanup fix rebase issue --- packages/beacon-node/src/chain/chain.ts | 37 ++ .../beacon-node/src/chain/regen/queued.ts | 50 +- .../chain/stateCache/blockStateCacheImpl.ts | 6 +- .../chain/stateCache/fifoBlockStateCache.ts | 4 + .../stateCache/inMemoryCheckpointsCache.ts | 4 + .../stateCache/persistentCheckpointsCache.ts | 4 + .../beacon-node/src/chain/stateCache/types.ts | 2 + .../src/eth1/eth1DepositDataTracker.ts | 32 +- packages/beacon-node/src/eth1/index.ts | 8 + packages/beacon-node/src/eth1/interface.ts | 5 + .../beacon-node/src/eth1/utils/deposits.ts | 16 +- .../beacon-node/src/execution/engine/http.ts | 10 +- .../src/execution/engine/interface.ts | 4 +- .../beacon-node/src/execution/engine/mock.ts | 4 + .../src/execution/engine/payloadIdCache.ts | 8 + .../beacon-node/src/execution/engine/types.ts | 68 ++- .../beacon-node/src/metrics/metrics/beacon.ts | 7 + .../src/metrics/metrics/lodestar.ts | 31 ++ .../test/memory/unfinalizedPubkey2Index.ts | 54 +++ .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../test/perf/chain/opPools/opPool.test.ts | 2 +- .../produceBlock/produceBlockBody.test.ts | 2 +- .../updateUnfinalizedPubkeys.test.ts | 110 +++++ .../scripts/el-interop/besu/common-setup.sh | 19 + .../test/scripts/el-interop/besu/electra.tmpl | 77 +++ .../scripts/el-interop/besu/post-merge.sh | 8 + .../el-interop/besudocker/common-setup.sh | 22 + .../el-interop/besudocker/electra.tmpl | 77 +++ .../el-interop/besudocker/post-merge.sh | 8 + .../test/sim/electra-interop.test.ts | 457 ++++++++++++++++++ .../test/spec/presets/fork.test.ts | 3 +- .../test/spec/presets/genesis.test.ts | 2 +- .../test/spec/presets/operations.test.ts | 9 +- .../test/spec/presets/ssz_static.test.ts | 1 + .../test/spec/utils/specTestIterator.ts | 1 - .../upgradeLightClientHeader.test.ts | 34 +- .../opPools/aggregatedAttestationPool.test.ts | 4 +- .../test/unit/chain/shufflingCache.test.ts | 8 +- .../stateCache/fifoBlockStateCache.test.ts | 3 +- .../test/unit/eth1/utils/deposits.test.ts | 52 +- .../test/unit/executionEngine/http.test.ts | 4 + .../beaconBlocksMaybeBlobsByRange.test.ts | 1 + packages/beacon-node/test/utils/state.ts | 23 +- .../test/utils/validationData/attestation.ts | 7 +- .../config/src/chainConfig/configs/mainnet.ts | 2 +- .../config/src/chainConfig/configs/minimal.ts | 3 +- packages/light-client/src/spec/utils.ts | 12 +- packages/params/src/index.ts | 5 + packages/params/src/presets/mainnet.ts | 3 + packages/params/src/presets/minimal.ts | 3 + packages/params/src/types.ts | 6 + packages/state-transition/package.json | 7 +- .../src/block/processDeposit.ts | 110 +++-- .../src/block/processDepositReceipt.ts | 17 + .../src/block/processOperations.ts | 17 +- .../state-transition/src/cache/epochCache.ts | 182 ++++++- .../state-transition/src/cache/pubkeyCache.ts | 22 +- .../state-transition/src/cache/stateCache.ts | 5 +- packages/state-transition/src/cache/types.ts | 1 + packages/state-transition/src/index.ts | 9 +- packages/state-transition/src/metrics.ts | 5 + .../src/signatureSets/attesterSlashings.ts | 3 +- packages/state-transition/src/slot/index.ts | 1 + .../src/slot/upgradeStateToElectra.ts | 33 ++ .../state-transition/src/stateTransition.ts | 5 + packages/state-transition/src/types.ts | 2 + packages/state-transition/src/util/deposit.ts | 24 + .../state-transition/src/util/execution.ts | 6 + packages/state-transition/src/util/genesis.ts | 12 + packages/state-transition/src/util/index.ts | 1 + .../test/unit/cachedBeaconState.test.ts | 62 ++- .../test/unit/upgradeState.test.ts | 16 + .../test/unit/util/deposit.test.ts | 99 ++++ packages/types/package.json | 3 + packages/types/src/electra/sszTypes.ts | 147 +++++- packages/types/src/electra/types.ts | 8 +- packages/types/src/primitive/sszTypes.ts | 1 + packages/validator/src/util/params.ts | 4 +- yarn.lock | 5 + 79 files changed, 1960 insertions(+), 171 deletions(-) create mode 100644 packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts create mode 100644 packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts create mode 100755 packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh create mode 100644 packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl create mode 100755 packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh create mode 100644 packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh create mode 100644 packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl create mode 100755 packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh create mode 100644 packages/beacon-node/test/sim/electra-interop.test.ts create mode 100644 packages/state-transition/src/block/processDepositReceipt.ts create mode 100644 packages/state-transition/src/slot/upgradeStateToElectra.ts create mode 100644 packages/state-transition/src/util/deposit.ts create mode 100644 packages/state-transition/test/unit/util/deposit.test.ts diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 75608ab33b2e..9f67857fe530 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1027,6 +1027,9 @@ export class BeaconChain implements IBeaconChain { metrics.forkChoice.balancesLength.set(forkChoiceMetrics.balancesLength); metrics.forkChoice.nodes.set(forkChoiceMetrics.nodes); metrics.forkChoice.indices.set(forkChoiceMetrics.indices); + + const headState = this.getHeadState(); + metrics.headState.unfinalizedPubkeyCacheSize.set(headState.epochCtx.unfinalizedPubkey2index.size); } private onClockSlot(slot: Slot): void { @@ -1115,6 +1118,40 @@ export class BeaconChain implements IBeaconChain { if (headState) { this.opPool.pruneAll(headBlock, headState); } + + const cpEpoch = cp.epoch; + const electraEpoch = headState?.config.ELECTRA_FORK_EPOCH ?? Infinity; + + if (headState === null) { + this.logger.verbose("Head state is null"); + } else if (cpEpoch >= electraEpoch) { + // Get the validator.length from the state at cpEpoch + // We are confident the last element in the list is from headEpoch + // Thus we query from the end of the list. (cpEpoch - headEpoch - 1) is negative number + const pivotValidatorIndex = headState.epochCtx.getValidatorCountAtEpoch(cpEpoch); + + if (pivotValidatorIndex !== undefined) { + // Note EIP-6914 will break this logic + const newFinalizedValidators = headState.epochCtx.unfinalizedPubkey2index.filter( + (index, _pubkey) => index < pivotValidatorIndex + ); + + // Populate finalized pubkey cache and remove unfinalized pubkey cache + if (!newFinalizedValidators.isEmpty()) { + this.regen.updateUnfinalizedPubkeys(newFinalizedValidators); + } + } + } + + // TODO-Electra: Deprecating eth1Data poll requires a check on a finalized checkpoint state. + // Will resolve this later + // if (cpEpoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity)) { + // // finalizedState can be safely casted to Electra state since cp is already post-Electra + // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositReceiptsStartIndex) { + // // Signal eth1 to stop polling eth1Data + // this.eth1.stopPollingEth1Data(); + // } + // } } async updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise { diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 082ffbe271e4..84907ea163af 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -1,6 +1,6 @@ import {phase0, Slot, RootHex, Epoch, BeaconBlock} from "@lodestar/types"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks, UnfinalizedPubkeyIndexMap, computeEpochAtSlot} from "@lodestar/state-transition"; import {Logger, toRootHex} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {CheckpointHex, toCheckpointHex} from "../stateCache/index.js"; @@ -206,6 +206,54 @@ export class QueuedStateRegenerator implements IStateRegenerator { return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch); } + /** + * Remove `validators` from all unfinalized cache's epochCtx.UnfinalizedPubkey2Index, + * and add them to epochCtx.pubkey2index and epochCtx.index2pubkey + */ + updateUnfinalizedPubkeys(validators: UnfinalizedPubkeyIndexMap): void { + let numStatesUpdated = 0; + const states = this.stateCache.getStates(); + const cpStates = this.checkpointStateCache.getStates(); + + // Add finalized pubkeys to all states. + const addTimer = this.metrics?.regenFnAddPubkeyTime.startTimer(); + + // We only need to add pubkeys to any one of the states since the finalized caches is shared globally across all states + const firstState = (states.next().value ?? cpStates.next().value) as CachedBeaconStateAllForks | undefined; + + if (firstState !== undefined) { + firstState.epochCtx.addFinalizedPubkeys(validators, this.metrics?.epochCache ?? undefined); + } else { + this.logger.warn("Attempt to delete finalized pubkey from unfinalized pubkey cache. But no state is available"); + } + + addTimer?.(); + + // Delete finalized pubkeys from unfinalized pubkey cache for all states + const deleteTimer = this.metrics?.regenFnDeletePubkeyTime.startTimer(); + const pubkeysToDelete = Array.from(validators.keys()); + + for (const s of states) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + numStatesUpdated++; + } + + for (const s of cpStates) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + numStatesUpdated++; + } + + // Since first state is consumed from the iterator. Will need to perform delete explicitly + if (firstState !== undefined) { + firstState?.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + numStatesUpdated++; + } + + deleteTimer?.(); + + this.metrics?.regenFnNumStatesUpdated.observe(numStatesUpdated); + } + /** * Get the state to run with `block`. * - State after `block.parentRoot` dialed forward to block.slot diff --git a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts index 05073d0e515f..1cb67cd6cf09 100644 --- a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts +++ b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts @@ -34,7 +34,7 @@ export class BlockStateCacheImpl implements BlockStateCache { this.maxStates = maxStates; this.cache = new MapTracker(metrics?.stateCache); if (metrics) { - this.metrics = metrics.stateCache; + this.metrics = {...metrics.stateCache, ...metrics.epochCache}; metrics.stateCache.size.addCollect(() => metrics.stateCache.size.set(this.cache.size)); } } @@ -137,6 +137,10 @@ export class BlockStateCacheImpl implements BlockStateCache { })); } + getStates(): IterableIterator { + return this.cache.values(); + } + private deleteAllEpochItems(epoch: Epoch): void { for (const rootHex of this.epochIndex.get(epoch) || []) { this.cache.delete(rootHex); diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 602b0abaee8d..f115806874f8 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -190,6 +190,10 @@ export class FIFOBlockStateCache implements BlockStateCache { })); } + getStates(): IterableIterator { + throw new Error("Method not implemented."); + } + /** * For unit test only. */ diff --git a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts index 03cdc84de166..38aeabb97955 100644 --- a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts @@ -176,6 +176,10 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { })); } + getStates(): IterableIterator { + return this.cache.values(); + } + /** ONLY FOR DEBUGGING PURPOSES. For spec tests on error */ dumpCheckpointKeys(): string[] { return Array.from(this.cache.keys()); diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 5c5901583ad8..c59c15de616d 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -594,6 +594,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { }); } + getStates(): IterableIterator { + throw new Error("Method not implemented."); + } + /** ONLY FOR DEBUGGING PURPOSES. For spec tests on error */ dumpCheckpointKeys(): string[] { return Array.from(this.cache.keys()); diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index 41e9b91aaa43..ad93def481e7 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -33,6 +33,7 @@ export interface BlockStateCache { prune(headStateRootHex: RootHex): void; deleteAllBeforeEpoch(finalizedEpoch: Epoch): void; dumpSummary(): routes.lodestar.StateCacheItem[]; + getStates(): IterableIterator; // Expose beacon states stored in cache. Use with caution } /** @@ -74,6 +75,7 @@ export interface CheckpointStateCache { processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise; clear(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; + getStates(): IterableIterator; // Expose beacon states stored in cache. Use with caution } export enum CacheItemType { diff --git a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts index f36f70abbbc4..c0b3ab35a73a 100644 --- a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts +++ b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts @@ -1,6 +1,11 @@ import {phase0, ssz} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; -import {BeaconStateAllForks, becomesNewEth1Data} from "@lodestar/state-transition"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + CachedBeaconStateElectra, + becomesNewEth1Data, +} from "@lodestar/state-transition"; import {ErrorAborted, TimeoutError, fromHex, Logger, isErrorAborted, sleep} from "@lodestar/utils"; import {IBeaconDb} from "../db/index.js"; @@ -67,6 +72,8 @@ export class Eth1DepositDataTracker { /** Dynamically adjusted batch size to fetch deposit logs */ private eth1GetLogsBatchSizeDynamic = MAX_BLOCKS_PER_LOG_QUERY; private readonly forcedEth1DataVote: phase0.Eth1Data | null; + /** To stop `runAutoUpdate()` in addition to AbortSignal */ + private stopPolling: boolean; constructor( opts: Eth1Options, @@ -81,6 +88,8 @@ export class Eth1DepositDataTracker { this.depositsCache = new Eth1DepositsCache(opts, config, db); this.eth1DataCache = new Eth1DataCache(config, db); this.eth1FollowDistance = config.ETH1_FOLLOW_DISTANCE; + // TODO Electra: fix scenario where node starts post-Electra and `stopPolling` will always be false + this.stopPolling = false; this.forcedEth1DataVote = opts.forcedEth1DataVote ? ssz.phase0.Eth1Data.deserialize(fromHex(opts.forcedEth1DataVote)) @@ -109,10 +118,22 @@ export class Eth1DepositDataTracker { } } + // TODO Electra: Figure out how an elegant way to stop eth1data polling + stopPollingEth1Data(): void { + this.stopPolling = true; + } + /** * Return eth1Data and deposits ready for block production for a given state */ - async getEth1DataAndDeposits(state: BeaconStateAllForks): Promise { + async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { + if ( + state.epochCtx.isAfterElectra() && + state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositReceiptsStartIndex + ) { + // No need to poll eth1Data since Electra deprecates the mechanism after depositReceiptsStartIndex is reached + return {eth1Data: state.eth1Data, deposits: []}; + } const eth1Data = this.forcedEth1DataVote ?? (await this.getEth1Data(state)); const deposits = await this.getDeposits(state, eth1Data); return {eth1Data, deposits}; @@ -141,7 +162,10 @@ export class Eth1DepositDataTracker { * Returns deposits to be included for a given state and eth1Data vote. * Requires internal caches to be updated regularly to return good results */ - private async getDeposits(state: BeaconStateAllForks, eth1DataVote: phase0.Eth1Data): Promise { + private async getDeposits( + state: CachedBeaconStateAllForks, + eth1DataVote: phase0.Eth1Data + ): Promise { // No new deposits have to be included, continue if (eth1DataVote.depositCount === state.eth1DepositIndex) { return []; @@ -162,7 +186,7 @@ export class Eth1DepositDataTracker { private async runAutoUpdate(): Promise { let lastRunMs = 0; - while (!this.signal.aborted) { + while (!this.signal.aborted && !this.stopPolling) { lastRunMs = Date.now(); try { diff --git a/packages/beacon-node/src/eth1/index.ts b/packages/beacon-node/src/eth1/index.ts index 9fdba90258a2..a8ba55c54141 100644 --- a/packages/beacon-node/src/eth1/index.ts +++ b/packages/beacon-node/src/eth1/index.ts @@ -106,6 +106,10 @@ export class Eth1ForBlockProduction implements IEth1ForBlockProduction { startPollingMergeBlock(): void { return this.eth1MergeBlockTracker.startPollingMergeBlock(); } + + stopPollingEth1Data(): void { + return this.eth1DepositDataTracker?.stopPollingEth1Data(); + } } /** @@ -140,4 +144,8 @@ export class Eth1ForBlockProductionDisabled implements IEth1ForBlockProduction { startPollingMergeBlock(): void { // Ignore } + + stopPollingEth1Data(): void { + // Ignore + } } diff --git a/packages/beacon-node/src/eth1/interface.ts b/packages/beacon-node/src/eth1/interface.ts index fc9626eb5b8a..898041ac8947 100644 --- a/packages/beacon-node/src/eth1/interface.ts +++ b/packages/beacon-node/src/eth1/interface.ts @@ -62,6 +62,11 @@ export interface IEth1ForBlockProduction { * - head state not isMergeTransitionComplete */ startPollingMergeBlock(): void; + + /** + * Should stop polling eth1Data after a Electra block is finalized AND deposit_receipts_start_index is reached + */ + stopPollingEth1Data(): void; } /** Different Eth1Block from phase0.Eth1Block with blockHash */ diff --git a/packages/beacon-node/src/eth1/utils/deposits.ts b/packages/beacon-node/src/eth1/utils/deposits.ts index 24916264e6d8..8d0331fc01d6 100644 --- a/packages/beacon-node/src/eth1/utils/deposits.ts +++ b/packages/beacon-node/src/eth1/utils/deposits.ts @@ -1,9 +1,9 @@ import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; -import {MAX_DEPOSITS} from "@lodestar/params"; -import {BeaconStateAllForks} from "@lodestar/state-transition"; +import {toRootHex} from "@lodestar/utils"; +import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {phase0, ssz} from "@lodestar/types"; import {FilterOptions} from "@lodestar/db"; -import {toRootHex} from "@lodestar/utils"; +import {getEth1DepositCount} from "@lodestar/state-transition"; import {Eth1Error, Eth1ErrorCode} from "../errors.js"; import {DepositTree} from "../../db/repositories/depositDataRoot.js"; @@ -11,7 +11,7 @@ export type DepositGetter = (indexRange: FilterOptions, eth1Data: pha export async function getDeposits( // eth1_deposit_index represents the next deposit index to be added - state: BeaconStateAllForks, + state: CachedBeaconStateAllForks, eth1Data: phase0.Eth1Data, depositsGetter: DepositGetter ): Promise { @@ -22,9 +22,11 @@ export async function getDeposits( throw new Eth1Error({code: Eth1ErrorCode.DEPOSIT_INDEX_TOO_HIGH, depositIndex, depositCount}); } - // Spec v0.12.2 - // assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - const depositsLen = Math.min(MAX_DEPOSITS, depositCount - depositIndex); + const depositsLen = getEth1DepositCount(state, eth1Data); + + if (depositsLen === 0) { + return []; // If depositsLen === 0, we can return early since no deposit with be returned from depositsGetter + } const indexRange = {gte: depositIndex, lt: depositIndex + depositsLen}; const deposits = await depositsGetter(indexRange, eth1Data); diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index c64a9715589f..d29387794327 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -198,7 +198,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { parentBlockRoot?: Root ): Promise { const method = - ForkSeq[fork] >= ForkSeq.deneb + ForkSeq[fork] >= ForkSeq.electra + ? "engine_newPayloadV6110" + : ForkSeq[fork] >= ForkSeq.deneb ? "engine_newPayloadV3" : ForkSeq[fork] >= ForkSeq.capella ? "engine_newPayloadV2" @@ -218,7 +220,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { const serializedVersionedHashes = serializeVersionedHashes(versionedHashes); const parentBeaconBlockRoot = serializeBeaconBlockRoot(parentBlockRoot); - const method = "engine_newPayloadV3"; + const method = ForkSeq[fork] >= ForkSeq.electra ? "engine_newPayloadV6110" : "engine_newPayloadV3"; engineRequest = { method, params: [serializedExecutionPayload, serializedVersionedHashes, parentBeaconBlockRoot], @@ -392,7 +394,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { shouldOverrideBuilder?: boolean; }> { const method = - ForkSeq[fork] >= ForkSeq.deneb + ForkSeq[fork] >= ForkSeq.electra + ? "engine_getPayloadV6110" + : ForkSeq[fork] >= ForkSeq.deneb ? "engine_getPayloadV3" : ForkSeq[fork] >= ForkSeq.capella ? "engine_getPayloadV2" diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index fa1da210cd12..91a8ce1cffe3 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -3,10 +3,10 @@ import {KZGCommitment, Blob, KZGProof} from "@lodestar/types/deneb"; import {Root, RootHex, capella, Wei, ExecutionPayload} from "@lodestar/types"; import {DATA} from "../../eth1/provider/utils.js"; -import {PayloadIdCache, PayloadId, WithdrawalV1} from "./payloadIdCache.js"; +import {PayloadIdCache, PayloadId, WithdrawalV1, DepositReceiptV1} from "./payloadIdCache.js"; import {ExecutionPayloadBody} from "./types.js"; -export {PayloadIdCache, type PayloadId, type WithdrawalV1}; +export {PayloadIdCache, type PayloadId, type WithdrawalV1, type DepositReceiptV1}; export enum ExecutionPayloadStatus { /** given payload is valid */ diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index a99a76508df8..5a50f1697de9 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -35,6 +35,7 @@ export type ExecutionEngineMockOpts = { onlyPredefinedResponses?: boolean; capellaForkTimestamp?: number; denebForkTimestamp?: number; + electraForkTimestamp?: number; }; type ExecutionBlock = { @@ -88,12 +89,14 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_newPayloadV1: this.notifyNewPayload.bind(this), engine_newPayloadV2: this.notifyNewPayload.bind(this), engine_newPayloadV3: this.notifyNewPayload.bind(this), + engine_newPayloadV6110: this.notifyNewPayload.bind(this), engine_forkchoiceUpdatedV1: this.notifyForkchoiceUpdate.bind(this), engine_forkchoiceUpdatedV2: this.notifyForkchoiceUpdate.bind(this), engine_forkchoiceUpdatedV3: this.notifyForkchoiceUpdate.bind(this), engine_getPayloadV1: this.getPayload.bind(this), engine_getPayloadV2: this.getPayload.bind(this), engine_getPayloadV3: this.getPayload.bind(this), + engine_getPayloadV6110: this.getPayload.bind(this), engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this), engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this), engine_getClientVersionV1: this.getClientVersionV1.bind(this), @@ -394,6 +397,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { } private timestampToFork(timestamp: number): ForkExecution { + if (timestamp > (this.opts.electraForkTimestamp ?? Infinity)) return ForkName.electra; if (timestamp > (this.opts.denebForkTimestamp ?? Infinity)) return ForkName.deneb; if (timestamp > (this.opts.capellaForkTimestamp ?? Infinity)) return ForkName.capella; return ForkName.bellatrix; diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index ea37e0922e9c..e5baa9fba92d 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -18,6 +18,14 @@ export type WithdrawalV1 = { amount: QUANTITY; }; +export type DepositReceiptV1 = { + pubkey: DATA; + withdrawalCredentials: DATA; + amount: QUANTITY; + signature: DATA; + index: QUANTITY; +}; + type FcuAttributes = {headBlockHash: DATA; finalizedBlockHash: DATA} & Omit; export class PayloadIdCache { diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 85f514c953b0..999c77f86b9c 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -1,4 +1,4 @@ -import {capella, deneb, Wei, bellatrix, Root, ExecutionPayload} from "@lodestar/types"; +import {capella, deneb, electra, Wei, bellatrix, Root, ExecutionPayload} from "@lodestar/types"; import { BYTES_PER_LOGS_BLOOM, FIELD_ELEMENTS_PER_BLOB, @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1} from "./payloadIdCache.js"; +import {WithdrawalV1, DepositReceiptV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -28,6 +28,7 @@ export type EngineApiRpcParamTypes = { engine_newPayloadV1: [ExecutionPayloadRpc]; engine_newPayloadV2: [ExecutionPayloadRpc]; engine_newPayloadV3: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; + engine_newPayloadV6110: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; /** * 1. Object - Payload validity status with respect to the consensus rules: * - blockHash: DATA, 32 Bytes - block hash value of the payload @@ -51,6 +52,7 @@ export type EngineApiRpcParamTypes = { engine_getPayloadV1: [QUANTITY]; engine_getPayloadV2: [QUANTITY]; engine_getPayloadV3: [QUANTITY]; + engine_getPayloadV6110: [QUANTITY]; /** * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure @@ -83,6 +85,7 @@ export type EngineApiRpcReturnTypes = { engine_newPayloadV1: PayloadStatus; engine_newPayloadV2: PayloadStatus; engine_newPayloadV3: PayloadStatus; + engine_newPayloadV6110: PayloadStatus; engine_forkchoiceUpdatedV1: { payloadStatus: PayloadStatus; payloadId: QUANTITY | null; @@ -101,6 +104,7 @@ export type EngineApiRpcReturnTypes = { engine_getPayloadV1: ExecutionPayloadRpc; engine_getPayloadV2: ExecutionPayloadResponse; engine_getPayloadV3: ExecutionPayloadResponse; + engine_getPayloadV6110: ExecutionPayloadResponse; engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[]; @@ -118,9 +122,17 @@ type ExecutionPayloadRpcWithValue = { }; type ExecutionPayloadResponse = ExecutionPayloadRpc | ExecutionPayloadRpcWithValue; -export type ExecutionPayloadBodyRpc = {transactions: DATA[]; withdrawals: WithdrawalV1[] | null}; +export type ExecutionPayloadBodyRpc = { + transactions: DATA[]; + withdrawals: WithdrawalV1[] | null | undefined; + depositReceipts: DepositReceiptV1[] | null | undefined; +}; -export type ExecutionPayloadBody = {transactions: bellatrix.Transaction[]; withdrawals: capella.Withdrawals | null}; +export type ExecutionPayloadBody = { + transactions: bellatrix.Transaction[]; + withdrawals: capella.Withdrawals | null; + depositReceipts: electra.DepositReceipts | null; +}; export type ExecutionPayloadRpc = { parentHash: DATA; // 32 bytes @@ -141,6 +153,7 @@ export type ExecutionPayloadRpc = { blobGasUsed?: QUANTITY; // DENEB excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB + depositReceipts?: DepositReceiptRpc[]; // ELECTRA }; export type WithdrawalRpc = { @@ -150,6 +163,14 @@ export type WithdrawalRpc = { amount: QUANTITY; }; +export type DepositReceiptRpc = { + pubkey: DATA; + withdrawalCredentials: DATA; + amount: QUANTITY; + signature: DATA; + index: QUANTITY; +}; + export type VersionedHashesRpc = DATA[]; export type PayloadAttributesRpc = { @@ -212,6 +233,12 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload payload.excessBlobGas = numToQuantity(excessBlobGas); } + // ELECTRA adds depositReceipts to the ExecutionPayload + if (ForkSeq[fork] >= ForkSeq.electra) { + const {depositReceipts} = data as electra.ExecutionPayload; + payload.depositReceipts = depositReceipts.map(serializeDepositReceipt); + } + return payload; } @@ -297,6 +324,17 @@ export function parseExecutionPayload( (executionPayload as deneb.ExecutionPayload).excessBlobGas = quantityToBigint(excessBlobGas); } + if (ForkSeq[fork] >= ForkSeq.electra) { + const {depositReceipts} = data; + // Geth can also reply with null + if (depositReceipts == null) { + throw Error( + `depositReceipts missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` + ); + } + (executionPayload as electra.ExecutionPayload).depositReceipts = depositReceipts.map(deserializeDepositReceipts); + } + return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; } @@ -363,11 +401,32 @@ export function deserializeWithdrawal(serialized: WithdrawalRpc): capella.Withdr } as capella.Withdrawal; } +export function serializeDepositReceipt(depositReceipt: electra.DepositReceipt): DepositReceiptRpc { + return { + pubkey: bytesToData(depositReceipt.pubkey), + withdrawalCredentials: bytesToData(depositReceipt.withdrawalCredentials), + amount: numToQuantity(depositReceipt.amount), + signature: bytesToData(depositReceipt.signature), + index: numToQuantity(depositReceipt.index), + }; +} + +export function deserializeDepositReceipts(serialized: DepositReceiptRpc): electra.DepositReceipt { + return { + pubkey: dataToBytes(serialized.pubkey, 48), + withdrawalCredentials: dataToBytes(serialized.withdrawalCredentials, 32), + amount: quantityToNum(serialized.amount), + signature: dataToBytes(serialized.signature, 96), + index: quantityToNum(serialized.index), + } as electra.DepositReceipt; +} + export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | null): ExecutionPayloadBody | null { return data ? { transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, + depositReceipts: data.depositReceipts ? data.depositReceipts.map(deserializeDepositReceipts) : null, } : null; } @@ -377,6 +436,7 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) ? { transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, + depositReceipts: data.depositReceipts ? data.depositReceipts.map(serializeDepositReceipt) : null, } : null; } diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index 141121de9079..96bb6bea4174 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -120,6 +120,13 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { }), }, + headState: { + unfinalizedPubkeyCacheSize: register.gauge({ + name: "head_state_unfinalized_pubkey_cache_size", + help: "Current size of the unfinalizedPubkey2Index cache in the head state", + }), + }, + parentBlockDistance: register.histogram({ name: "beacon_imported_block_parent_distance", help: "Histogram of distance to parent block of valid imported blocks", diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index f43a3f1cdbe6..595e4a7a45fc 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -374,6 +374,17 @@ export function createLodestarMetrics( help: "Total count state.validators nodesPopulated is false on stfn for post state", }), + epochCache: { + finalizedPubkeyDuplicateInsert: register.gauge({ + name: "lodestar_epoch_cache_finalized_pubkey_duplicate_insert", + help: "Total count of duplicate insert of finalized pubkeys", + }), + newUnFinalizedPubkey: register.gauge({ + name: "lodestar_epoch_cache_new_unfinalized_pubkey", + help: "Total count of unfinalized pubkeys added", + }), + }, + // BLS verifier thread pool and queue bls: { @@ -1205,6 +1216,11 @@ export function createLodestarMetrics( help: "Histogram of time to serialize state to db", buckets: [0.1, 0.5, 1, 2, 3, 4], }), + numStatesUpdated: register.histogram({ + name: "lodestar_cp_state_cache_state_updated_count", + help: "Histogram of number of state cache items updated every time removing and adding pubkeys to pubkey cache", + buckets: [1, 2, 5, 10, 50, 250], + }), statePruneFromMemoryCount: register.gauge({ name: "lodestar_cp_state_cache_state_prune_from_memory_count", help: "Total number of states pruned from memory", @@ -1373,6 +1389,21 @@ export function createLodestarMetrics( help: "regen function total errors", labelNames: ["entrypoint", "caller"], }), + regenFnAddPubkeyTime: register.histogram({ + name: "lodestar_regen_fn_add_pubkey_time_seconds", + help: "Historgram of time spent on adding pubkeys to all state cache items in seconds", + buckets: [0.01, 0.1, 0.5, 1, 2, 5], + }), + regenFnDeletePubkeyTime: register.histogram({ + name: "lodestar_regen_fn_delete_pubkey_time_seconds", + help: "Histrogram of time spent on deleting pubkeys from all state cache items in seconds", + buckets: [0.01, 0.1, 0.5, 1, 2, 5], + }), + regenFnNumStatesUpdated: register.histogram({ + name: "lodestar_regen_state_cache_state_updated_count", + help: "Histogram of number of state cache items updated every time removing pubkeys from unfinalized cache", + buckets: [1, 2, 5, 10, 50, 250], + }), unhandledPromiseRejections: register.gauge({ name: "lodestar_unhandled_promise_rejections_total", help: "UnhandledPromiseRejection total count", diff --git a/packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts b/packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts new file mode 100644 index 000000000000..b37967d16ca4 --- /dev/null +++ b/packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts @@ -0,0 +1,54 @@ +import crypto from "node:crypto"; +import {Map} from "immutable"; +import {ValidatorIndex} from "@lodestar/types"; +import {toMemoryEfficientHexStr} from "@lodestar/state-transition/src/cache/pubkeyCache.js"; +import {testRunnerMemory} from "./testRunnerMemory.js"; + +// Results in MacOS Nov 2023 +// +// UnfinalizedPubkey2Index 1000 keys - 274956.5 bytes / instance +// UnfinalizedPubkey2Index 10000 keys - 2591129.3 bytes / instance +// UnfinalizedPubkey2Index 100000 keys - 27261443.4 bytes / instance + +testRunnerMemoryBpi([ + { + id: "UnfinalizedPubkey2Index 1000 keys", + getInstance: () => getRandomMap(1000, () => toMemoryEfficientHexStr(crypto.randomBytes(48))), + }, + { + id: "UnfinalizedPubkey2Index 10000 keys", + getInstance: () => getRandomMap(10000, () => toMemoryEfficientHexStr(crypto.randomBytes(48))), + }, + { + id: "UnfinalizedPubkey2Index 100000 keys", + getInstance: () => getRandomMap(100000, () => toMemoryEfficientHexStr(crypto.randomBytes(48))), + }, +]); + +function getRandomMap(n: number, getKey: (i: number) => string): Map { + const map = Map(); + + return map.withMutations((m) => { + for (let i = 0; i < n; i++) { + m.set(getKey(i), i); + } + }); +} + +/** + * Test bytes per instance in different representations of raw binary data + */ +function testRunnerMemoryBpi(testCases: {getInstance: (bytes: number) => unknown; id: string}[]): void { + const longestId = Math.max(...testCases.map(({id}) => id.length)); + + for (const {id, getInstance} of testCases) { + const bpi = testRunnerMemory({ + getInstance, + convergeFactor: 1 / 100, + sampleEvery: 5, + }); + + // eslint-disable-next-line no-console + console.log(`${id.padEnd(longestId)} - ${bpi.toFixed(1)} bytes / instance`); + } +} diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 60ff6ce48302..7a882b03af0e 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -32,7 +32,7 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { before(function () { this.timeout(5 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}); + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}) as unknown as CachedBeaconStateAltair; const {blockHeader, checkpoint} = computeAnchorCheckpoint(originalState.config, originalState); // TODO figure out why getBlockRootAtSlot(originalState, justifiedSlot) is not the same to justifiedCheckpoint.root diff --git a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts index 6e420f0e1011..7998a204f09d 100644 --- a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts @@ -24,7 +24,7 @@ describe("opPool", () => { before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}); + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}) as unknown as CachedBeaconStateAltair; }); itBench({ diff --git a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts index 7bf8c2f7252f..2386f3538205 100644 --- a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts +++ b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts @@ -25,7 +25,7 @@ describe("produceBlockBody", () => { before(async () => { db = new BeaconDb(config, await LevelDbController.create({name: ".tmpdb"}, {logger})); - state = stateOg.clone(); + state = stateOg.clone() as unknown as CachedBeaconStateAltair; chain = new BeaconChain( { proposerBoost: true, diff --git a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts new file mode 100644 index 000000000000..900f6a6fb873 --- /dev/null +++ b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts @@ -0,0 +1,110 @@ +import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {Map} from "immutable"; +import {toBufferBE} from "bigint-buffer"; +import {digest} from "@chainsafe/as-sha256"; +import type {SecretKey} from "@chainsafe/bls/types"; +import bls from "@chainsafe/bls"; +import {ssz} from "@lodestar/types"; +import {type CachedBeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; +import {bytesToBigInt, intToBytes} from "@lodestar/utils"; +import {CheckpointStateCache, StateContextCache} from "../../../../src/chain/stateCache/index.js"; +import {generateCachedElectraState} from "../../../utils/state.js"; + +// Benchmark date from Mon Nov 21 2023 - Intel Core i7-9750H @ 2.60Ghz +// ✔ updateUnfinalizedPubkeys - updating 10 pubkeys 1444.173 ops/s 692.4380 us/op - 1057 runs 6.03 s +// ✔ updateUnfinalizedPubkeys - updating 100 pubkeys 189.5965 ops/s 5.274358 ms/op - 57 runs 1.15 s +// ✔ updateUnfinalizedPubkeys - updating 1000 pubkeys 12.90495 ops/s 77.48967 ms/op - 13 runs 1.62 s +describe("updateUnfinalizedPubkeys perf tests", function () { + setBenchOpts({noThreshold: true}); + + const numPubkeysToBeFinalizedCases = [10, 100, 1000]; + const numCheckpointStateCache = 8; + const numStateCache = 3 * 32; + + let checkpointStateCache: CheckpointStateCache; + let stateCache: StateContextCache; + + const unfinalizedPubkey2Index = generatePubkey2Index(0, Math.max.apply(null, numPubkeysToBeFinalizedCases)); + const baseState = generateCachedElectraState(); + + for (const numPubkeysToBeFinalized of numPubkeysToBeFinalizedCases) { + itBench({ + id: `updateUnfinalizedPubkeys - updating ${numPubkeysToBeFinalized} pubkeys`, + beforeEach: async () => { + baseState.epochCtx.unfinalizedPubkey2index = Map(unfinalizedPubkey2Index.map); + baseState.epochCtx.pubkey2index = new PubkeyIndexMap(); + baseState.epochCtx.index2pubkey = []; + + checkpointStateCache = new CheckpointStateCache({}); + stateCache = new StateContextCache({}); + + for (let i = 0; i < numCheckpointStateCache; i++) { + const clonedState = baseState.clone(); + const checkpoint = ssz.phase0.Checkpoint.defaultValue(); + + clonedState.slot = i; + checkpoint.epoch = i; // Assigning arbitrary non-duplicate values to ensure checkpointStateCache correctly saves all the states + + checkpointStateCache.add(checkpoint, clonedState); + } + + for (let i = 0; i < numStateCache; i++) { + const clonedState = baseState.clone(); + clonedState.slot = i; + stateCache.add(clonedState); + } + }, + fn: async () => { + const newFinalizedValidators = baseState.epochCtx.unfinalizedPubkey2index.filter( + (index, _pubkey) => index < numPubkeysToBeFinalized + ); + + const states = stateCache.getStates(); + const cpStates = checkpointStateCache.getStates(); + + const firstState = states.next().value as CachedBeaconStateAllForks; + firstState.epochCtx.addFinalizedPubkeys(newFinalizedValidators); + + const pubkeysToDelete = Array.from(newFinalizedValidators.keys()); + + firstState.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + + for (const s of states) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + } + + for (const s of cpStates) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + } + }, + }); + } + + function generatePubkey2Index(startIndex: number, endIndex: number): PubkeyIndexMap { + const pubkey2Index = new PubkeyIndexMap(); + const pubkeys = generatePubkeys(endIndex - startIndex); + + for (let i = startIndex; i < endIndex; i++) { + pubkey2Index.set(pubkeys[i], i); + } + + return pubkey2Index; + } + + function generatePubkeys(validatorCount: number): Uint8Array[] { + const keys = []; + + for (let i = 0; i < validatorCount; i++) { + const sk = generatePrivateKey(i); + const pk = sk.toPublicKey().toBytes(); + keys.push(pk); + } + + return keys; + } + + function generatePrivateKey(index: number): SecretKey { + const secretKeyBytes = toBufferBE(bytesToBigInt(digest(intToBytes(index, 32))) % BigInt("38581184513"), 32); + return bls.SecretKey.fromBytes(secretKeyBytes); + } +}); diff --git a/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh b/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh new file mode 100755 index 000000000000..1444f6d3a479 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash -x + +echo $TTD +echo $DATA_DIR +echo $EL_BINARY_DIR +echo $JWT_SECRET_HEX +echo $TEMPLATE_FILE + +echo $scriptDir +echo $currentDir + + +env TTD=$TTD envsubst < $scriptDir/$TEMPLATE_FILE > $DATA_DIR/genesis.json +echo "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" > $DATA_DIR/sk.json +echo "12345678" > $DATA_DIR/password.txt +pubKey="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + +# echo a hex encoded 256 bit secret into a file +echo $JWT_SECRET_HEX> $DATA_DIR/jwtsecret \ No newline at end of file diff --git a/packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl b/packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl new file mode 100644 index 000000000000..7a63bfbe36d6 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl @@ -0,0 +1,77 @@ +{ + "config": { + "chainId":6110, + "homesteadBlock":0, + "eip150Block":0, + "eip155Block":0, + "eip158Block":0, + "byzantiumBlock":0, + "constantinopleBlock":0, + "petersburgBlock":0, + "istanbulBlock":0, + "muirGlacierBlock":0, + "berlinBlock":0, + "londonBlock":0, + "terminalTotalDifficulty":0, + "cancunTime":0, + "experimentalEipsTime":10, + "clique": { + "period": 5, + "epoch": 30000 + }, + "depositContractAddress": "0x4242424242424242424242424242424242424242" + }, + "nonce":"0x42", + "timestamp":"0x0", + "extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit":"0x1C9C380", + "difficulty":"0x400000000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase":"0x0000000000000000000000000000000000000000", + "alloc":{ + "0xa4664C40AACeBD82A2Db79f0ea36C06Bc6A19Adb": { + "balance": "1000000000000000000000000000" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + } + }, + "number":"0x0", + "gasUsed":"0x0", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas":"0x7" +} diff --git a/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh b/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh new file mode 100755 index 000000000000..47bec71cf8bb --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh @@ -0,0 +1,8 @@ +#!/bin/bash -x + +scriptDir=$(dirname $0) +currentDir=$(pwd) + +. $scriptDir/common-setup.sh + +$EL_BINARY_DIR/besu --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret $currentDir/$DATA_DIR/jwtsecret --data-path $DATA_DIR --data-storage-format BONSAI --genesis-file $DATA_DIR/genesis.json \ No newline at end of file diff --git a/packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh b/packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh new file mode 100644 index 000000000000..b3d93190ef2d --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash -x + +echo $TTD +echo $DATA_DIR +echo $EL_BINARY_DIR +echo $JWT_SECRET_HEX +echo $TEMPLATE_FILE + +echo $scriptDir +echo $currentDir + + +env TTD=$TTD envsubst < $scriptDir/$TEMPLATE_FILE > $DATA_DIR/genesis.json +echo "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" > $DATA_DIR/sk.json +echo "12345678" > $DATA_DIR/password.txt +pubKey="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + +# echo a hex encoded 256 bit secret into a file +echo $JWT_SECRET_HEX> $DATA_DIR/jwtsecret +# clear any previous docker dangling docker run +docker rm -f custom-execution +rm -rf $DATA_DIR/besu diff --git a/packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl b/packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl new file mode 100644 index 000000000000..7a63bfbe36d6 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl @@ -0,0 +1,77 @@ +{ + "config": { + "chainId":6110, + "homesteadBlock":0, + "eip150Block":0, + "eip155Block":0, + "eip158Block":0, + "byzantiumBlock":0, + "constantinopleBlock":0, + "petersburgBlock":0, + "istanbulBlock":0, + "muirGlacierBlock":0, + "berlinBlock":0, + "londonBlock":0, + "terminalTotalDifficulty":0, + "cancunTime":0, + "experimentalEipsTime":10, + "clique": { + "period": 5, + "epoch": 30000 + }, + "depositContractAddress": "0x4242424242424242424242424242424242424242" + }, + "nonce":"0x42", + "timestamp":"0x0", + "extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit":"0x1C9C380", + "difficulty":"0x400000000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase":"0x0000000000000000000000000000000000000000", + "alloc":{ + "0xa4664C40AACeBD82A2Db79f0ea36C06Bc6A19Adb": { + "balance": "1000000000000000000000000000" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + } + }, + "number":"0x0", + "gasUsed":"0x0", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas":"0x7" +} diff --git a/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh b/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh new file mode 100755 index 000000000000..fb091a7d838b --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh @@ -0,0 +1,8 @@ +#!/bin/bash -x + +scriptDir=$(dirname $0) +currentDir=$(pwd) + +. $scriptDir/common-setup.sh + +docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --name custom-execution -p $ETH_PORT:$ETH_PORT -p $ENGINE_PORT:$ENGINE_PORT -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret /data/jwtsecret --data-path /data/besu --data-storage-format BONSAI --genesis-file /data/genesis.json \ No newline at end of file diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts new file mode 100644 index 000000000000..c1663211cbd9 --- /dev/null +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -0,0 +1,457 @@ +import fs from "node:fs"; +import {describe, it, vi, afterAll, afterEach} from "vitest"; +/* eslint-disable @typescript-eslint/naming-convention */ +import _ from "lodash"; +import {LogLevel, sleep} from "@lodestar/utils"; +import {ForkName, SLOTS_PER_EPOCH, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {electra, Epoch, Slot} from "@lodestar/types"; +import {ValidatorProposerConfig} from "@lodestar/validator"; + +import {ChainConfig} from "@lodestar/config"; +import {TimestampFormatCode} from "@lodestar/logger"; +import {CachedBeaconStateElectra} from "@lodestar/state-transition"; +import {initializeExecutionEngine} from "../../src/execution/index.js"; +import {ExecutionPayloadStatus, PayloadAttributes} from "../../src/execution/engine/interface.js"; + +import {testLogger, TestLoggerOpts} from "../utils/logger.js"; +import {runEL, ELStartMode, ELClient, sendRawTransactionBig} from "../utils/runEl.js"; +import {defaultExecutionEngineHttpOpts} from "../../src/execution/engine/http.js"; +import {getDevBeaconNode} from "../utils/node/beacon.js"; +import {BeaconRestApiServerOpts} from "../../src/api/index.js"; +import {simTestInfoTracker} from "../utils/node/simTest.js"; +import {getAndInitDevValidators} from "../utils/node/validator.js"; +import {ClockEvent} from "../../src/util/clock.js"; +import {dataToBytes} from "../../src/eth1/provider/utils.js"; +import {bytesToData} from "../../lib/eth1/provider/utils.js"; +import {BeaconNode} from "../../src/index.js"; +import {logFilesDir} from "./params.js"; +import {shell} from "./shell.js"; + +// NOTE: How to run +// DEV_RUN=true EL_BINARY_DIR=naviechan/besu:v6110 EL_SCRIPT_DIR=besudocker yarn vitest --run test/sim/electra-interop.test.ts +// or +// DEV_RUN=true EL_BINARY_DIR=/Volumes/fast_boi/navie/Documents/workspace/besu/build/install/besu/bin EL_SCRIPT_DIR=besu yarn vitest --run test/sim/electra-interop.test.ts +// ``` + +/* eslint-disable no-console, @typescript-eslint/naming-convention */ + +const jwtSecretHex = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; +const retries = defaultExecutionEngineHttpOpts.retries; +const retryDelay = defaultExecutionEngineHttpOpts.retryDelay; +describe("executionEngine / ExecutionEngineHttp", function () { + if (!process.env.EL_BINARY_DIR || !process.env.EL_SCRIPT_DIR) { + throw Error( + `EL ENV must be provided, EL_BINARY_DIR: ${process.env.EL_BINARY_DIR}, EL_SCRIPT_DIR: ${process.env.EL_SCRIPT_DIR}` + ); + } + vi.setConfig({testTimeout: 1000 * 60 * 10, hookTimeout: 1000 * 60 * 10}); + + const dataPath = fs.mkdtempSync("lodestar-test-electra"); + const elSetupConfig = { + elScriptDir: process.env.EL_SCRIPT_DIR, + elBinaryDir: process.env.EL_BINARY_DIR, + }; + const elRunOptions = { + dataPath, + jwtSecretHex, + enginePort: parseInt(process.env.ENGINE_PORT ?? "8551"), + ethPort: parseInt(process.env.ETH_PORT ?? "8545"), + }; + + const controller = new AbortController(); + afterAll(async () => { + controller?.abort(); + await shell(`sudo rm -rf ${dataPath}`); + }); + + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + + it("Send and get payloads with depositReceipts to/from EL", async () => { + const {elClient, tearDownCallBack} = await runEL( + {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, + {...elRunOptions, ttd: BigInt(0)}, + controller.signal + ); + afterEachCallbacks.push(() => tearDownCallBack()); + const {genesisBlockHash, engineRpcUrl, ethRpcUrl} = elClient; + console.log({genesisBlockHash}); + + const loggerExecutionEngine = testLogger("executionEngine"); + + const executionEngine = initializeExecutionEngine( + {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retries, retryDelay}, + {signal: controller.signal, logger: loggerExecutionEngine} + ); + + // 1. Prepare payload + const preparePayloadParams: PayloadAttributes = { + // Note: this is created with a pre-defined genesis.json + timestamp: 10, + prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + suggestedFeeRecipient: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + withdrawals: [], + parentBeaconBlockRoot: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + }; + const payloadId = await executionEngine.notifyForkchoiceUpdate( + ForkName.electra, + genesisBlockHash, + //use finalizedBlockHash as safeBlockHash + genesisBlockHash, + genesisBlockHash, + preparePayloadParams + ); + if (!payloadId) throw Error("InvalidPayloadId"); + + // 2. Send raw deposit transaction A and B. tx A is to be imported via newPayload, tx B is to be included in payload via getPayload + const depositTransactionA = + "0x02f9021c8217de808459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120749715de5d1226545c6b3790f515d551a5cc5bf1d49c87a696860554d2fc4f14000000000000000000000000000000000000000000000000000000000000003096a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9c080a09f597089338d7f44f5c59f8230bb38f243849228a8d4e9d2e2956e6050f5b2c7a076486996c7e62802b8f95eee114783e4b403fd11093ba96286ff42c595f24452"; + const depositReceiptA = { + amount: 32000000000, + index: 0, + pubkey: dataToBytes( + "0x96a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9", + 48 + ), + signature: dataToBytes( + "0xb1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9", + 96 + ), + withdrawalCredentials: dataToBytes("0x003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef2", 32), + }; + + const depositTransactionB = + "0x02f9021c8217de018459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120a18b4c7cab0afa273ea9504904521ea8421a4e32740b7611bd3d5095ca99f0cb0000000000000000000000000000000000000000000000000000000000000030a5c85a60ba2905c215f6a12872e62b1ee037051364244043a5f639aa81b04a204c55e7cc851f29c7c183be253ea1510b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020001db70c485b6264692f26b8aeaab5b0c384180df8e2184a21a808a3ec8e86ca00000000000000000000000000000000000000000000000000000000000000609561731785b48cf1886412234531e4940064584463e96ac63a1a154320227e333fb51addc4a89b7e0d3f862d7c1fd4ea03bd8eb3d8806f1e7daf591cbbbb92b0beb74d13c01617f22c5026b4f9f9f294a8a7c32db895de3b01bee0132c9209e1c001a0644e0a763a34b4bfb9f56a677857b57fcf15e3db57e2f57060e92084f75f3d82a018ba8eaacbd8e6f6917675b1d0362b12ca82850ca8ef9c010430760c2b2e0cb5"; + const depositReceiptB = { + amount: 32000000000, + index: 1, + pubkey: dataToBytes( + "0xa5c85a60ba2905c215f6a12872e62b1ee037051364244043a5f639aa81b04a204c55e7cc851f29c7c183be253ea1510b", + 48 + ), + signature: dataToBytes( + "0x9561731785b48cf1886412234531e4940064584463e96ac63a1a154320227e333fb51addc4a89b7e0d3f862d7c1fd4ea03bd8eb3d8806f1e7daf591cbbbb92b0beb74d13c01617f22c5026b4f9f9f294a8a7c32db895de3b01bee0132c9209e1", + 96 + ), + withdrawalCredentials: dataToBytes("0x001db70c485b6264692f26b8aeaab5b0c384180df8e2184a21a808a3ec8e86ca", 32), + }; + + sendRawTransactionBig(ethRpcUrl, depositTransactionA, `${dataPath}/deposit.json`).catch((e: Error) => { + loggerExecutionEngine.error("Fail to send raw deposit transaction A", undefined, e); + }); + + sendRawTransactionBig(ethRpcUrl, depositTransactionB, `${dataPath}/deposit.json`).catch((e: Error) => { + loggerExecutionEngine.error("Fail to send raw deposit transaction B", undefined, e); + }); + + // 3. Import new payload with tx A and deposit receipt A + const newPayloadBlockHash = "0xfd1189e6ea0814b7d40d4e50b31ae5feabbb2acff39399457bbdda7cb5ccd490"; + const newPayload = { + parentHash: dataToBytes("0x26118cf71453320edcebbc4ebb34af5b578087a32385b80108bf691fa23efc42", 32), + feeRecipient: dataToBytes("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", 20), + stateRoot: dataToBytes("0x14208ac0e218167936e220b72d5d5887a963cb858ea2f2d268518f014a3da3fa", 32), + logsBloom: dataToBytes( + "0x10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000", + 256 + ), + prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + gasLimit: 30000000, + gasUsed: 84846, + timestamp: 16, + extraData: dataToBytes("0x", 0), + baseFeePerGas: 7n, + excessBlobGas: 0n, + transactions: [dataToBytes(depositTransactionA, null)], + withdrawals: [], + depositReceipts: [depositReceiptA], + blockNumber: 1, + blockHash: dataToBytes(newPayloadBlockHash, 32), + receiptsRoot: dataToBytes("0x79ee3424eb720a3ad4b1c5a372bb8160580cbe4d893778660f34213c685627a9", 32), + blobGasUsed: 0n, + }; + const parentBeaconBlockRoot = dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32); + const payloadResult = await executionEngine.notifyNewPayload( + ForkName.electra, + newPayload, + [], + parentBeaconBlockRoot + ); + if (payloadResult.status !== ExecutionPayloadStatus.VALID) { + throw Error("getPayload returned payload that notifyNewPayload deems invalid"); + } + + // 4. Update fork choice + const preparePayloadParams2: PayloadAttributes = { + timestamp: 48, + prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + suggestedFeeRecipient: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + withdrawals: [], + parentBeaconBlockRoot: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + }; + + const payloadId2 = await executionEngine.notifyForkchoiceUpdate( + ForkName.electra, + newPayloadBlockHash, + //use finalizedBlockHash as safeBlockHash + newPayloadBlockHash, + newPayloadBlockHash, + preparePayloadParams2 + ); + if (!payloadId2) throw Error("InvalidPayloadId"); + + // 5. Get the payload. Check depositReceipts field contains deposit + // Wait a bit first for besu to pick up tx from the tx pool. + await sleep(1000); + const payloadAndBlockValue = await executionEngine.getPayload(ForkName.electra, payloadId2); + const payload = payloadAndBlockValue.executionPayload as electra.ExecutionPayload; + + if (payload.transactions.length !== 1) { + throw Error(`Number of transactions mismatched. Expected: 1, actual: ${payload.transactions.length}`); + } else { + const actualTransaction = bytesToData(payload.transactions[0]); + + if (actualTransaction !== depositTransactionB) { + throw Error(`Transaction mismatched. Expected: ${depositTransactionB}, actual: ${actualTransaction}`); + } + } + + if (payload.depositReceipts.length !== 1) { + throw Error(`Number of depositReceipts mismatched. Expected: 1, actual: ${payload.depositReceipts.length}`); + } + + const actualDepositReceipt = payload.depositReceipts[0]; + if (!_.isEqual(actualDepositReceipt, depositReceiptB)) { + throw Error( + `Deposit receipts mismatched. Expected: ${JSON.stringify(depositReceiptB)}, actual: ${JSON.stringify( + actualDepositReceipt + )}` + ); + } + }); + + it("Post-merge, run for a few blocks", async function () { + console.log("\n\nPost-merge, run for a few blocks\n\n"); + const {elClient, tearDownCallBack} = await runEL( + {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, + {...elRunOptions, ttd: BigInt(0)}, + controller.signal + ); + afterEachCallbacks.push(() => tearDownCallBack()); + + await runNodeWithEL({ + elClient, + electraEpoch: 0, + testName: "post-merge", + }); + }); + + /** + * Want to test two things: + * 1) Send two raw deposit transactions, and see if two new validators with corrent balances show up in the state.validators and unfinalized cache + * 2) Upon state-transition, see if the two new validators move from unfinalized cache to finalized cache + */ + async function runNodeWithEL({ + elClient, + electraEpoch, + testName, + }: { + elClient: ELClient; + electraEpoch: Epoch; + testName: string; + }): Promise { + const {genesisBlockHash, ttd, engineRpcUrl, ethRpcUrl} = elClient; + const validatorClientCount = 1; + const validatorsPerClient = 32; + + const testParams: Pick = { + SECONDS_PER_SLOT: 2, + }; + + // Just enough to have a checkpoint finalized + const expectedEpochsToFinish = 4; + // 1 epoch of margin of error + const epochsOfMargin = 1; + const timeoutSetupMargin = 30 * 1000; // Give extra 30 seconds of margin + + // delay a bit so regular sync sees it's up to date and sync is completed from the beginning + const genesisSlotsDelay = 8; + + const timeout = + ((epochsOfMargin + expectedEpochsToFinish) * SLOTS_PER_EPOCH + genesisSlotsDelay) * + testParams.SECONDS_PER_SLOT * + 1000; + + vi.setConfig({testTimeout: timeout + 2 * timeoutSetupMargin}); + + const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; + + const testLoggerOpts: TestLoggerOpts = { + level: LogLevel.info, + file: { + filepath: `${logFilesDir}/mergemock-${testName}.log`, + level: LogLevel.debug, + }, + timestampFormat: { + format: TimestampFormatCode.EpochSlot, + genesisTime, + slotsPerEpoch: SLOTS_PER_EPOCH, + secondsPerSlot: testParams.SECONDS_PER_SLOT, + }, + }; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: { + ...testParams, + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: electraEpoch, + TERMINAL_TOTAL_DIFFICULTY: ttd, + }, + options: { + api: {rest: {enabled: true} as BeaconRestApiServerOpts}, + sync: {isSingleNode: true}, + network: {allowPublishToZeroPeers: true, discv5: null}, + // Now eth deposit/merge tracker methods directly available on engine endpoints + eth1: {enabled: false, providerUrls: [engineRpcUrl], jwtSecretHex}, + executionEngine: {urls: [engineRpcUrl], jwtSecretHex}, + chain: {suggestedFeeRecipient: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, + }, + validatorCount: validatorClientCount * validatorsPerClient, + logger: loggerNodeA, + genesisTime, + eth1BlockHash: dataToBytes(genesisBlockHash, 32), + withEth1Credentials: true, + }); + + afterEachCallbacks.push(async function () { + await bn.close(); + await sleep(1000); + }); + + const stopInfoTracker = simTestInfoTracker(bn, loggerNodeA); + const valProposerConfig = { + defaultConfig: { + feeRecipient: "0xcccccccccccccccccccccccccccccccccccccccc", + }, + } as ValidatorProposerConfig; + + const {validators} = await getAndInitDevValidators({ + node: bn, + logPrefix: "Node-A", + validatorsPerClient, + validatorClientCount, + startIndex: 0, + // At least one sim test must use the REST API for beacon <-> validator comms + useRestApi: true, + testLoggerOpts, + valProposerConfig, + }); + + afterEachCallbacks.push(async function () { + await Promise.all(validators.map((v) => v.close())); + }); + + await waitForSlot(bn, 1); + + // send raw tx at slot 1 + const depositTransaction = + "0x02f9021e8217de8085012a05f20085019254d380830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120ef950826b191ebea0bbafa92a2c6bffa8239c6f456d92891ce2852b8360f0d30000000000000000000000000000000000000000000000000000000000000003095e4f91aea91a9e00387fad9d60997cff6cbf68d42d1b6629a7b248cdef255f94a2a2381e5d4125273fe42da5f7aa0e1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b6c06e65228046268aa918baf78e072c25e65aa0bcf258cefcac3371c47df81bc4d43ca942f5fc28f9a563e925fd9c5010bc8c300add3faf3af0d61fabaaf03694020feaafb03e47c1bc4fcf082684c7ed3f7d5839d1722214b24f95ad2b226cc080a0be1161617492e4ca2fcb89edcadf5e71e8cac0d6447d18cfde9b55e5a8412417a07ec8c47dd484036c745049bb2e2980d44e38d4dacac50dc4a14a2f23c52f2e5f"; + sendRawTransactionBig(ethRpcUrl, depositTransaction, `${dataPath}/deposit.json`).catch((e: Error) => { + loggerNodeA.error("Fail to send raw deposit transaction", undefined, e); + }); + + await waitForSlot(bn, 5); + // Expect new validator to be in unfinalized cache, in state.validators and not in finalized cache + let headState = bn.chain.getHeadState(); + let epochCtx = headState.epochCtx; + if (headState.validators.length !== 33 || headState.balances.length !== 33) { + throw Error("New validator is not reflected in the beacon state at slot 5"); + } + if (epochCtx.index2pubkey.length !== 32 || epochCtx.pubkey2index.size !== 32) { + throw Error("Finalized cache is modified."); + } + if (epochCtx.unfinalizedPubkey2index.size !== 1) { + throw Error( + `Unfinalized cache is missing the expected validator. Size: ${epochCtx.unfinalizedPubkey2index.size}` + ); + } + // validator count at epoch 1 should be empty at this point since no epoch transition has happened. + if (epochCtx.getValidatorCountAtEpoch(1) !== undefined) { + throw Error("Historical validator lengths is modified"); + } + + await new Promise((resolve, _reject) => { + bn.chain.clock.on(ClockEvent.epoch, (epoch) => { + // Resolve only if the finalized checkpoint includes execution payload + if (epoch >= expectedEpochsToFinish) { + console.log("\nGot event epoch, stopping validators and nodes\n"); + resolve(); + } + }); + }); + + // Stop chain and un-subscribe events so the execution engine won't update it's head + // Allow some time to broadcast finalized events and complete the importBlock routine + await Promise.all(validators.map((v) => v.close())); + await bn.close(); + await sleep(500); + + // Check if new validator is in finalized cache + headState = bn.chain.getHeadState() as CachedBeaconStateElectra; + epochCtx = headState.epochCtx; + + if (headState.validators.length !== 33 || headState.balances.length !== 33) { + throw Error("New validator is not reflected in the beacon state."); + } + if (epochCtx.index2pubkey.length !== 33 || epochCtx.pubkey2index.size !== 33) { + throw Error("New validator is not in finalized cache"); + } + if (!epochCtx.unfinalizedPubkey2index.isEmpty()) { + throw Error("Unfinalized cache still contains new validator"); + } + // After 4 epochs, headState's finalized cp epoch should be 2 + // epochCtx should only have validator count for epoch 3 and 4. + if (epochCtx.getValidatorCountAtEpoch(4) === undefined || epochCtx.getValidatorCountAtEpoch(3) === undefined) { + throw Error("Missing historical validator length for epoch 3 or 4"); + } + + if (epochCtx.getValidatorCountAtEpoch(4) !== 33 || epochCtx.getValidatorCountAtEpoch(3) !== 33) { + throw Error("Incorrect historical validator length for epoch 3 or 4"); + } + + if (epochCtx.getValidatorCountAtEpoch(2) !== undefined || epochCtx.getValidatorCountAtEpoch(1) !== undefined) { + throw Error("Historical validator length for epoch 1 or 2 is not dropped properly"); + } + + if (headState.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + throw Error("state.depositReceiptsStartIndex is not set upon processing new deposit receipt"); + } + + // wait for 1 slot to print current epoch stats + await sleep(1 * bn.config.SECONDS_PER_SLOT * 1000); + stopInfoTracker(); + console.log("\n\nDone\n\n"); + } +}); + +async function waitForSlot(bn: BeaconNode, targetSlot: Slot): Promise { + await new Promise((resolve, reject) => { + bn.chain.clock.on(ClockEvent.slot, (currentSlot) => { + if (currentSlot === targetSlot) { + resolve(); + return; + } + if (currentSlot > targetSlot) { + reject(Error(`Beacon node has passed target slot ${targetSlot}. Current slot ${currentSlot}`)); + } + }); + }); +} diff --git a/packages/beacon-node/test/spec/presets/fork.test.ts b/packages/beacon-node/test/spec/presets/fork.test.ts index c880d24bbbe3..c121e651fcea 100644 --- a/packages/beacon-node/test/spec/presets/fork.test.ts +++ b/packages/beacon-node/test/spec/presets/fork.test.ts @@ -5,6 +5,7 @@ import { CachedBeaconStateAltair, CachedBeaconStatePhase0, CachedBeaconStateCapella, + CachedBeaconStateDeneb, } from "@lodestar/state-transition"; import * as slotFns from "@lodestar/state-transition/slot"; import {phase0, ssz} from "@lodestar/types"; @@ -36,7 +37,7 @@ const fork: TestRunnerFn = (forkNext) => { case ForkName.deneb: return slotFns.upgradeStateToDeneb(preState as CachedBeaconStateCapella); case ForkName.electra: - throw Error("not Implemented"); + return slotFns.upgradeStateToElectra(preState as CachedBeaconStateDeneb); } }, options: { diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index f03f2595a566..d8f85dd9b5b8 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -61,7 +61,7 @@ const genesisInitialization: TestRunnerFn + testcase["execution_payload_header"] as ExecutionPayloadHeader ) ); }, diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 4c1c10e0cb66..cc032bec6fa2 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -4,11 +4,12 @@ import { CachedBeaconStateAllForks, CachedBeaconStateBellatrix, CachedBeaconStateCapella, + CachedBeaconStateElectra, ExecutionPayloadStatus, getBlockRootAtSlot, } from "@lodestar/state-transition"; import * as blockFns from "@lodestar/state-transition/block"; -import {ssz, phase0, altair, bellatrix, capella, sszTypesFor} from "@lodestar/types"; +import {ssz, phase0, altair, bellatrix, capella, electra, sszTypesFor} from "@lodestar/types"; import {InputType} from "@lodestar/spec-test-util"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; @@ -56,6 +57,11 @@ const operationFns: Record> = blockFns.processDeposit(fork, state, testCase.deposit); }, + deposit_receipt: (state, testCase: {deposit_receipt: electra.DepositReceipt}) => { + const fork = state.config.getForkSeq(state.slot); + blockFns.processDepositReceipt(fork, state as CachedBeaconStateElectra, testCase.deposit_receipt); + }, + proposer_slashing: (state, testCase: {proposer_slashing: phase0.ProposerSlashing}) => { const fork = state.config.getForkSeq(state.slot); blockFns.processProposerSlashing(fork, state, testCase.proposer_slashing); @@ -121,6 +127,7 @@ const operations: TestRunnerFn = (fork, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, + deposit_receipt: ssz.electra.DepositReceipt, proposer_slashing: ssz.phase0.ProposerSlashing, voluntary_exit: ssz.phase0.SignedVoluntaryExit, // Altair diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index d81b9dee0098..d81b47ecf587 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -45,6 +45,7 @@ const sszStatic = /* eslint-disable @typescript-eslint/strict-boolean-expressions */ const sszType = (sszTypesFor(fork) as Types)[typeName] || + (ssz.deneb as Types)[typeName] || (ssz.capella as Types)[typeName] || (ssz.bellatrix as Types)[typeName] || (ssz.altair as Types)[typeName] || diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 6aa683cb6530..3201fb15dbf3 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -57,7 +57,6 @@ const coveredTestRunners = [ // ], // ``` export const defaultSkipOpts: SkipOpts = { - skippedForks: ["eip6110"], // TODO: capella // BeaconBlockBody proof in lightclient is the new addition in v1.3.0-rc.2-hotfix // Skip them for now to enable subsequently diff --git a/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts b/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts index ba8014c47c9e..abb520bddafb 100644 --- a/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts +++ b/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts @@ -15,7 +15,7 @@ describe("UpgradeLightClientHeader", function () { BELLATRIX_FORK_EPOCH: 2, CAPELLA_FORK_EPOCH: 3, DENEB_FORK_EPOCH: 4, - ELECTRA_FORK_EPOCH: Infinity, + ELECTRA_FORK_EPOCH: 5, }); const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); @@ -37,18 +37,12 @@ describe("UpgradeLightClientHeader", function () { bellatrix: 17, capella: 25, deneb: 33, - electra: 0, + electra: 41, }; }); - // Since electra is not implemented for loop is till deneb (Object.values(ForkName).length-1) - // Once electra is implemnted run for loop till Object.values(ForkName).length - - // for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { - // for (let j = i + 1; j < Object.values(ForkName).length; j++) { - - for (let i = ForkSeq.altair; i < Object.values(ForkName).length - 1; i++) { - for (let j = i + 1; j < Object.values(ForkName).length - 1; j++) { + for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { + for (let j = i + 1; j < Object.values(ForkName).length; j++) { const fromFork = ForkName[ForkSeq[i] as ForkName]; const toFork = ForkName[ForkSeq[j] as ForkName]; @@ -62,27 +56,7 @@ describe("UpgradeLightClientHeader", function () { } } - // for electra not implemented for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { - const fromFork = ForkName[ForkSeq[i] as ForkName]; - const toFork = ForkName["electra"]; - - it(`Throw error ${fromFork}=>${toFork}`, function () { - lcHeaderByFork[fromFork].beacon.slot = testSlots[fromFork]; - lcHeaderByFork[toFork].beacon.slot = testSlots[fromFork]; - - expect(() => { - upgradeLightClientHeader(config, toFork, lcHeaderByFork[fromFork]); - }).toThrow("Not Implemented"); - }); - } - - // Since electra is not implemented for loop is till deneb (Object.values(ForkName).length-1) - // Once electra is implemnted run for loop till Object.values(ForkName).length - - // for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { - - for (let i = ForkSeq.altair; i < Object.values(ForkName).length - 1; i++) { for (let j = i; j > 0; j--) { const fromFork = ForkName[ForkSeq[i] as ForkName]; const toFork = ForkName[ForkSeq[j] as ForkName]; diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 800984fa84bc..41c205cdd0e9 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -61,9 +61,9 @@ describe("AggregatedAttestationPool", function () { epochParticipation[committee[i]] = 0b000; } } - (originalState as CachedBeaconStateAltair).previousEpochParticipation = + (originalState as unknown as CachedBeaconStateAltair).previousEpochParticipation = ssz.altair.EpochParticipation.toViewDU(epochParticipation); - (originalState as CachedBeaconStateAltair).currentEpochParticipation = + (originalState as unknown as CachedBeaconStateAltair).currentEpochParticipation = ssz.altair.EpochParticipation.toViewDU(epochParticipation); originalState.commit(); let altairState: CachedBeaconStateAllForks; diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 6295a993c072..e6a7e8706bbe 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect, beforeEach} from "vitest"; -import {getShufflingDecisionBlock} from "@lodestar/state-transition"; +import {getShufflingDecisionBlock, CachedBeaconStateAllForks} from "@lodestar/state-transition"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; @@ -14,7 +14,7 @@ describe("ShufflingCache", function () { beforeEach(() => { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.processState(state, currentEpoch); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch); }); it("should get shuffling from cache", async function () { @@ -29,7 +29,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shufflings at other epochs does prune the cache - shufflingCache.processState(state, currentEpoch + 1); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch + 1); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); }); @@ -39,7 +39,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - shufflingCache.processState(state, currentEpoch + 1); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch + 1); expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); }); diff --git a/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts index 7d3f34ddac36..b4aac92dd9bb 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts @@ -1,8 +1,7 @@ import {describe, it, expect, beforeEach} from "vitest"; import {toHexString} from "@chainsafe/ssz"; -import {EpochShuffling} from "@lodestar/state-transition"; +import {EpochShuffling, CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {CachedBeaconStateAllForks} from "@lodestar/state-transition/src/types.js"; import {FIFOBlockStateCache} from "../../../../src/chain/stateCache/index.js"; import {generateCachedState} from "../../../utils/state.js"; diff --git a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts index ce0d7fae1fad..6cc3ae8f1ce0 100644 --- a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts +++ b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts @@ -1,13 +1,15 @@ import {describe, it, expect} from "vitest"; import {phase0, ssz} from "@lodestar/types"; -import {MAX_DEPOSITS} from "@lodestar/params"; +import {MAX_DEPOSITS, SLOTS_PER_EPOCH} from "@lodestar/params"; import {verifyMerkleBranch} from "@lodestar/utils"; +import {createChainForkConfig} from "@lodestar/config"; import {filterBy} from "../../../utils/db.js"; import {Eth1ErrorCode} from "../../../../src/eth1/errors.js"; import {generateState} from "../../../utils/state.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; import {getDeposits, getDepositsWithProofs, DepositGetter} from "../../../../src/eth1/utils/deposits.js"; import {DepositTree} from "../../../../src/db/repositories/depositDataRoot.js"; +import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState.js"; describe("eth1 / util / deposits", function () { describe("getDeposits", () => { @@ -18,6 +20,7 @@ describe("eth1 / util / deposits", function () { depositIndexes: number[]; expectedReturnedIndexes?: number[]; error?: Eth1ErrorCode; + postElectra?: boolean; }; const testCases: TestCase[] = [ @@ -70,18 +73,59 @@ describe("eth1 / util / deposits", function () { depositIndexes: [], expectedReturnedIndexes: [], }, + { + id: "No deposits to be included post Electra after deposit_receipts_start_index", + depositCount: 2030, + eth1DepositIndex: 2025, + depositIndexes: Array.from({length: 2030}, (_, i) => i), + expectedReturnedIndexes: [], + postElectra: true, + }, + { + id: "Should return deposits post Electra before deposit_receipts_start_index", + depositCount: 2022, + eth1DepositIndex: 2018, + depositIndexes: Array.from({length: 2022}, (_, i) => i), + expectedReturnedIndexes: [2018, 2019, 2020, 2021], + postElectra: true, + }, + { + id: "Should return deposits less than MAX_DEPOSITS post Electra before deposit_receipts_start_index", + depositCount: 10 * MAX_DEPOSITS, + eth1DepositIndex: 0, + depositIndexes: Array.from({length: 10 * MAX_DEPOSITS}, (_, i) => i), + expectedReturnedIndexes: Array.from({length: MAX_DEPOSITS}, (_, i) => i), + postElectra: true, + }, ]; + /* eslint-disable @typescript-eslint/naming-convention */ + const postElectraConfig = createChainForkConfig({ + ALTAIR_FORK_EPOCH: 1, + BELLATRIX_FORK_EPOCH: 2, + CAPELLA_FORK_EPOCH: 3, + DENEB_FORK_EPOCH: 4, + ELECTRA_FORK_EPOCH: 5, + }); + const postElectraSlot = postElectraConfig.ELECTRA_FORK_EPOCH * SLOTS_PER_EPOCH + 1; + for (const testCase of testCases) { - const {id, depositIndexes, eth1DepositIndex, depositCount, expectedReturnedIndexes, error} = testCase; + const {id, depositIndexes, eth1DepositIndex, depositCount, expectedReturnedIndexes, error, postElectra} = + testCase; it(id, async function () { - const state = generateState({eth1DepositIndex}); + const state = postElectra + ? generateState({slot: postElectraSlot, eth1DepositIndex}, postElectraConfig) + : generateState({eth1DepositIndex}); + const cachedState = createCachedBeaconStateTest( + state, + postElectra ? postElectraConfig : createChainForkConfig({}) + ); const eth1Data = generateEth1Data(depositCount); const deposits = depositIndexes.map((index) => generateDepositEvent(index)); const depositsGetter: DepositGetter = async (indexRange) => filterBy(deposits, indexRange, (deposit) => deposit.index); - const resultPromise = getDeposits(state, eth1Data, depositsGetter); + const resultPromise = getDeposits(cachedState, eth1Data, depositsGetter); if (expectedReturnedIndexes) { const result = await resultPromise; diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index aa33c7dbbc40..b0e20ad1f2b5 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -193,6 +193,7 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], + depositReceipts: null, // depositReceipts is null pre-electra }, null, // null returned for missing blocks { @@ -201,6 +202,7 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella + depositReceipts: null, // depositReceipts is null pre-electra }, ], }; @@ -248,6 +250,7 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], + depositReceipts: null, // depositReceipts is null pre-electra }, null, // null returned for missing blocks { @@ -256,6 +259,7 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella + depositReceipts: null, // depositReceipts is null pre-electra }, ], }; diff --git a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts index 3fb9cb8e1c79..c7de35d468df 100644 --- a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts +++ b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts @@ -24,6 +24,7 @@ describe("beaconBlocksMaybeBlobsByRange", () => { BELLATRIX_FORK_EPOCH: 0, CAPELLA_FORK_EPOCH: 0, DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, }); const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); const config = createBeaconConfig(chainConfig, genesisValidatorsRoot); diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index 1e9f614e8093..ad4dd4b88dac 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -7,8 +7,10 @@ import { PubkeyIndexMap, CachedBeaconStateBellatrix, BeaconStateBellatrix, + CachedBeaconStateElectra, + BeaconStateElectra, } from "@lodestar/state-transition"; -import {BeaconState, altair, bellatrix, ssz} from "@lodestar/types"; +import {BeaconState, altair, bellatrix, electra, ssz} from "@lodestar/types"; import {createBeaconConfig, ChainForkConfig} from "@lodestar/config"; import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; @@ -64,6 +66,7 @@ export function generateState( : generateValidators(numValidators, validatorOpts)); state.genesisTime = Math.floor(Date.now() / 1000); + state.slot = stateSlot; state.fork.previousVersion = config.GENESIS_FORK_VERSION; state.fork.currentVersion = config.GENESIS_FORK_VERSION; state.latestBlockHeader.bodyRoot = ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()); @@ -92,6 +95,12 @@ export function generateState( }; } + if (forkSeq >= ForkSeq.electra) { + const stateElectra = state as electra.BeaconState; + stateElectra.depositReceiptsStartIndex = 2023n; + stateElectra.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue(); + } + return config.getForkTypes(stateSlot).BeaconState.toViewDU(state); } @@ -137,6 +146,18 @@ export function generateCachedBellatrixState(opts?: TestBeaconState): CachedBeac }); } +/** + * This generates state with default pubkey + */ +export function generateCachedElectraState(opts?: TestBeaconState): CachedBeaconStateElectra { + const config = getConfig(ForkName.electra); + const state = generateState(opts, config); + return createCachedBeaconState(state as BeaconStateElectra, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }); +} export const zeroProtoBlock: ProtoBlock = { slot: 0, blockRoot: ZERO_HASH_HEX, diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index c33d942dabc5..335ec81870c6 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -4,6 +4,7 @@ import { computeSigningRoot, computeStartSlotAtEpoch, getShufflingDecisionBlock, + CachedBeaconStateAllForks, } from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; @@ -82,8 +83,8 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { }; const shufflingCache = new ShufflingCache(); - shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); - shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, state.epochCtx.currentShuffling.epoch); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, state.epochCtx.nextShuffling.epoch); const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); const forkChoice = { @@ -133,7 +134,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { getState: async () => state, // TODO: remove this once we have a better way to get state getStateSync: () => state, - } as Partial as IStateRegenerator; + } as unknown as Partial as IStateRegenerator; const chain = { clock, diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 0de1bee666ec..6d1fd9b75952 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -49,7 +49,7 @@ export const chainConfig: ChainConfig = { DENEB_FORK_VERSION: b("0x04000000"), DENEB_FORK_EPOCH: 269568, // March 13, 2024, 01:55:35pm UTC - // Electra + // ELECTRA ELECTRA_FORK_VERSION: b("0x05000000"), ELECTRA_FORK_EPOCH: Infinity, diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index c99a76d1ee40..44a28ca36ec7 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -45,8 +45,7 @@ export const chainConfig: ChainConfig = { // Deneb DENEB_FORK_VERSION: b("0x04000001"), DENEB_FORK_EPOCH: Infinity, - - // Electra + // ELECTRA ELECTRA_FORK_VERSION: b("0x05000001"), ELECTRA_FORK_EPOCH: Infinity, diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index aafd81c9250a..152c55ac6d15 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -115,7 +115,11 @@ export function upgradeLightClientHeader( // eslint-disable-next-line no-fallthrough case ForkName.electra: - throw Error("Not Implemented"); + (upgradedHeader as LightClientHeader).execution.depositReceiptsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.depositReceiptsRoot.defaultValue(); + + // Break if no further upgrades is required else fall through + if (ForkSeq[targetFork] <= ForkSeq.electra) break; } return upgradedHeader; } @@ -149,6 +153,12 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC } } + if (epoch < config.ELECTRA_FORK_EPOCH) { + if ((header as LightClientHeader).execution.depositReceiptsRoot !== undefined) { + return false; + } + } + return isValidMerkleBranch( config .getExecutionForkTypes(header.beacon.slot) diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 6a95e3ca632e..482e591123b6 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -93,6 +93,8 @@ export const { MAX_BLOB_COMMITMENTS_PER_BLOCK, MAX_BLOBS_PER_BLOCK, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, + + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, } = activePreset; //////////// @@ -244,3 +246,6 @@ export const KZG_COMMITMENT_SUBTREE_INDEX0 = KZG_COMMITMENT_GINDEX0 - 2 ** KZG_C // ssz.deneb.BlobSidecars.elementType.fixedSize export const BLOBSIDECAR_FIXED_SIZE = ACTIVE_PRESET === PresetName.minimal ? 131672 : 131928; + +// Electra Misc +export const UNSET_DEPOSIT_RECEIPTS_START_INDEX = 2n ** 64n - 1n; diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 42a705a07f03..86f5c39c539e 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -118,4 +118,7 @@ export const mainnetPreset: BeaconPreset = { MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096, MAX_BLOBS_PER_BLOCK: 6, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17, + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index b940841a0429..6239a8c1a3eb 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -119,4 +119,7 @@ export const minimalPreset: BeaconPreset = { MAX_BLOB_COMMITMENTS_PER_BLOCK: 16, MAX_BLOBS_PER_BLOCK: 6, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9, + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index 3c5ba6381131..57ee8230a77d 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -82,6 +82,9 @@ export type BeaconPreset = { MAX_BLOB_COMMITMENTS_PER_BLOCK: number; MAX_BLOBS_PER_BLOCK: number; KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: number; + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: number; }; /** @@ -167,6 +170,9 @@ export const beaconPresetTypes: BeaconPresetTypes = { MAX_BLOB_COMMITMENTS_PER_BLOCK: "number", MAX_BLOBS_PER_BLOCK: "number", KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: "number", + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: "number", }; type BeaconPresetTypes = { diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 6fcf70f58987..61b73ef7efc1 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -67,7 +67,12 @@ "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0", "@lodestar/utils": "^1.21.0", - "bigint-buffer": "^1.1.5" + "bigint-buffer": "^1.1.5", + "buffer-xor": "^2.0.2", + "immutable": "^4.3.2" + }, + "devDependencies": { + "@types/buffer-xor": "^2.0.0" }, "keywords": [ "ethereum", diff --git a/packages/state-transition/src/block/processDeposit.ts b/packages/state-transition/src/block/processDeposit.ts index 7ade79fd7739..5a2a892a5c9b 100644 --- a/packages/state-transition/src/block/processDeposit.ts +++ b/packages/state-transition/src/block/processDeposit.ts @@ -1,5 +1,5 @@ import {PublicKey, Signature, verify} from "@chainsafe/blst"; -import {phase0, ssz} from "@lodestar/types"; +import {BLSPubkey, Bytes32, UintNum64, phase0, ssz} from "@lodestar/types"; import {verifyMerkleBranch} from "@lodestar/utils"; import { @@ -11,6 +11,8 @@ import { MAX_EFFECTIVE_BALANCE, } from "@lodestar/params"; +import {DepositData} from "@lodestar/types/lib/phase0/types.js"; +import {DepositReceipt} from "@lodestar/types/lib/electra/types.js"; import {ZERO_HASH} from "../constants/index.js"; import {computeDomain, computeSigningRoot, increaseBalance} from "../util/index.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; @@ -22,8 +24,6 @@ import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; * PERF: Work depends on number of Deposit per block. On regular networks the average is 0 / block. */ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, deposit: phase0.Deposit): void { - const {config, validators, epochCtx} = state; - // verify the merkle branch if ( !verifyMerkleBranch( @@ -40,15 +40,29 @@ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, // deposits must be processed in order state.eth1DepositIndex += 1; - const pubkey = deposit.data.pubkey; // Drop tree - const amount = deposit.data.amount; - const cachedIndex = epochCtx.pubkey2index.get(pubkey); + applyDeposit(fork, state, deposit.data); +} + +/** + * Adds a new validator into the registry. Or increase balance if already exist. + * Follows applyDeposit() in consensus spec. Will be used by processDeposit() and processDepositReceipt() + * + */ +export function applyDeposit( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + deposit: DepositData | DepositReceipt +): void { + const {config, validators, epochCtx} = state; + const {pubkey, withdrawalCredentials, amount} = deposit; + + const cachedIndex = epochCtx.getValidatorIndex(pubkey); if (cachedIndex === undefined || !Number.isSafeInteger(cachedIndex) || cachedIndex >= validators.length) { // verify the deposit signature (proof of posession) which is not checked by the deposit contract const depositMessage = { - pubkey: deposit.data.pubkey, // Retain tree for hashing - withdrawalCredentials: deposit.data.withdrawalCredentials, // Retain tree for hashing - amount: deposit.data.amount, + pubkey, + withdrawalCredentials, + amount, }; // fork-agnostic domain since deposits are valid across forks const domain = computeDomain(DOMAIN_DEPOSIT, config.GENESIS_FORK_VERSION, ZERO_HASH); @@ -63,45 +77,55 @@ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, } catch (e) { return; // Catch all BLS errors: failed key validation, failed signature validation, invalid signature } + addValidatorToRegistry(fork, state, pubkey, withdrawalCredentials, amount); + } else { + // increase balance by deposit amount + increaseBalance(state, cachedIndex, amount); + } +} - // add validator and balance entries - const effectiveBalance = Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); - validators.push( - ssz.phase0.Validator.toViewDU({ - pubkey, - withdrawalCredentials: deposit.data.withdrawalCredentials, - activationEligibilityEpoch: FAR_FUTURE_EPOCH, - activationEpoch: FAR_FUTURE_EPOCH, - exitEpoch: FAR_FUTURE_EPOCH, - withdrawableEpoch: FAR_FUTURE_EPOCH, - effectiveBalance, - slashed: false, - }) - ); - state.balances.push(amount); +function addValidatorToRegistry( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + pubkey: BLSPubkey, + withdrawalCredentials: Bytes32, + amount: UintNum64 +): void { + const {validators, epochCtx} = state; + // add validator and balance entries + const effectiveBalance = Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); + validators.push( + ssz.phase0.Validator.toViewDU({ + pubkey, + withdrawalCredentials, + activationEligibilityEpoch: FAR_FUTURE_EPOCH, + activationEpoch: FAR_FUTURE_EPOCH, + exitEpoch: FAR_FUTURE_EPOCH, + withdrawableEpoch: FAR_FUTURE_EPOCH, + effectiveBalance, + slashed: false, + }) + ); + state.balances.push(amount); - const validatorIndex = validators.length - 1; - // Updating here is better than updating at once on epoch transition - // - Simplify genesis fn applyDeposits(): effectiveBalanceIncrements is populated immediately - // - Keep related code together to reduce risk of breaking this cache - // - Should have equal performance since it sets a value in a flat array - epochCtx.effectiveBalanceIncrementsSet(validatorIndex, effectiveBalance); + const validatorIndex = validators.length - 1; + // Updating here is better than updating at once on epoch transition + // - Simplify genesis fn applyDeposits(): effectiveBalanceIncrements is populated immediately + // - Keep related code together to reduce risk of breaking this cache + // - Should have equal performance since it sets a value in a flat array + epochCtx.effectiveBalanceIncrementsSet(validatorIndex, effectiveBalance); - // now that there is a new validator, update the epoch context with the new pubkey - epochCtx.addPubkey(validatorIndex, pubkey); + // now that there is a new validator, update the epoch context with the new pubkey + epochCtx.addPubkey(validatorIndex, pubkey); - // Only after altair: - if (fork >= ForkSeq.altair) { - const stateAltair = state as CachedBeaconStateAltair; + // Only after altair: + if (fork >= ForkSeq.altair) { + const stateAltair = state as CachedBeaconStateAltair; - stateAltair.inactivityScores.push(0); + stateAltair.inactivityScores.push(0); - // add participation caches - stateAltair.previousEpochParticipation.push(0); - stateAltair.currentEpochParticipation.push(0); - } - } else { - // increase balance by deposit amount - increaseBalance(state, cachedIndex, amount); + // add participation caches + stateAltair.previousEpochParticipation.push(0); + stateAltair.currentEpochParticipation.push(0); } } diff --git a/packages/state-transition/src/block/processDepositReceipt.ts b/packages/state-transition/src/block/processDepositReceipt.ts new file mode 100644 index 000000000000..140e38d634fd --- /dev/null +++ b/packages/state-transition/src/block/processDepositReceipt.ts @@ -0,0 +1,17 @@ +import {electra} from "@lodestar/types"; +import {ForkSeq, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; + +import {CachedBeaconStateElectra} from "../types.js"; +import {applyDeposit} from "./processDeposit.js"; + +export function processDepositReceipt( + fork: ForkSeq, + state: CachedBeaconStateElectra, + depositReceipt: electra.DepositReceipt +): void { + if (state.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + state.depositReceiptsStartIndex = BigInt(depositReceipt.index); + } + + applyDeposit(fork, state, depositReceipt); +} diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 38716bb42a40..7b8b39efe743 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -1,13 +1,15 @@ -import {BeaconBlockBody, capella} from "@lodestar/types"; -import {ForkSeq, MAX_DEPOSITS} from "@lodestar/params"; +import {BeaconBlockBody, capella, electra} from "@lodestar/types"; +import {ForkSeq} from "@lodestar/params"; -import {CachedBeaconStateAllForks, CachedBeaconStateCapella} from "../types.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateCapella, CachedBeaconStateElectra} from "../types.js"; +import {getEth1DepositCount} from "../util/deposit.js"; import {processAttestations} from "./processAttestations.js"; import {processProposerSlashing} from "./processProposerSlashing.js"; import {processAttesterSlashing} from "./processAttesterSlashing.js"; import {processDeposit} from "./processDeposit.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; +import {processDepositReceipt} from "./processDepositReceipt.js"; import {ProcessBlockOpts} from "./types.js"; export { @@ -17,6 +19,7 @@ export { processDeposit, processVoluntaryExit, processBlsToExecutionChange, + processDepositReceipt, }; export function processOperations( @@ -26,7 +29,7 @@ export function processOperations( opts: ProcessBlockOpts = {verifySignatures: true} ): void { // verify that outstanding deposits are processed up to the maximum number of deposits - const maxDeposits = Math.min(MAX_DEPOSITS, state.eth1Data.depositCount - state.eth1DepositIndex); + const maxDeposits = getEth1DepositCount(state); if (body.deposits.length !== maxDeposits) { throw new Error( `Block contains incorrect number of deposits: depositCount=${body.deposits.length} expected=${maxDeposits}` @@ -54,4 +57,10 @@ export function processOperations( processBlsToExecutionChange(state as CachedBeaconStateCapella, blsToExecutionChange); } } + + if (fork >= ForkSeq.electra) { + for (const depositReceipt of (body as electra.BeaconBlockBody).executionPayload.depositReceipts) { + processDepositReceipt(fork, state as CachedBeaconStateElectra, depositReceipt); + } + } } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 191aa7f3985c..41f8031d903e 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,4 +1,6 @@ import {PublicKey} from "@chainsafe/blst"; +import * as immutable from "immutable"; +import {fromHexString} from "@chainsafe/ssz"; import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainConfig} from "@lodestar/config"; import { @@ -29,8 +31,17 @@ import {computeEpochShuffling, EpochShuffling, getShufflingDecisionBlock} from " import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js"; import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js"; import {getTotalSlashingsByIncrement} from "../epoch/processSlashings.js"; +import {EpochCacheMetrics} from "../metrics.js"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js"; -import {Index2PubkeyCache, PubkeyIndexMap, syncPubkeys} from "./pubkeyCache.js"; +import { + Index2PubkeyCache, + PubkeyIndexMap, + UnfinalizedPubkeyIndexMap, + syncPubkeys, + toMemoryEfficientHexStr, + PubkeyHex, + newUnfinalizedPubkeyIndexMap, +} from "./pubkeyCache.js"; import {BeaconStateAllForks, BeaconStateAltair, ShufflingGetter} from "./types.js"; import { computeSyncCommitteeCache, @@ -82,23 +93,31 @@ type ProposersDeferred = {computed: false; seed: Uint8Array} | {computed: true; export class EpochCache { config: BeaconConfig; /** - * Unique globally shared pubkey registry. There should only exist one for the entire application. + * Unique globally shared finalized pubkey registry. There should only exist one for the entire application. * * TODO: this is a hack, we need a safety mechanism in case a bad eth1 majority vote is in, * or handle non finalized data differently, or use an immutable.js structure for cheap copies - * Warning: may contain pubkeys that do not yet exist in the current state, but do in a later processed state. + * + * New: This would include only validators whose activation_eligibility_epoch != FAR_FUTURE_EPOCH and hence it is + * insert only. Validators could be 1) Active 2) In the activation queue 3) Initialized but pending queued * * $VALIDATOR_COUNT x 192 char String -> Number Map */ pubkey2index: PubkeyIndexMap; /** - * Unique globally shared pubkey registry. There should only exist one for the entire application. + * Unique globally shared finalized pubkey registry. There should only exist one for the entire application. * - * Warning: may contain indices that do not yet exist in the current state, but do in a later processed state. + * New: This would include only validators whose activation_eligibility_epoch != FAR_FUTURE_EPOCH and hence it is + * insert only. Validators could be 1) Active 2) In the activation queue 3) Initialized but pending queued * * $VALIDATOR_COUNT x BLST deserialized pubkey (Jacobian coordinates) */ index2pubkey: Index2PubkeyCache; + /** + * Unique pubkey registry shared in the same fork. There should only exist one for the fork. + */ + unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + /** * Indexes of the block proposers for the current epoch. * @@ -198,11 +217,21 @@ export class EpochCache { // TODO: Helper stats epoch: Epoch; syncPeriod: SyncPeriod; + /** + * state.validators.length of every state at epoch boundary + * They are saved in increasing order of epoch. + * The first validator length in the list corresponds to the state AFTER the latest finalized checkpoint state. ie. state.finalizedCheckpoint.epoch - 1 + * The last validator length corresponds to the latest epoch state ie. this.epoch + * eg. latest epoch = 105, latest finalized cp state epoch = 102 + * then the list will be (in terms of epoch) [103, 104, 105] + */ + private historicalValidatorLengths: immutable.List; constructor(data: { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; + unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; @@ -225,10 +254,12 @@ export class EpochCache { nextSyncCommitteeIndexed: SyncCommitteeCache; epoch: Epoch; syncPeriod: SyncPeriod; + historialValidatorLengths: immutable.List; }) { this.config = data.config; this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; + this.unfinalizedPubkey2index = data.unfinalizedPubkey2index; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; @@ -251,11 +282,12 @@ export class EpochCache { this.nextSyncCommitteeIndexed = data.nextSyncCommitteeIndexed; this.epoch = data.epoch; this.syncPeriod = data.syncPeriod; + this.historicalValidatorLengths = data.historialValidatorLengths; } /** * Create an epoch cache - * @param validators cached validators that matches `state.validators` + * @param state a finalized beacon state. Passing in unfinalized state may cause unexpected behaviour eg. empty unfinalized cache * * SLOW CODE - 🐢 */ @@ -431,6 +463,8 @@ export class EpochCache { config, pubkey2index, index2pubkey, + // `createFromFinalizedState()` creates cache with empty unfinalizedPubkey2index. Be cautious to only pass in finalized state + unfinalizedPubkey2index: newUnfinalizedPubkeyIndexMap(), proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, @@ -454,6 +488,7 @@ export class EpochCache { nextSyncCommitteeIndexed, epoch: currentEpoch, syncPeriod: computeSyncPeriodAtEpoch(currentEpoch), + historialValidatorLengths: immutable.List(), }); } @@ -469,6 +504,8 @@ export class EpochCache { // Common append-only structures shared with all states, no need to clone pubkey2index: this.pubkey2index, index2pubkey: this.index2pubkey, + // No need to clone this reference. On each mutation the `unfinalizedPubkey2index` reference is replaced, @see `addPubkey` + unfinalizedPubkey2index: this.unfinalizedPubkey2index, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, @@ -495,6 +532,7 @@ export class EpochCache { nextSyncCommitteeIndexed: this.nextSyncCommitteeIndexed, epoch: this.epoch, syncPeriod: this.syncPeriod, + historialValidatorLengths: this.historicalValidatorLengths, }); } @@ -505,6 +543,7 @@ export class EpochCache { afterProcessEpoch( state: BeaconStateAllForks, epochTransitionCache: { + indicesEligibleForActivationQueue: ValidatorIndex[]; nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; nextEpochShufflingActiveIndicesLength: number; nextEpochTotalActiveBalanceByIncrement: number; @@ -579,6 +618,25 @@ export class EpochCache { // ``` this.epoch = computeEpochAtSlot(state.slot); this.syncPeriod = computeSyncPeriodAtEpoch(this.epoch); + // ELECTRA Only: Add current cpState.validators.length + // Only keep validatorLength for epochs after finalized cpState.epoch + // eg. [100(epoch 1), 102(epoch 2)].push(104(epoch 3)), this.epoch = 3, finalized cp epoch = 1 + // We keep the last (3 - 1) items = [102, 104] + if (currEpoch >= this.config.ELECTRA_FORK_EPOCH) { + this.historicalValidatorLengths = this.historicalValidatorLengths.push(state.validators.length); + + // If number of validatorLengths we want to keep exceeds the current list size, it implies + // finalized checkpoint hasn't advanced, and no need to slice + const hasFinalizedCpAdvanced = + this.epoch - state.finalizedCheckpoint.epoch < this.historicalValidatorLengths.size; + + if (hasFinalizedCpAdvanced) { + // We use finalized cp epoch - this.epoch which is a negative number to keep the last n entries and discard the rest + this.historicalValidatorLengths = this.historicalValidatorLengths.slice( + state.finalizedCheckpoint.epoch - this.epoch + ); + } + } } beforeEpochTransition(): void { @@ -766,9 +824,75 @@ export class EpochCache { return isAggregatorFromCommitteeLength(committee.length, slotSignature); } + /** + * Return finalized pubkey given the validator index. + * Only finalized pubkey as we do not store unfinalized pubkey because no where in the spec has a + * need to make such enquiry + */ + getPubkey(index: ValidatorIndex): PublicKey | undefined { + return this.index2pubkey[index]; + } + + getValidatorIndex(pubkey: Uint8Array | PubkeyHex): ValidatorIndex | undefined { + if (this.isAfterElectra()) { + return this.pubkey2index.get(pubkey) ?? this.unfinalizedPubkey2index.get(toMemoryEfficientHexStr(pubkey)); + } else { + return this.pubkey2index.get(pubkey); + } + } + + /** + * + * Add unfinalized pubkeys + * + */ addPubkey(index: ValidatorIndex, pubkey: Uint8Array): void { + if (this.isAfterElectra()) { + this.addUnFinalizedPubkey(index, pubkey); + } else { + // deposit mechanism pre ELECTRA follows a safe distance with assumption + // that they are already canonical + this.addFinalizedPubkey(index, pubkey); + } + } + + addUnFinalizedPubkey(index: ValidatorIndex, pubkey: PubkeyHex | Uint8Array, metrics?: EpochCacheMetrics): void { + this.unfinalizedPubkey2index = this.unfinalizedPubkey2index.set(toMemoryEfficientHexStr(pubkey), index); + metrics?.newUnFinalizedPubkey.inc(); + } + + addFinalizedPubkeys(pubkeyMap: UnfinalizedPubkeyIndexMap, metrics?: EpochCacheMetrics): void { + pubkeyMap.forEach((index, pubkey) => this.addFinalizedPubkey(index, pubkey, metrics)); + } + + /** + * Add finalized validator index and pubkey into finalized cache. + * Since addFinalizedPubkey() primarily takes pubkeys from unfinalized cache, it can take pubkey hex string directly + */ + addFinalizedPubkey(index: ValidatorIndex, pubkey: PubkeyHex | Uint8Array, metrics?: EpochCacheMetrics): void { + const existingIndex = this.pubkey2index.get(pubkey); + + if (existingIndex !== undefined) { + if (existingIndex === index) { + // Repeated insert. + metrics?.finalizedPubkeyDuplicateInsert.inc(); + return; + } else { + // attempt to insert the same pubkey with different index, should never happen. + throw Error("inserted existing pubkey into finalizedPubkey2index cache with a different index"); + } + } + this.pubkey2index.set(pubkey, index); - this.index2pubkey[index] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + const pubkeyBytes = pubkey instanceof Uint8Array ? pubkey : fromHexString(pubkey); + this.index2pubkey[index] = PublicKey.fromBytes(pubkeyBytes); // Optimize for aggregation + } + + /** + * Delete pubkeys from unfinalized cache + */ + deleteUnfinalizedPubkeys(pubkeys: Iterable): void { + this.unfinalizedPubkey2index = this.unfinalizedPubkey2index.deleteAll(pubkeys); } getShufflingAtSlot(slot: Slot): EpochShuffling { @@ -845,15 +969,53 @@ export class EpochCache { } effectiveBalanceIncrementsSet(index: number, effectiveBalance: number): void { - if (index >= this.effectiveBalanceIncrements.length) { - // Clone and extend effectiveBalanceIncrements + if (this.isAfterElectra()) { + // TODO: electra + // getting length and setting getEffectiveBalanceIncrementsByteLen is not fork safe + // so each time we add an index, we should new the Uint8Array to keep it forksafe + // one simple optimization could be to increment the length once per block rather + // on each add/set + // + // there could still be some unused length remaining from the prev ELECTRA padding + const newLength = + index >= this.effectiveBalanceIncrements.length ? index + 1 : this.effectiveBalanceIncrements.length; const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements = new Uint8Array(newLength); this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); + } else { + if (index >= this.effectiveBalanceIncrements.length) { + // Clone and extend effectiveBalanceIncrements + const effectiveBalanceIncrements = this.effectiveBalanceIncrements; + this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); + } } this.effectiveBalanceIncrements[index] = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); } + + isAfterElectra(): boolean { + return this.epoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity); + } + + getValidatorCountAtEpoch(targetEpoch: Epoch): number | undefined { + const currentEpoch = this.epoch; + + if (targetEpoch === currentEpoch) { + return this.historicalValidatorLengths.get(-1); + } + + // Attempt to get validator count from future epoch + if (targetEpoch > currentEpoch) { + return undefined; + } + + // targetEpoch is so far back that historicalValidatorLengths doesnt contain such info + if (targetEpoch < currentEpoch - this.historicalValidatorLengths.size + 1) { + return undefined; + } + return this.historicalValidatorLengths.get(targetEpoch - currentEpoch - 1); + } } function getEffectiveBalanceIncrementsByteLen(validatorCount: number): number { diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index b2b45ca09d25..a7eacb7961fd 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,9 +1,18 @@ import {PublicKey} from "@chainsafe/blst"; +import * as immutable from "immutable"; import {ValidatorIndex, phase0} from "@lodestar/types"; +import {BeaconStateAllForks} from "./types.js"; export type Index2PubkeyCache = PublicKey[]; +/** + * OrderedMap preserves the order of entries in which they are `set()`. + * We assume `values()` yields validator indices in strictly increasing order + * as new validator indices are assigned in increasing order. + * EIP-6914 will break this assumption. + */ +export type UnfinalizedPubkeyIndexMap = immutable.Map; -type PubkeyHex = string; +export type PubkeyHex = string; /** * toHexString() creates hex strings via string concatenation, which are very memory inefficient. @@ -13,7 +22,7 @@ type PubkeyHex = string; * * See https://github.com/ChainSafe/lodestar/issues/3446 */ -function toMemoryEfficientHexStr(hex: Uint8Array | string): string { +export function toMemoryEfficientHexStr(hex: Uint8Array | string): string { if (typeof hex === "string") { if (hex.startsWith("0x")) { hex = hex.slice(2); @@ -24,6 +33,13 @@ function toMemoryEfficientHexStr(hex: Uint8Array | string): string { return Buffer.from(hex.buffer, hex.byteOffset, hex.byteLength).toString("hex"); } +/** + * A wrapper for calling immutable.js. To abstract the initialization of UnfinalizedPubkeyIndexMap + */ +export function newUnfinalizedPubkeyIndexMap(): UnfinalizedPubkeyIndexMap { + return immutable.Map(); +} + export class PubkeyIndexMap { // We don't really need the full pubkey. We could just use the first 20 bytes like an Ethereum address readonly map = new Map(); @@ -39,7 +55,7 @@ export class PubkeyIndexMap { return this.map.get(toMemoryEfficientHexStr(key)); } - set(key: Uint8Array, value: ValidatorIndex): void { + set(key: Uint8Array | PubkeyHex, value: ValidatorIndex): void { this.map.set(toMemoryEfficientHexStr(key), value); } } diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index f4e637e5d665..0435e8829d21 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -10,6 +10,7 @@ import { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, } from "./types.js"; import {RewardCache, createEmptyRewardCache} from "./rewardCache.js"; @@ -130,11 +131,13 @@ export type CachedBeaconStateAltair = CachedBeaconState; export type CachedBeaconStateBellatrix = CachedBeaconState; export type CachedBeaconStateCapella = CachedBeaconState; export type CachedBeaconStateDeneb = CachedBeaconState; +export type CachedBeaconStateElectra = CachedBeaconState; export type CachedBeaconStateAllForks = CachedBeaconState; export type CachedBeaconStateExecutions = CachedBeaconState; /** * Create CachedBeaconState computing a new EpochCache instance + * TODO ELECTRA: rename this to createFinalizedCachedBeaconState() as it's intended for finalized state only */ export function createCachedBeaconState( state: T, @@ -158,7 +161,7 @@ export function createCachedBeaconState( * Create a CachedBeaconState given a cached seed state and state bytes * This guarantees that the returned state shares the same tree with the seed state * Check loadState() api for more details - * // TODO: rename to loadUnfinalizedCachedBeaconState() due to EIP-6110 + * // TODO: rename to loadUnfinalizedCachedBeaconState() due to ELECTRA */ export function loadCachedBeaconState( cachedSeedState: T, diff --git a/packages/state-transition/src/cache/types.ts b/packages/state-transition/src/cache/types.ts index d6d8a3c37904..b3fe6fc8ed5b 100644 --- a/packages/state-transition/src/cache/types.ts +++ b/packages/state-transition/src/cache/types.ts @@ -8,6 +8,7 @@ export type BeaconStateAltair = CompositeViewDU>; export type BeaconStateCapella = CompositeViewDU>; export type BeaconStateDeneb = CompositeViewDU>; +export type BeaconStateElectra = CompositeViewDU>; export type BeaconStateAllForks = CompositeViewDU>; export type BeaconStateExecutions = CompositeViewDU>; diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 0ef460e784af..4ed801e3c490 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -11,6 +11,7 @@ export type { CachedBeaconStateBellatrix, CachedBeaconStateCapella, CachedBeaconStateDeneb, + CachedBeaconStateElectra, CachedBeaconStateAllForks, CachedBeaconStateExecutions, // Non-cached states @@ -19,6 +20,7 @@ export type { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, BeaconStateAllForks, BeaconStateExecutions, } from "./types.js"; @@ -42,7 +44,12 @@ export { export {type EpochTransitionCache, beforeProcessEpoch} from "./cache/epochTransitionCache.js"; // Aux data-structures -export {PubkeyIndexMap, type Index2PubkeyCache} from "./cache/pubkeyCache.js"; +export { + PubkeyIndexMap, + type Index2PubkeyCache, + type UnfinalizedPubkeyIndexMap, + newUnfinalizedPubkeyIndexMap, +} from "./cache/pubkeyCache.js"; export { type EffectiveBalanceIncrements, diff --git a/packages/state-transition/src/metrics.ts b/packages/state-transition/src/metrics.ts index 12cec46d9a49..48e0bc921876 100644 --- a/packages/state-transition/src/metrics.ts +++ b/packages/state-transition/src/metrics.ts @@ -30,6 +30,11 @@ export type BeaconStateTransitionMetrics = { ) => void; }; +export type EpochCacheMetrics = { + finalizedPubkeyDuplicateInsert: Gauge; + newUnFinalizedPubkey: Gauge; +}; + export function onStateCloneMetrics( state: CachedBeaconStateAllForks, metrics: BeaconStateTransitionMetrics, diff --git a/packages/state-transition/src/signatureSets/attesterSlashings.ts b/packages/state-transition/src/signatureSets/attesterSlashings.ts index f0de50e5d0b2..256582afe368 100644 --- a/packages/state-transition/src/signatureSets/attesterSlashings.ts +++ b/packages/state-transition/src/signatureSets/attesterSlashings.ts @@ -27,13 +27,12 @@ export function getIndexedAttestationBigintSignatureSet( state: CachedBeaconStateAllForks, indexedAttestation: phase0.IndexedAttestationBigint ): ISignatureSet { - const {index2pubkey} = state.epochCtx; const slot = computeStartSlotAtEpoch(Number(indexedAttestation.data.target.epoch as bigint)); const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_ATTESTER, slot); return { type: SignatureSetType.aggregate, - pubkeys: indexedAttestation.attestingIndices.map((i) => index2pubkey[i]), + pubkeys: indexedAttestation.attestingIndices.map((i) => state.epochCtx.index2pubkey[i]), signingRoot: computeSigningRoot(ssz.phase0.AttestationDataBigint, indexedAttestation.data, domain), signature: indexedAttestation.signature, }; diff --git a/packages/state-transition/src/slot/index.ts b/packages/state-transition/src/slot/index.ts index 6c4add1d1230..b05bd7ac93f2 100644 --- a/packages/state-transition/src/slot/index.ts +++ b/packages/state-transition/src/slot/index.ts @@ -7,6 +7,7 @@ export {upgradeStateToAltair} from "./upgradeStateToAltair.js"; export {upgradeStateToBellatrix} from "./upgradeStateToBellatrix.js"; export {upgradeStateToCapella} from "./upgradeStateToCapella.js"; export {upgradeStateToDeneb} from "./upgradeStateToDeneb.js"; +export {upgradeStateToElectra} from "./upgradeStateToElectra.js"; /** * Dial state to next slot. Common for all forks diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts new file mode 100644 index 000000000000..19cac8811f77 --- /dev/null +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -0,0 +1,33 @@ +import {ssz} from "@lodestar/types"; +import {UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {CachedBeaconStateDeneb} from "../types.js"; +import {CachedBeaconStateElectra, getCachedBeaconState} from "../cache/stateCache.js"; + +/** + * Upgrade a state from Capella to Deneb. + */ +export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): CachedBeaconStateElectra { + const {config} = stateDeneb; + + const stateElectraNode = ssz.deneb.BeaconState.commitViewDU(stateDeneb); + const stateElectraView = ssz.electra.BeaconState.getViewDU(stateElectraNode); + + const stateElectra = getCachedBeaconState(stateElectraView, stateDeneb); + + stateElectra.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: stateDeneb.fork.currentVersion, + currentVersion: config.ELECTRA_FORK_VERSION, + epoch: stateDeneb.epochCtx.epoch, + }); + + // latestExecutionPayloadHeader's depositReceiptsRoot set to zeros by default + // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + + // Commit new added fields ViewDU to the root node + stateElectra.commit(); + // Clear cache to ensure the cache of deneb fields is not used by new ELECTRA fields + stateElectra["clearCache"](); + + return stateElectra; +} diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index a454bf4b34d0..5f433918415a 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -9,6 +9,7 @@ import { CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateCapella, + CachedBeaconStateDeneb, } from "./types.js"; import {computeEpochAtSlot} from "./util/index.js"; import {verifyProposerSignature} from "./signatureSets/index.js"; @@ -18,6 +19,7 @@ import { upgradeStateToBellatrix, upgradeStateToCapella, upgradeStateToDeneb, + upgradeStateToElectra, } from "./slot/index.js"; import {processBlock} from "./block/index.js"; import {EpochTransitionStep, processEpoch} from "./epoch/index.js"; @@ -239,6 +241,9 @@ function processSlotsWithTransientCache( if (stateSlot === config.DENEB_FORK_EPOCH) { postState = upgradeStateToDeneb(postState as CachedBeaconStateCapella) as CachedBeaconStateAllForks; } + if (stateSlot === config.ELECTRA_FORK_EPOCH) { + postState = upgradeStateToElectra(postState as CachedBeaconStateDeneb) as CachedBeaconStateAllForks; + } } else { postState.slot++; } diff --git a/packages/state-transition/src/types.ts b/packages/state-transition/src/types.ts index 6b6b1f6260b2..d3a1ed69a7a9 100644 --- a/packages/state-transition/src/types.ts +++ b/packages/state-transition/src/types.ts @@ -9,6 +9,7 @@ export type { CachedBeaconStateBellatrix, CachedBeaconStateCapella, CachedBeaconStateDeneb, + CachedBeaconStateElectra, } from "./cache/stateCache.js"; export type { @@ -19,4 +20,5 @@ export type { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, } from "./cache/types.js"; diff --git a/packages/state-transition/src/util/deposit.ts b/packages/state-transition/src/util/deposit.ts new file mode 100644 index 000000000000..493099fdd982 --- /dev/null +++ b/packages/state-transition/src/util/deposit.ts @@ -0,0 +1,24 @@ +import {ForkSeq, MAX_DEPOSITS} from "@lodestar/params"; +import {UintNum64, phase0} from "@lodestar/types"; +import {CachedBeaconStateAllForks, CachedBeaconStateElectra} from "../types.js"; + +export function getEth1DepositCount(state: CachedBeaconStateAllForks, eth1Data?: phase0.Eth1Data): UintNum64 { + const eth1DataToUse = eth1Data ?? state.eth1Data; + if (state.config.getForkSeq(state.slot) >= ForkSeq.electra) { + const electraState = state as CachedBeaconStateElectra; + // eth1DataIndexLimit = min(UintNum64, UintBn64) can be safely casted as UintNum64 + // since the result lies within upper and lower bound of UintNum64 + const eth1DataIndexLimit: UintNum64 = + eth1DataToUse.depositCount < electraState.depositReceiptsStartIndex + ? eth1DataToUse.depositCount + : Number(electraState.depositReceiptsStartIndex); + + if (state.eth1DepositIndex < eth1DataIndexLimit) { + return Math.min(MAX_DEPOSITS, eth1DataIndexLimit - state.eth1DepositIndex); + } else { + return 0; + } + } else { + return Math.min(MAX_DEPOSITS, eth1DataToUse.depositCount - state.eth1DepositIndex); + } +} diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 1c5046354fcb..d9a88e5475b5 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -2,6 +2,7 @@ import { bellatrix, capella, deneb, + electra, isBlindedBeaconBlockBody, ssz, BeaconBlock, @@ -170,5 +171,10 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio ).excessBlobGas; } + if (fork >= ForkSeq.electra) { + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositReceiptsRoot = + ssz.electra.DepositReceipts.hashTreeRoot((payload as electra.ExecutionPayload).depositReceipts); + } + return bellatrixPayloadFields; } diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 1041c33d0eb3..6c700436d7f6 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -7,6 +7,7 @@ import { GENESIS_EPOCH, GENESIS_SLOT, MAX_EFFECTIVE_BALANCE, + UNSET_DEPOSIT_RECEIPTS_START_INDEX, } from "@lodestar/params"; import {Bytes32, phase0, Root, ssz, TimeSeconds} from "@lodestar/types"; @@ -214,6 +215,7 @@ export function initializeBeaconStateFromEth1( | typeof ssz.bellatrix.ExecutionPayloadHeader | typeof ssz.capella.ExecutionPayloadHeader | typeof ssz.deneb.ExecutionPayloadHeader + | typeof ssz.electra.ExecutionPayloadHeader > ): CachedBeaconStateAllForks { const stateView = getGenesisBeaconState( @@ -284,6 +286,16 @@ export function initializeBeaconStateFromEth1( ssz.deneb.ExecutionPayloadHeader.defaultViewDU(); } + if (GENESIS_SLOT >= config.ELECTRA_FORK_EPOCH) { + const stateElectra = state as CompositeViewDU; + stateElectra.fork.previousVersion = config.ELECTRA_FORK_VERSION; + stateElectra.fork.currentVersion = config.ELECTRA_FORK_VERSION; + stateElectra.latestExecutionPayloadHeader = + (executionPayloadHeader as CompositeViewDU) ?? + ssz.electra.ExecutionPayloadHeader.defaultViewDU(); + stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + } + state.commit(); return state; diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index 3f2e91da9a77..ba998f65b254 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -23,3 +23,4 @@ export * from "./slot.js"; export * from "./syncCommittee.js"; export * from "./validator.js"; export * from "./weakSubjectivity.js"; +export * from "./deposit.js"; diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 2891cd3e6216..9b05b8947f73 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -1,8 +1,9 @@ +import {fromHexString} from "@chainsafe/ssz"; import {describe, it, expect} from "vitest"; import {Epoch, ssz, RootHex} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {config as defaultConfig} from "@lodestar/config/default"; -import {createBeaconConfig} from "@lodestar/config"; +import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {createCachedBeaconStateTest} from "../utils/state.js"; import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; @@ -28,6 +29,65 @@ describe("CachedBeaconState", () => { expect(state2.epochCtx.epoch).toBe(0); }); + it("Clone and mutate cache pre-Electra", () => { + const stateView = ssz.altair.BeaconState.defaultViewDU(); + const state1 = createCachedBeaconStateTest(stateView); + + const pubkey1 = fromHexString( + "0x84105a985058fc8740a48bf1ede9d223ef09e8c6b1735ba0a55cf4a9ff2ff92376b778798365e488dab07a652eb04576" + ); + const index1 = 123; + const pubkey2 = fromHexString( + "0xa41726266b1d83ef609d759ba7796d54cfe549154e01e4730a3378309bc81a7638140d7e184b33593c072595f23f032d" + ); + const index2 = 456; + + state1.epochCtx.addPubkey(index1, pubkey1); + + const state2 = state1.clone(); + state2.epochCtx.addPubkey(index2, pubkey2); + + expect(state1.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state2.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state1.epochCtx.getValidatorIndex(pubkey2)).toBe(index2); + expect(state2.epochCtx.getValidatorIndex(pubkey2)).toBe(index2); + }); + + /* eslint-disable @typescript-eslint/naming-convention */ + it("Clone and mutate cache post-Electra", () => { + const stateView = ssz.electra.BeaconState.defaultViewDU(); + const state1 = createCachedBeaconStateTest( + stateView, + createChainForkConfig({ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + }), + {skipSyncCommitteeCache: true, skipSyncPubkeys: true} + ); + + const pubkey1 = fromHexString( + "0x84105a985058fc8740a48bf1ede9d223ef09e8c6b1735ba0a55cf4a9ff2ff92376b778798365e488dab07a652eb04576" + ); + const index1 = 123; + const pubkey2 = fromHexString( + "0xa41726266b1d83ef609d759ba7796d54cfe549154e01e4730a3378309bc81a7638140d7e184b33593c072595f23f032d" + ); + const index2 = 456; + + state1.epochCtx.addPubkey(index1, pubkey1); + + const state2 = state1.clone(); + state2.epochCtx.addPubkey(index2, pubkey2); + + expect(state1.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state2.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state1.epochCtx.getValidatorIndex(pubkey2)).toBe(undefined); + expect(state2.epochCtx.getValidatorIndex(pubkey2)).toBe(index2); + }); + it("Auto-commit on hashTreeRoot", () => { // Use Checkpoint instead of BeaconState to speed up the test const cp1 = ssz.phase0.Checkpoint.defaultViewDU(); diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index 75ba415c1bea..df9b052542f9 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -5,6 +5,7 @@ import {createBeaconConfig, ChainForkConfig, createChainForkConfig} from "@lodes import {config as chainConfig} from "@lodestar/config/default"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; +import {upgradeStateToElectra} from "../../src/slot/upgradeStateToElectra.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; @@ -24,6 +25,21 @@ describe("upgradeState", () => { const newState = upgradeStateToDeneb(stateView); expect(() => newState.toValue()).not.toThrow(); }); + it("upgradeStateToElectra", () => { + const denebState = ssz.deneb.BeaconState.defaultViewDU(); + const config = getConfig(ForkName.deneb); + const stateView = createCachedBeaconState( + denebState, + { + config: createBeaconConfig(config, denebState.genesisValidatorsRoot), + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }, + {skipSyncCommitteeCache: true} + ); + const newState = upgradeStateToElectra(stateView); + expect(() => newState.toValue()).not.toThrow(); + }); }); const ZERO_HASH = Buffer.alloc(32, 0); diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts new file mode 100644 index 000000000000..e8f7f7a86af8 --- /dev/null +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -0,0 +1,99 @@ +import {describe, it, expect} from "vitest"; +import {ssz} from "@lodestar/types"; +import {createChainForkConfig} from "@lodestar/config"; +import {MAX_DEPOSITS} from "@lodestar/params"; +import {CachedBeaconStateElectra, getEth1DepositCount} from "../../../src/index.js"; +import {createCachedBeaconStateTest} from "../../utils/state.js"; + +describe("getEth1DepositCount", () => { + it("Pre Electra", () => { + const stateView = ssz.altair.BeaconState.defaultViewDU(); + const preElectraState = createCachedBeaconStateTest(stateView); + + if (preElectraState.epochCtx.isAfterElectra()) { + throw Error("Not a pre-Electra state"); + } + + preElectraState.eth1Data.depositCount = 123; + + // 1. Should get less than MAX_DEPOSIT + preElectraState.eth1DepositIndex = 120; + expect(getEth1DepositCount(preElectraState)).toBe(3); + + // 2. Should get MAX_DEPOSIT + preElectraState.eth1DepositIndex = 100; + expect(getEth1DepositCount(preElectraState)).toBe(MAX_DEPOSITS); + }); + it("Post Electra with eth1 deposit", () => { + const stateView = ssz.electra.BeaconState.defaultViewDU(); + const postElectraState = createCachedBeaconStateTest( + stateView, + createChainForkConfig({ + /* eslint-disable @typescript-eslint/naming-convention */ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + }), + {skipSyncCommitteeCache: true, skipSyncPubkeys: true} + ) as CachedBeaconStateElectra; + + if (!postElectraState.epochCtx.isAfterElectra()) { + throw Error("Not a post-Electra state"); + } + + postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.eth1Data.depositCount = 995; + + // 1. Should get less than MAX_DEPOSIT + postElectraState.eth1DepositIndex = 990; + expect(getEth1DepositCount(postElectraState)).toBe(5); + + // 2. Should get MAX_DEPOSIT + postElectraState.eth1DepositIndex = 100; + expect(getEth1DepositCount(postElectraState)).toBe(MAX_DEPOSITS); + + // 3. Should be 0 + postElectraState.eth1DepositIndex = 1000; + expect(getEth1DepositCount(postElectraState)).toBe(0); + }); + it("Post Electra without eth1 deposit", () => { + const stateView = ssz.electra.BeaconState.defaultViewDU(); + const postElectraState = createCachedBeaconStateTest( + stateView, + createChainForkConfig({ + /* eslint-disable @typescript-eslint/naming-convention */ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + }), + {skipSyncCommitteeCache: true, skipSyncPubkeys: true} + ) as CachedBeaconStateElectra; + + if (!postElectraState.epochCtx.isAfterElectra()) { + throw Error("Not a post-Electra state"); + } + + postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.eth1Data.depositCount = 1005; + + // Before eth1DepositIndex reaching the start index + // 1. Should get less than MAX_DEPOSIT + postElectraState.eth1DepositIndex = 990; + expect(getEth1DepositCount(postElectraState)).toBe(10); + + // 2. Should get MAX_DEPOSIT + postElectraState.eth1DepositIndex = 983; + expect(getEth1DepositCount(postElectraState)).toBe(MAX_DEPOSITS); + + // After eth1DepositIndex reaching the start index + // 1. Should be 0 + postElectraState.eth1DepositIndex = 1000; + expect(getEth1DepositCount(postElectraState)).toBe(0); + postElectraState.eth1DepositIndex = 1003; + expect(getEth1DepositCount(postElectraState)).toBe(0); + }); +}); diff --git a/packages/types/package.json b/packages/types/package.json index 9a36e311def2..f29229485e99 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -29,6 +29,9 @@ "./deneb": { "import": "./lib/deneb/index.js" }, + "./electra": { + "import": "./lib/electra/index.js" + }, "./phase0": { "import": "./lib/phase0/index.js" } diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 30690a499845..b347b20df9c7 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -1,12 +1,37 @@ -import {ContainerType} from "@chainsafe/ssz"; +import {ContainerType, ListCompositeType, VectorCompositeType} from "@chainsafe/ssz"; +import { + HISTORICAL_ROOTS_LIMIT, + BLOCK_BODY_EXECUTION_PAYLOAD_DEPTH as EXECUTION_PAYLOAD_DEPTH, + EPOCHS_PER_SYNC_COMMITTEE_PERIOD, + SLOTS_PER_EPOCH, + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, +} from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; +import {ssz as phase0Ssz} from "../phase0/index.js"; +import {ssz as altairSsz} from "../altair/index.js"; +import {ssz as bellatrixSsz} from "../bellatrix/index.js"; +import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; -const {BLSSignature} = primitiveSsz; +const {UintNum64, Slot, Root, BLSSignature, UintBn256, Bytes32, BLSPubkey, DepositIndex, UintBn64} = primitiveSsz; + +export const DepositReceipt = new ContainerType( + { + pubkey: BLSPubkey, + withdrawalCredentials: Bytes32, + amount: UintNum64, + signature: BLSSignature, + index: DepositIndex, + }, + {typeName: "DepositReceipt", jsonCase: "eth2"} +); + +export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, + depositReceipts: DepositReceipts, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -14,13 +39,18 @@ export const ExecutionPayload = new ContainerType( export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, + depositReceiptsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); +// We have to preserve Fields ordering while changing the type of ExecutionPayload export const BeaconBlockBody = new ContainerType( { - ...denebSsz.BeaconBlockBody.fields, + ...altairSsz.BeaconBlockBody.fields, + executionPayload: ExecutionPayload, // Modified in ELECTRA + blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, + blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, }, {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -28,28 +58,25 @@ export const BeaconBlockBody = new ContainerType( export const BeaconBlock = new ContainerType( { ...denebSsz.BeaconBlock.fields, + body: BeaconBlockBody, // Modified in ELECTRA }, {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} ); export const SignedBeaconBlock = new ContainerType( { - message: BeaconBlock, + message: BeaconBlock, // Modified in ELECTRA signature: BLSSignature, }, {typeName: "SignedBeaconBlock", jsonCase: "eth2"} ); -export const BlobSidecar = new ContainerType( - { - ...denebSsz.BlobSidecar.fields, - }, - {typeName: "BlobSidecar", jsonCase: "eth2"} -); - export const BlindedBeaconBlockBody = new ContainerType( { - ...denebSsz.BlindedBeaconBlockBody.fields, + ...altairSsz.BeaconBlockBody.fields, + executionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, + blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, }, {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -57,13 +84,14 @@ export const BlindedBeaconBlockBody = new ContainerType( export const BlindedBeaconBlock = new ContainerType( { ...denebSsz.BlindedBeaconBlock.fields, + body: BlindedBeaconBlockBody, // Modified in ELECTRA }, {typeName: "BlindedBeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} ); export const SignedBlindedBeaconBlock = new ContainerType( { - message: BlindedBeaconBlock, + message: BlindedBeaconBlock, // Modified in ELECTRA signature: BLSSignature, }, {typeName: "SignedBlindedBeaconBlock", jsonCase: "eth2"} @@ -71,7 +99,10 @@ export const SignedBlindedBeaconBlock = new ContainerType( export const BuilderBid = new ContainerType( { - ...denebSsz.BuilderBid.fields, + header: ExecutionPayloadHeader, // Modified in ELECTRA + blindedBlobsBundle: denebSsz.BlobKzgCommitments, + value: UintBn256, + pubkey: BLSPubkey, }, {typeName: "BuilderBid", jsonCase: "eth2"} ); @@ -86,63 +117,133 @@ export const SignedBuilderBid = new ContainerType( export const ExecutionPayloadAndBlobsBundle = new ContainerType( { - ...denebSsz.ExecutionPayloadAndBlobsBundle.fields, + executionPayload: ExecutionPayload, // Modified in ELECTRA + blobsBundle: denebSsz.BlobsBundle, }, {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} ); +// We don't spread deneb.BeaconState fields since we need to replace +// latestExecutionPayloadHeader and we cannot keep order doing that export const BeaconState = new ContainerType( { - ...denebSsz.BeaconState.fields, + genesisTime: UintNum64, + genesisValidatorsRoot: Root, + slot: primitiveSsz.Slot, + fork: phase0Ssz.Fork, + // History + latestBlockHeader: phase0Ssz.BeaconBlockHeader, + blockRoots: phase0Ssz.HistoricalBlockRoots, + stateRoots: phase0Ssz.HistoricalStateRoots, + // historical_roots Frozen in Capella, replaced by historical_summaries + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), + // Eth1 + eth1Data: phase0Ssz.Eth1Data, + eth1DataVotes: phase0Ssz.Eth1DataVotes, + eth1DepositIndex: UintNum64, + // Registry + validators: phase0Ssz.Validators, + balances: phase0Ssz.Balances, + randaoMixes: phase0Ssz.RandaoMixes, + // Slashings + slashings: phase0Ssz.Slashings, + // Participation + previousEpochParticipation: altairSsz.EpochParticipation, + currentEpochParticipation: altairSsz.EpochParticipation, + // Finality + justificationBits: phase0Ssz.JustificationBits, + previousJustifiedCheckpoint: phase0Ssz.Checkpoint, + currentJustifiedCheckpoint: phase0Ssz.Checkpoint, + finalizedCheckpoint: phase0Ssz.Checkpoint, + // Inactivity + inactivityScores: altairSsz.InactivityScores, + // Sync + currentSyncCommittee: altairSsz.SyncCommittee, + nextSyncCommittee: altairSsz.SyncCommittee, + // Execution + latestExecutionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + // Withdrawals + nextWithdrawalIndex: capellaSsz.BeaconState.fields.nextWithdrawalIndex, + nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, + // Deep history valid from Capella onwards + historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, + depositReceiptsStartIndex: UintBn64, // New in ELECTRA }, {typeName: "BeaconState", jsonCase: "eth2"} ); export const LightClientHeader = new ContainerType( { - ...denebSsz.LightClientHeader.fields, + beacon: phase0Ssz.BeaconBlockHeader, + execution: ExecutionPayloadHeader, // Modified in ELECTRA + executionBranch: new VectorCompositeType(Bytes32, EXECUTION_PAYLOAD_DEPTH), }, {typeName: "LightClientHeader", jsonCase: "eth2"} ); export const LightClientBootstrap = new ContainerType( { - ...denebSsz.LightClientBootstrap.fields, + header: LightClientHeader, + currentSyncCommittee: altairSsz.SyncCommittee, + currentSyncCommitteeBranch: altairSsz.LightClientBootstrap.fields.currentSyncCommitteeBranch, }, {typeName: "LightClientBootstrap", jsonCase: "eth2"} ); export const LightClientUpdate = new ContainerType( { - ...denebSsz.LightClientUpdate.fields, + attestedHeader: LightClientHeader, + nextSyncCommittee: altairSsz.SyncCommittee, + nextSyncCommitteeBranch: altairSsz.LightClientUpdate.fields.nextSyncCommitteeBranch, + finalizedHeader: LightClientHeader, + finalityBranch: altairSsz.LightClientUpdate.fields.finalityBranch, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientUpdate", jsonCase: "eth2"} ); export const LightClientFinalityUpdate = new ContainerType( { - ...denebSsz.LightClientFinalityUpdate.fields, + attestedHeader: LightClientHeader, + finalizedHeader: LightClientHeader, + finalityBranch: altairSsz.LightClientFinalityUpdate.fields.finalityBranch, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientFinalityUpdate", jsonCase: "eth2"} ); export const LightClientOptimisticUpdate = new ContainerType( { - ...denebSsz.LightClientOptimisticUpdate.fields, + attestedHeader: LightClientHeader, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientOptimisticUpdate", jsonCase: "eth2"} ); export const LightClientStore = new ContainerType( { - ...denebSsz.LightClientStore.fields, + snapshot: LightClientBootstrap, + validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), }, {typeName: "LightClientStore", jsonCase: "eth2"} ); +// PayloadAttributes primarily for SSE event +export const PayloadAttributes = new ContainerType( + { + ...capellaSsz.PayloadAttributes.fields, + parentBeaconBlockRoot: Root, + }, + {typeName: "PayloadAttributes", jsonCase: "eth2"} +); + export const SSEPayloadAttributes = new ContainerType( { - ...denebSsz.SSEPayloadAttributes.fields, + ...bellatrixSsz.SSEPayloadAttributesCommon.fields, + payloadAttributes: PayloadAttributes, }, {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 198259eed1dd..3286c10a0334 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -1,12 +1,14 @@ import {ValueOf} from "@chainsafe/ssz"; import * as ssz from "./sszTypes.js"; -export type BlobSidecar = ValueOf; -export type ExecutionPayloadAndBlobsBundle = ValueOf; +export type DepositReceipt = ValueOf; +export type DepositReceipts = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; +export type ExecutionPayloadAndBlobsBundle = ValueOf; + export type BeaconBlockBody = ValueOf; export type BeaconBlock = ValueOf; export type SignedBeaconBlock = ValueOf; @@ -17,6 +19,8 @@ export type BlindedBeaconBlockBody = ValueOf; export type BlindedBeaconBlock = ValueOf; export type SignedBlindedBeaconBlock = ValueOf; +export type FullOrBlindedExecutionPayload = ExecutionPayload | ExecutionPayloadHeader; + export type BuilderBid = ValueOf; export type SignedBuilderBid = ValueOf; export type SSEPayloadAttributes = ValueOf; diff --git a/packages/types/src/primitive/sszTypes.ts b/packages/types/src/primitive/sszTypes.ts index 068a32e2cc17..376e17c3f1b6 100644 --- a/packages/types/src/primitive/sszTypes.ts +++ b/packages/types/src/primitive/sszTypes.ts @@ -50,6 +50,7 @@ export const SubcommitteeIndex = UintNum64; */ export const ValidatorIndex = UintNum64; export const WithdrawalIndex = UintNum64; +export const DepositIndex = UintNum64; export const Gwei = UintBn64; export const Wei = UintBn256; export const Root = new ByteVectorType(32); diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 0afede39b951..298707c07303 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -106,7 +106,6 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Fri, 12 Apr 2024 23:38:07 +0800 Subject: [PATCH 023/145] chore: fix CI failure due to recent merge from `unstable` (#6646) --- .../beacon-node/src/execution/engine/http.ts | 16 ++++++++-------- .../updateUnfinalizedPubkeys.test.ts | 6 +++--- .../test/sim/electra-interop.test.ts | 18 +++++++++--------- .../test/spec/utils/specTestIterator.ts | 1 + .../state-transition/src/cache/epochCache.ts | 2 +- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index d29387794327..ac3402852c35 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -201,10 +201,10 @@ export class ExecutionEngineHttp implements IExecutionEngine { ForkSeq[fork] >= ForkSeq.electra ? "engine_newPayloadV6110" : ForkSeq[fork] >= ForkSeq.deneb - ? "engine_newPayloadV3" - : ForkSeq[fork] >= ForkSeq.capella - ? "engine_newPayloadV2" - : "engine_newPayloadV1"; + ? "engine_newPayloadV3" + : ForkSeq[fork] >= ForkSeq.capella + ? "engine_newPayloadV2" + : "engine_newPayloadV1"; const serializedExecutionPayload = serializeExecutionPayload(fork, executionPayload); @@ -397,10 +397,10 @@ export class ExecutionEngineHttp implements IExecutionEngine { ForkSeq[fork] >= ForkSeq.electra ? "engine_getPayloadV6110" : ForkSeq[fork] >= ForkSeq.deneb - ? "engine_getPayloadV3" - : ForkSeq[fork] >= ForkSeq.capella - ? "engine_getPayloadV2" - : "engine_getPayloadV1"; + ? "engine_getPayloadV3" + : ForkSeq[fork] >= ForkSeq.capella + ? "engine_getPayloadV2" + : "engine_getPayloadV1"; const payloadResponse = await this.rpc.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method] diff --git a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts index 900f6a6fb873..39bf1a1551c9 100644 --- a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts +++ b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts @@ -7,7 +7,7 @@ import bls from "@chainsafe/bls"; import {ssz} from "@lodestar/types"; import {type CachedBeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; import {bytesToBigInt, intToBytes} from "@lodestar/utils"; -import {CheckpointStateCache, StateContextCache} from "../../../../src/chain/stateCache/index.js"; +import {InMemoryCheckpointStateCache, StateContextCache} from "../../../../src/chain/stateCache/index.js"; import {generateCachedElectraState} from "../../../utils/state.js"; // Benchmark date from Mon Nov 21 2023 - Intel Core i7-9750H @ 2.60Ghz @@ -21,7 +21,7 @@ describe("updateUnfinalizedPubkeys perf tests", function () { const numCheckpointStateCache = 8; const numStateCache = 3 * 32; - let checkpointStateCache: CheckpointStateCache; + let checkpointStateCache: InMemoryCheckpointStateCache; let stateCache: StateContextCache; const unfinalizedPubkey2Index = generatePubkey2Index(0, Math.max.apply(null, numPubkeysToBeFinalizedCases)); @@ -35,7 +35,7 @@ describe("updateUnfinalizedPubkeys perf tests", function () { baseState.epochCtx.pubkey2index = new PubkeyIndexMap(); baseState.epochCtx.index2pubkey = []; - checkpointStateCache = new CheckpointStateCache({}); + checkpointStateCache = new InMemoryCheckpointStateCache({}); stateCache = new StateContextCache({}); for (let i = 0; i < numCheckpointStateCache; i++) { diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index c1663211cbd9..428bba3c63ad 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; +import assert from "node:assert"; import {describe, it, vi, afterAll, afterEach} from "vitest"; -/* eslint-disable @typescript-eslint/naming-convention */ -import _ from "lodash"; + import {LogLevel, sleep} from "@lodestar/utils"; import {ForkName, SLOTS_PER_EPOCH, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; import {electra, Epoch, Slot} from "@lodestar/types"; @@ -225,13 +225,13 @@ describe("executionEngine / ExecutionEngineHttp", function () { } const actualDepositReceipt = payload.depositReceipts[0]; - if (!_.isEqual(actualDepositReceipt, depositReceiptB)) { - throw Error( - `Deposit receipts mismatched. Expected: ${JSON.stringify(depositReceiptB)}, actual: ${JSON.stringify( - actualDepositReceipt - )}` - ); - } + assert.deepStrictEqual( + actualDepositReceipt, + depositReceiptB, + `Deposit receipts mismatched. Expected: ${JSON.stringify(depositReceiptB)}, actual: ${JSON.stringify( + actualDepositReceipt + )}` + ); }); it("Post-merge, run for a few blocks", async function () { diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 3201fb15dbf3..6aa683cb6530 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -57,6 +57,7 @@ const coveredTestRunners = [ // ], // ``` export const defaultSkipOpts: SkipOpts = { + skippedForks: ["eip6110"], // TODO: capella // BeaconBlockBody proof in lightclient is the new addition in v1.3.0-rc.2-hotfix // Skip them for now to enable subsequently diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 41f8031d903e..a7433e848a44 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -225,7 +225,7 @@ export class EpochCache { * eg. latest epoch = 105, latest finalized cp state epoch = 102 * then the list will be (in terms of epoch) [103, 104, 105] */ - private historicalValidatorLengths: immutable.List; + historicalValidatorLengths: immutable.List; constructor(data: { config: BeaconConfig; From 4b707a9c8bdb8fc8827275a624bf4f9a62d20081 Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 22 Apr 2024 19:18:35 +0530 Subject: [PATCH 024/145] feat: implement execution layer exits eip 7002 (#6651) * feat: implement execution layer exits eip 7002 * lint and tsc fix * apply feedback * improve comment --- .../src/execution/engine/payloadIdCache.ts | 5 ++ .../beacon-node/src/execution/engine/types.ts | 48 +++++++++++----- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../test/perf/chain/opPools/opPool.test.ts | 2 +- .../produceBlock/produceBlockBody.test.ts | 2 +- .../test/sim/electra-interop.test.ts | 1 + .../opPools/aggregatedAttestationPool.test.ts | 4 +- .../test/unit/chain/shufflingCache.test.ts | 6 +- .../test/unit/executionEngine/http.test.ts | 4 ++ .../test/utils/validationData/attestation.ts | 4 +- packages/light-client/src/spec/utils.ts | 7 ++- packages/params/src/index.ts | 1 + packages/params/src/presets/mainnet.ts | 1 + packages/params/src/presets/minimal.ts | 1 + packages/params/src/types.ts | 2 + .../src/block/processExecutionLayerExit.ts | 56 +++++++++++++++++++ .../src/block/processOperations.ts | 8 +++ .../src/slot/upgradeStateToElectra.ts | 2 +- .../state-transition/src/util/execution.ts | 3 + packages/types/src/electra/sszTypes.ts | 15 ++++- packages/types/src/electra/types.ts | 3 + packages/validator/src/util/params.ts | 1 + 22 files changed, 152 insertions(+), 26 deletions(-) create mode 100644 packages/state-transition/src/block/processExecutionLayerExit.ts diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index e5baa9fba92d..960b061f12da 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -26,6 +26,11 @@ export type DepositReceiptV1 = { index: QUANTITY; }; +export type ExecutionLayerExitV1 = { + sourceAddress: DATA; + validatorPubkey: DATA; +}; + type FcuAttributes = {headBlockHash: DATA; finalizedBlockHash: DATA} & Omit; export class PayloadIdCache { diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 999c77f86b9c..f12f98e1a0f7 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1, DepositReceiptV1} from "./payloadIdCache.js"; +import {WithdrawalV1, DepositReceiptV1, ExecutionLayerExitV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -126,12 +126,14 @@ export type ExecutionPayloadBodyRpc = { transactions: DATA[]; withdrawals: WithdrawalV1[] | null | undefined; depositReceipts: DepositReceiptV1[] | null | undefined; + exits: ExecutionLayerExitV1[] | null | undefined; }; export type ExecutionPayloadBody = { transactions: bellatrix.Transaction[]; withdrawals: capella.Withdrawals | null; depositReceipts: electra.DepositReceipts | null; + exits: electra.ExecutionLayerExits | null; }; export type ExecutionPayloadRpc = { @@ -154,6 +156,7 @@ export type ExecutionPayloadRpc = { excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB depositReceipts?: DepositReceiptRpc[]; // ELECTRA + exits?: ExecutionLayerExitRpc[]; // ELECTRA }; export type WithdrawalRpc = { @@ -163,13 +166,8 @@ export type WithdrawalRpc = { amount: QUANTITY; }; -export type DepositReceiptRpc = { - pubkey: DATA; - withdrawalCredentials: DATA; - amount: QUANTITY; - signature: DATA; - index: QUANTITY; -}; +export type DepositReceiptRpc = DepositReceiptV1; +export type ExecutionLayerExitRpc = ExecutionLayerExitV1; export type VersionedHashesRpc = DATA[]; @@ -235,8 +233,9 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload // ELECTRA adds depositReceipts to the ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositReceipts} = data as electra.ExecutionPayload; + const {depositReceipts, exits} = data as electra.ExecutionPayload; payload.depositReceipts = depositReceipts.map(serializeDepositReceipt); + payload.exits = exits.map(serializeExecutionLayerExit); } return payload; @@ -325,14 +324,21 @@ export function parseExecutionPayload( } if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositReceipts} = data; + const {depositReceipts, exits} = data; // Geth can also reply with null if (depositReceipts == null) { throw Error( `depositReceipts missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` ); } - (executionPayload as electra.ExecutionPayload).depositReceipts = depositReceipts.map(deserializeDepositReceipts); + (executionPayload as electra.ExecutionPayload).depositReceipts = depositReceipts.map(deserializeDepositReceipt); + + if (exits == null) { + throw Error( + `exits missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` + ); + } + (executionPayload as electra.ExecutionPayload).exits = exits.map(deserializeExecutionLayerExit); } return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; @@ -411,7 +417,7 @@ export function serializeDepositReceipt(depositReceipt: electra.DepositReceipt): }; } -export function deserializeDepositReceipts(serialized: DepositReceiptRpc): electra.DepositReceipt { +export function deserializeDepositReceipt(serialized: DepositReceiptRpc): electra.DepositReceipt { return { pubkey: dataToBytes(serialized.pubkey, 48), withdrawalCredentials: dataToBytes(serialized.withdrawalCredentials, 32), @@ -421,12 +427,27 @@ export function deserializeDepositReceipts(serialized: DepositReceiptRpc): elect } as electra.DepositReceipt; } +export function serializeExecutionLayerExit(exit: electra.ExecutionLayerExit): ExecutionLayerExitRpc { + return { + sourceAddress: bytesToData(exit.sourceAddress), + validatorPubkey: bytesToData(exit.validatorPubkey), + }; +} + +export function deserializeExecutionLayerExit(exit: ExecutionLayerExitRpc): electra.ExecutionLayerExit { + return { + sourceAddress: dataToBytes(exit.sourceAddress, 20), + validatorPubkey: dataToBytes(exit.validatorPubkey, 48), + }; +} + export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | null): ExecutionPayloadBody | null { return data ? { transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, - depositReceipts: data.depositReceipts ? data.depositReceipts.map(deserializeDepositReceipts) : null, + depositReceipts: data.depositReceipts ? data.depositReceipts.map(deserializeDepositReceipt) : null, + exits: data.exits ? data.exits.map(deserializeExecutionLayerExit) : null, } : null; } @@ -437,6 +458,7 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, depositReceipts: data.depositReceipts ? data.depositReceipts.map(serializeDepositReceipt) : null, + exits: data.exits ? data.exits.map(serializeExecutionLayerExit) : null, } : null; } diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 7a882b03af0e..ee14eb27b51d 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -32,7 +32,7 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { before(function () { this.timeout(5 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}) as unknown as CachedBeaconStateAltair; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}) as CachedBeaconStateAltair; const {blockHeader, checkpoint} = computeAnchorCheckpoint(originalState.config, originalState); // TODO figure out why getBlockRootAtSlot(originalState, justifiedSlot) is not the same to justifiedCheckpoint.root diff --git a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts index 7998a204f09d..2632c593e78c 100644 --- a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts @@ -24,7 +24,7 @@ describe("opPool", () => { before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}) as unknown as CachedBeaconStateAltair; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}) as CachedBeaconStateAltair; }); itBench({ diff --git a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts index 2386f3538205..f90f148f3e01 100644 --- a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts +++ b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts @@ -25,7 +25,7 @@ describe("produceBlockBody", () => { before(async () => { db = new BeaconDb(config, await LevelDbController.create({name: ".tmpdb"}, {logger})); - state = stateOg.clone() as unknown as CachedBeaconStateAltair; + state = stateOg.clone() as CachedBeaconStateAltair; chain = new BeaconChain( { proposerBoost: true, diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 428bba3c63ad..ab3f1ae6b2c1 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -173,6 +173,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { blockHash: dataToBytes(newPayloadBlockHash, 32), receiptsRoot: dataToBytes("0x79ee3424eb720a3ad4b1c5a372bb8160580cbe4d893778660f34213c685627a9", 32), blobGasUsed: 0n, + exits: [], }; const parentBeaconBlockRoot = dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32); const payloadResult = await executionEngine.notifyNewPayload( diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 41c205cdd0e9..800984fa84bc 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -61,9 +61,9 @@ describe("AggregatedAttestationPool", function () { epochParticipation[committee[i]] = 0b000; } } - (originalState as unknown as CachedBeaconStateAltair).previousEpochParticipation = + (originalState as CachedBeaconStateAltair).previousEpochParticipation = ssz.altair.EpochParticipation.toViewDU(epochParticipation); - (originalState as unknown as CachedBeaconStateAltair).currentEpochParticipation = + (originalState as CachedBeaconStateAltair).currentEpochParticipation = ssz.altair.EpochParticipation.toViewDU(epochParticipation); originalState.commit(); let altairState: CachedBeaconStateAllForks; diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index e6a7e8706bbe..035746438563 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -14,7 +14,7 @@ describe("ShufflingCache", function () { beforeEach(() => { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch); + shufflingCache.processState(state as CachedBeaconStateAllForks, currentEpoch); }); it("should get shuffling from cache", async function () { @@ -29,7 +29,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shufflings at other epochs does prune the cache - shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch + 1); + shufflingCache.processState(state as CachedBeaconStateAllForks, currentEpoch + 1); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); }); @@ -39,7 +39,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch + 1); + shufflingCache.processState(state as CachedBeaconStateAllForks, currentEpoch + 1); expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); }); diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index b0e20ad1f2b5..d58eefa10368 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -194,6 +194,7 @@ describe("ExecutionEngine / http", () => { }, ], depositReceipts: null, // depositReceipts is null pre-electra + exits: null, }, null, // null returned for missing blocks { @@ -203,6 +204,7 @@ describe("ExecutionEngine / http", () => { ], withdrawals: null, // withdrawals is null pre-capella depositReceipts: null, // depositReceipts is null pre-electra + exits: null, }, ], }; @@ -251,6 +253,7 @@ describe("ExecutionEngine / http", () => { }, ], depositReceipts: null, // depositReceipts is null pre-electra + exits: null, }, null, // null returned for missing blocks { @@ -260,6 +263,7 @@ describe("ExecutionEngine / http", () => { ], withdrawals: null, // withdrawals is null pre-capella depositReceipts: null, // depositReceipts is null pre-electra + exits: null, }, ], }; diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 335ec81870c6..82bdca901889 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -83,8 +83,8 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { }; const shufflingCache = new ShufflingCache(); - shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, state.epochCtx.currentShuffling.epoch); - shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, state.epochCtx.nextShuffling.epoch); + shufflingCache.processState(state as CachedBeaconStateAllForks, state.epochCtx.currentShuffling.epoch); + shufflingCache.processState(state as CachedBeaconStateAllForks, state.epochCtx.nextShuffling.epoch); const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); const forkChoice = { diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 152c55ac6d15..8e6b3456a600 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -117,6 +117,8 @@ export function upgradeLightClientHeader( case ForkName.electra: (upgradedHeader as LightClientHeader).execution.depositReceiptsRoot = ssz.electra.LightClientHeader.fields.execution.fields.depositReceiptsRoot.defaultValue(); + (upgradedHeader as electra.LightClientHeader).execution.exitsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.exitsRoot.defaultValue(); // Break if no further upgrades is required else fall through if (ForkSeq[targetFork] <= ForkSeq.electra) break; @@ -154,7 +156,10 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC } if (epoch < config.ELECTRA_FORK_EPOCH) { - if ((header as LightClientHeader).execution.depositReceiptsRoot !== undefined) { + if ( + (header as LightClientHeader).execution.depositReceiptsRoot !== undefined || + (header as LightClientHeader).execution.exitsRoot !== undefined + ) { return false; } } diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 482e591123b6..d903a404dfb5 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -95,6 +95,7 @@ export const { KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, + MAX_EXECUTION_LAYER_EXITS, } = activePreset; //////////// diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 86f5c39c539e..802a6691c311 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -121,4 +121,5 @@ export const mainnetPreset: BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192, + MAX_EXECUTION_LAYER_EXITS: 16, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 6239a8c1a3eb..d10b420ed97c 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -122,4 +122,5 @@ export const minimalPreset: BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4, + MAX_EXECUTION_LAYER_EXITS: 16, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index 57ee8230a77d..7856f1be72ba 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -85,6 +85,7 @@ export type BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: number; + MAX_EXECUTION_LAYER_EXITS: number; }; /** @@ -173,6 +174,7 @@ export const beaconPresetTypes: BeaconPresetTypes = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: "number", + MAX_EXECUTION_LAYER_EXITS: "number", }; type BeaconPresetTypes = { diff --git a/packages/state-transition/src/block/processExecutionLayerExit.ts b/packages/state-transition/src/block/processExecutionLayerExit.ts new file mode 100644 index 000000000000..5068d9af8667 --- /dev/null +++ b/packages/state-transition/src/block/processExecutionLayerExit.ts @@ -0,0 +1,56 @@ +import {CompositeViewDU} from "@chainsafe/ssz"; +import {electra, ssz} from "@lodestar/types"; +import {ETH1_ADDRESS_WITHDRAWAL_PREFIX, FAR_FUTURE_EPOCH} from "@lodestar/params"; + +import {isActiveValidator} from "../util/index.js"; +import {CachedBeaconStateElectra} from "../types.js"; +import {initiateValidatorExit} from "./index.js"; + +/** + * Process execution layer exit messages and initiate exit incase they belong to a valid active validator + * otherwise silent ignore. + */ +export function processExecutionLayerExit(state: CachedBeaconStateElectra, exit: electra.ExecutionLayerExit): void { + const validator = isValidExecutionLayerExit(state, exit); + if (validator === null) { + return; + } + + initiateValidatorExit(state, validator); +} + +export function isValidExecutionLayerExit( + state: CachedBeaconStateElectra, + exit: electra.ExecutionLayerExit +): CompositeViewDU | null { + const {config, epochCtx} = state; + const validatorIndex = epochCtx.getValidatorIndex(exit.validatorPubkey); + const validator = validatorIndex !== undefined ? state.validators.getReadonly(validatorIndex) : undefined; + if (validator === undefined) { + return null; + } + + const {withdrawalCredentials} = validator; + if (withdrawalCredentials[0] !== ETH1_ADDRESS_WITHDRAWAL_PREFIX) { + return null; + } + + const executionAddress = withdrawalCredentials.subarray(12, 32); + if (Buffer.compare(executionAddress, exit.sourceAddress) !== 0) { + return null; + } + + const currentEpoch = epochCtx.epoch; + if ( + // verify the validator is active + isActiveValidator(validator, currentEpoch) && + // verify exit has not been initiated + validator.exitEpoch === FAR_FUTURE_EPOCH && + // verify the validator had been active long enough + currentEpoch >= validator.activationEpoch + config.SHARD_COMMITTEE_PERIOD + ) { + return validator; + } else { + return null; + } +} diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 7b8b39efe743..c4879da4aa71 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -8,6 +8,7 @@ import {processProposerSlashing} from "./processProposerSlashing.js"; import {processAttesterSlashing} from "./processAttesterSlashing.js"; import {processDeposit} from "./processDeposit.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; +import {processExecutionLayerExit} from "./processExecutionLayerExit.js"; import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; import {processDepositReceipt} from "./processDepositReceipt.js"; import {ProcessBlockOpts} from "./types.js"; @@ -18,6 +19,7 @@ export { processAttestations, processDeposit, processVoluntaryExit, + processExecutionLayerExit, processBlsToExecutionChange, processDepositReceipt, }; @@ -48,9 +50,15 @@ export function processOperations( for (const deposit of body.deposits) { processDeposit(fork, state, deposit); } + for (const voluntaryExit of body.voluntaryExits) { processVoluntaryExit(state, voluntaryExit, opts.verifySignatures); } + if (fork >= ForkSeq.electra) { + for (const elExit of (body as electra.BeaconBlockBody).executionPayload.exits) { + processExecutionLayerExit(state as CachedBeaconStateElectra, elExit); + } + } if (fork >= ForkSeq.capella) { for (const blsToExecutionChange of (body as capella.BeaconBlockBody).blsToExecutionChanges) { diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 19cac8811f77..369ab19c447b 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -20,7 +20,7 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache epoch: stateDeneb.epochCtx.epoch, }); - // latestExecutionPayloadHeader's depositReceiptsRoot set to zeros by default + // latestExecutionPayloadHeader's depositReceiptsRoot and exitsRoot set to zeros by default // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index d9a88e5475b5..2975b84bf747 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -174,6 +174,9 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio if (fork >= ForkSeq.electra) { (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositReceiptsRoot = ssz.electra.DepositReceipts.hashTreeRoot((payload as electra.ExecutionPayload).depositReceipts); + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).exitsRoot = ssz.electra.ExecutionLayerExits.hashTreeRoot( + (payload as electra.ExecutionPayload).exits + ); } return bellatrixPayloadFields; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index b347b20df9c7..7b5ed51fb786 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -5,6 +5,7 @@ import { EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, + MAX_EXECUTION_LAYER_EXITS, } from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; @@ -13,7 +14,8 @@ import {ssz as bellatrixSsz} from "../bellatrix/index.js"; import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; -const {UintNum64, Slot, Root, BLSSignature, UintBn256, Bytes32, BLSPubkey, DepositIndex, UintBn64} = primitiveSsz; +const {UintNum64, Slot, Root, BLSSignature, UintBn256, Bytes32, BLSPubkey, DepositIndex, UintBn64, ExecutionAddress} = + primitiveSsz; export const DepositReceipt = new ContainerType( { @@ -28,10 +30,20 @@ export const DepositReceipt = new ContainerType( export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); +export const ExecutionLayerExit = new ContainerType( + { + sourceAddress: ExecutionAddress, + validatorPubkey: BLSPubkey, + }, + {typeName: "ExecutionLayerExit", jsonCase: "eth2"} +); +export const ExecutionLayerExits = new ListCompositeType(ExecutionLayerExit, MAX_EXECUTION_LAYER_EXITS); + export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, depositReceipts: DepositReceipts, // New in ELECTRA + exits: ExecutionLayerExits, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -40,6 +52,7 @@ export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, depositReceiptsRoot: Root, // New in ELECTRA + exitsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 3286c10a0334..1b9b42217b8c 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -4,6 +4,9 @@ import * as ssz from "./sszTypes.js"; export type DepositReceipt = ValueOf; export type DepositReceipts = ValueOf; +export type ExecutionLayerExit = ValueOf; +export type ExecutionLayerExits = ValueOf; + export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 298707c07303..ca1b36883a90 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -224,5 +224,6 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Wed, 1 May 2024 21:46:27 +0800 Subject: [PATCH 025/145] chore: update spec test version for electra fork (#6717) * Update spec-test version * Skip electra --- packages/beacon-node/test/spec/specTestVersioning.ts | 2 +- packages/beacon-node/test/spec/utils/specTestIterator.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index 37167c9bd5a1..06b02ab5304e 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.4.0-beta.6", + specVersion: "v1.5.0-alpha.1", // Target directory is the host package root: 'packages/*/spec-tests' outputDir: path.join(__dirname, "../../spec-tests"), specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests", diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 6aa683cb6530..88d7cbdea9e6 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -57,7 +57,7 @@ const coveredTestRunners = [ // ], // ``` export const defaultSkipOpts: SkipOpts = { - skippedForks: ["eip6110"], + skippedForks: ["electra", "eip7594"], // TODO: capella // BeaconBlockBody proof in lightclient is the new addition in v1.3.0-rc.2-hotfix // Skip them for now to enable subsequently @@ -65,7 +65,7 @@ export const defaultSkipOpts: SkipOpts = { "capella/light_client/single_merkle_proof/BeaconBlockBody", "deneb/light_client/single_merkle_proof/BeaconBlockBody", ], - skippedRunners: ["merkle_proof"], + skippedRunners: ["merkle_proof", "networking"], }; /** From bef3eff3acdbb812e8fb7bbb91295c9fff1c9faa Mon Sep 17 00:00:00 2001 From: NC Date: Sat, 4 May 2024 21:24:26 +0800 Subject: [PATCH 026/145] feat: add presets and ssz types for EIP-7549 (#6715) * Add types * Update unit test * lint * Address comments * Address comments * Lint * Update packages/beacon-node/src/util/sszBytes.ts Co-authored-by: tuyennhv * Add isElectraAttestation * Update unit test * Update unit test * chore: add comments for sszBytes.ts --------- Co-authored-by: tuyennhv Co-authored-by: Tuyen Nguyen Co-authored-by: Gajinder --- .../src/chain/validation/attestation.ts | 4 +- packages/beacon-node/src/util/sszBytes.ts | 63 +++++++--- .../test/unit/util/sszBytes.test.ts | 46 +++++-- packages/params/src/index.ts | 2 + packages/params/src/presets/mainnet.ts | 2 + packages/params/src/presets/minimal.ts | 2 + packages/params/src/types.ts | 4 + packages/types/src/electra/sszTypes.ts | 113 +++++++++++++++++- packages/types/src/electra/types.ts | 8 ++ packages/types/src/sszTypes.ts | 2 +- packages/types/src/types.ts | 8 ++ packages/types/src/utils/typeguards.ts | 9 +- packages/validator/src/util/params.ts | 2 + 13 files changed, 233 insertions(+), 32 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 1116e87e1d25..a66b95b782e4 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -305,7 +305,7 @@ async function validateGossipAttestationNoSignatureCheck( // > TODO: Do this check **before** getting the target state but don't recompute zipIndexes const aggregationBits = attestationOrCache.attestation ? attestationOrCache.attestation.aggregationBits - : getAggregationBitsFromAttestationSerialized(attestationOrCache.serializedData); + : getAggregationBitsFromAttestationSerialized(fork, attestationOrCache.serializedData); if (aggregationBits === null) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SERIALIZED_BYTES, @@ -414,7 +414,7 @@ async function validateGossipAttestationNoSignatureCheck( let attDataRootHex: RootHex; const signature = attestationOrCache.attestation ? attestationOrCache.attestation.signature - : getSignatureFromAttestationSerialized(attestationOrCache.serializedData); + : getSignatureFromAttestationSerialized(fork, attestationOrCache.serializedData); if (signature === null) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SERIALIZED_BYTES, diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index ce4125140075..3ba2691be357 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -1,15 +1,30 @@ import {BitArray, deserializeUint8ArrayBitListFromBytes} from "@chainsafe/ssz"; import {BLSSignature, RootHex, Slot} from "@lodestar/types"; -import {BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB} from "@lodestar/params"; +import { + BYTES_PER_FIELD_ELEMENT, + FIELD_ELEMENTS_PER_BLOB, + ForkName, + ForkSeq, + MAX_COMMITTEES_PER_SLOT, +} from "@lodestar/params"; export type BlockRootHex = RootHex; export type AttDataBase64 = string; +// pre-electra // class Attestation(Container): // aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - offset 4 // data: AttestationData - target data - 128 // signature: BLSSignature - 96 + +// electra +// class Attestation(Container): +// aggregation_bits: BitList[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT] - offset 4 +// data: AttestationData - target data - 128 +// committee_bits: BitVector[MAX_COMMITTEES_PER_SLOT] +// signature: BLSSignature - 96 // +// for all forks // class AttestationData(Container): 128 bytes fixed size // slot: Slot - data 8 // index: CommitteeIndex - data 8 @@ -22,6 +37,7 @@ const ATTESTATION_BEACON_BLOCK_ROOT_OFFSET = VARIABLE_FIELD_OFFSET + 8 + 8; const ROOT_SIZE = 32; const SLOT_SIZE = 8; const ATTESTATION_DATA_SIZE = 128; +const COMMITTEE_BITS_SIZE = Math.max(Math.ceil(MAX_COMMITTEES_PER_SLOT / 8), 1); const SIGNATURE_SIZE = 96; // shared Buffers to convert bytes to hex/base64 @@ -73,16 +89,17 @@ export function getAttDataBase64FromAttestationSerialized(data: Uint8Array): Att * Extract aggregation bits from attestation serialized bytes. * Return null if data is not long enough to extract aggregation bits. */ -export function getAggregationBitsFromAttestationSerialized(data: Uint8Array): BitArray | null { - if (data.length < VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE) { +export function getAggregationBitsFromAttestationSerialized(fork: ForkName, data: Uint8Array): BitArray | null { + const aggregationBitsStartIndex = + ForkSeq[fork] >= ForkSeq.electra + ? VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE + SIGNATURE_SIZE + : VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE; + + if (data.length < aggregationBitsStartIndex) { return null; } - const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes( - data, - VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE, - data.length - ); + const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes(data, aggregationBitsStartIndex, data.length); return new BitArray(uint8Array, bitLen); } @@ -90,15 +107,33 @@ export function getAggregationBitsFromAttestationSerialized(data: Uint8Array): B * Extract signature from attestation serialized bytes. * Return null if data is not long enough to extract signature. */ -export function getSignatureFromAttestationSerialized(data: Uint8Array): BLSSignature | null { - if (data.length < VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE) { +export function getSignatureFromAttestationSerialized(fork: ForkName, data: Uint8Array): BLSSignature | null { + const signatureStartIndex = + ForkSeq[fork] >= ForkSeq.electra + ? VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE + : VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE; + + if (data.length < signatureStartIndex + SIGNATURE_SIZE) { return null; } - return data.subarray( - VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE, - VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE - ); + return data.subarray(signatureStartIndex, signatureStartIndex + SIGNATURE_SIZE); +} + +/** + * Extract committee bits from Electra attestation serialized bytes. + * Return null if data is not long enough to extract committee bits. + */ +export function getCommitteeBitsFromAttestationSerialized(data: Uint8Array): BitArray | null { + const committeeBitsStartIndex = VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE; + + if (data.length < committeeBitsStartIndex + COMMITTEE_BITS_SIZE) { + return null; + } + + const uint8Array = data.subarray(committeeBitsStartIndex, committeeBitsStartIndex + COMMITTEE_BITS_SIZE); + + return new BitArray(uint8Array, MAX_COMMITTEES_PER_SLOT); } // diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 4285f4ca88b5..dd1b5aa747bf 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -1,6 +1,8 @@ import {describe, it, expect} from "vitest"; -import {deneb, Epoch, phase0, RootHex, Slot, ssz} from "@lodestar/types"; +import {BitArray} from "@chainsafe/ssz"; +import {allForks, deneb, Epoch, isElectraAttestation, phase0, RootHex, Slot, ssz} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; +import {ForkName, MAX_COMMITTEES_PER_SLOT} from "@lodestar/params"; import { getAttDataBase64FromAttestationSerialized, getAttDataBase64FromSignedAggregateAndProofSerialized, @@ -12,10 +14,11 @@ import { getSignatureFromAttestationSerialized, getSlotFromSignedBeaconBlockSerialized, getSlotFromBlobSidecarSerialized, + getCommitteeBitsFromAttestationSerialized, } from "../../../src/util/sszBytes.js"; describe("attestation SSZ serialized picking", () => { - const testCases: phase0.Attestation[] = [ + const testCases: allForks.Attestation[] = [ ssz.phase0.Attestation.defaultValue(), attestationFromValues( 4_000_000, @@ -23,18 +26,40 @@ describe("attestation SSZ serialized picking", () => { 200_00, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffff" ), + ssz.electra.Attestation.defaultValue(), + { + ...attestationFromValues( + 4_000_000, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 200_00, + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffff" + ), + committeeBits: BitArray.fromSingleBit(MAX_COMMITTEES_PER_SLOT, 3), + }, ]; for (const [i, attestation] of testCases.entries()) { it(`attestation ${i}`, () => { - const bytes = ssz.phase0.Attestation.serialize(attestation); + const isElectra = isElectraAttestation(attestation); + const bytes = isElectra + ? ssz.electra.Attestation.serialize(attestation) + : ssz.phase0.Attestation.serialize(attestation); expect(getSlotFromAttestationSerialized(bytes)).toBe(attestation.data.slot); expect(getBlockRootFromAttestationSerialized(bytes)).toBe(toHex(attestation.data.beaconBlockRoot)); - expect(getAggregationBitsFromAttestationSerialized(bytes)?.toBoolArray()).toEqual( - attestation.aggregationBits.toBoolArray() - ); - expect(getSignatureFromAttestationSerialized(bytes)).toEqual(attestation.signature); + + if (isElectra) { + expect(getAggregationBitsFromAttestationSerialized(ForkName.electra, bytes)?.toBoolArray()).toEqual( + attestation.aggregationBits.toBoolArray() + ); + expect(getCommitteeBitsFromAttestationSerialized(bytes)).toEqual(attestation.committeeBits); + expect(getSignatureFromAttestationSerialized(ForkName.electra, bytes)).toEqual(attestation.signature); + } else { + expect(getAggregationBitsFromAttestationSerialized(ForkName.phase0, bytes)?.toBoolArray()).toEqual( + attestation.aggregationBits.toBoolArray() + ); + expect(getSignatureFromAttestationSerialized(ForkName.phase0, bytes)).toEqual(attestation.signature); + } const attDataBase64 = ssz.phase0.AttestationData.serialize(attestation.data); expect(getAttDataBase64FromAttestationSerialized(bytes)).toBe(Buffer.from(attDataBase64).toString("base64")); @@ -65,14 +90,16 @@ describe("attestation SSZ serialized picking", () => { it("getAggregateionBitsFromAttestationSerialized - invalid data", () => { const invalidAggregationBitsDataSizes = [0, 4, 100, 128, 227]; for (const size of invalidAggregationBitsDataSizes) { - expect(getAggregationBitsFromAttestationSerialized(Buffer.alloc(size))).toBeNull(); + expect(getAggregationBitsFromAttestationSerialized(ForkName.phase0, Buffer.alloc(size))).toBeNull(); + expect(getAggregationBitsFromAttestationSerialized(ForkName.electra, Buffer.alloc(size))).toBeNull(); } }); it("getSignatureFromAttestationSerialized - invalid data", () => { const invalidSignatureDataSizes = [0, 4, 100, 128, 227]; for (const size of invalidSignatureDataSizes) { - expect(getSignatureFromAttestationSerialized(Buffer.alloc(size))).toBeNull(); + expect(getSignatureFromAttestationSerialized(ForkName.phase0, Buffer.alloc(size))).toBeNull(); + expect(getSignatureFromAttestationSerialized(ForkName.electra, Buffer.alloc(size))).toBeNull(); } }); }); @@ -86,6 +113,7 @@ describe("aggregateAndProof SSZ serialized picking", () => { 200_00, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffff" ), + ssz.electra.SignedAggregateAndProof.defaultValue(), ]; for (const [i, signedAggregateAndProof] of testCases.entries()) { diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index d903a404dfb5..3e56effc4138 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -96,6 +96,8 @@ export const { MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, MAX_EXECUTION_LAYER_EXITS, + MAX_ATTESTER_SLASHINGS_ELECTRA, + MAX_ATTESTATIONS_ELECTRA, } = activePreset; //////////// diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 802a6691c311..27cb7640b2dd 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -122,4 +122,6 @@ export const mainnetPreset: BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192, MAX_EXECUTION_LAYER_EXITS: 16, + MAX_ATTESTER_SLASHINGS_ELECTRA: 1, + MAX_ATTESTATIONS_ELECTRA: 8, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index d10b420ed97c..022532a49e6f 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -123,4 +123,6 @@ export const minimalPreset: BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4, MAX_EXECUTION_LAYER_EXITS: 16, + MAX_ATTESTER_SLASHINGS_ELECTRA: 1, + MAX_ATTESTATIONS_ELECTRA: 8, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index 7856f1be72ba..34f40a66707e 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -86,6 +86,8 @@ export type BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: number; MAX_EXECUTION_LAYER_EXITS: number; + MAX_ATTESTER_SLASHINGS_ELECTRA: number; + MAX_ATTESTATIONS_ELECTRA: number; }; /** @@ -175,6 +177,8 @@ export const beaconPresetTypes: BeaconPresetTypes = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: "number", MAX_EXECUTION_LAYER_EXITS: "number", + MAX_ATTESTER_SLASHINGS_ELECTRA: "number", + MAX_ATTESTATIONS_ELECTRA: "number", }; type BeaconPresetTypes = { diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 7b5ed51fb786..3404baf04110 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -1,10 +1,21 @@ -import {ContainerType, ListCompositeType, VectorCompositeType} from "@chainsafe/ssz"; +import { + BitListType, + BitVectorType, + ContainerType, + ListBasicType, + ListCompositeType, + VectorCompositeType, +} from "@chainsafe/ssz"; import { HISTORICAL_ROOTS_LIMIT, BLOCK_BODY_EXECUTION_PAYLOAD_DEPTH as EXECUTION_PAYLOAD_DEPTH, EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, + MAX_VALIDATORS_PER_COMMITTEE, + MAX_COMMITTEES_PER_SLOT, + MAX_ATTESTATIONS_ELECTRA, + MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_EXECUTION_LAYER_EXITS, } from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; @@ -14,8 +25,84 @@ import {ssz as bellatrixSsz} from "../bellatrix/index.js"; import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; -const {UintNum64, Slot, Root, BLSSignature, UintBn256, Bytes32, BLSPubkey, DepositIndex, UintBn64, ExecutionAddress} = - primitiveSsz; +const { + UintNum64, + Slot, + Root, + BLSSignature, + UintBn256, + Bytes32, + BLSPubkey, + DepositIndex, + UintBn64, + ExecutionAddress, + ValidatorIndex, +} = primitiveSsz; + +export const AggregationBits = new BitListType(MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT); + +// This CommitteeBits serves a different purpose than CommitteeBits in phase0 +// TODO Electra: Rename phase0.CommitteeBits to ParticipationBits to avoid confusion +export const CommitteeBits = new BitVectorType(MAX_COMMITTEES_PER_SLOT); + +export const AttestingIndices = new ListBasicType( + ValidatorIndex, + MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT +); + +export const Attestation = new ContainerType( + { + aggregationBits: AggregationBits, // Modified in ELECTRA + data: phase0Ssz.AttestationData, + committeeBits: CommitteeBits, // New in ELECTRA + signature: BLSSignature, + }, + {typeName: "Attestation", jsonCase: "eth2"} +); + +export const IndexedAttestation = new ContainerType( + { + attestingIndices: AttestingIndices, // Modified in ELECTRA + data: phase0Ssz.AttestationData, + signature: BLSSignature, + }, + {typeName: "IndexedAttestation", jsonCase: "eth2"} +); + +/** Same as `IndexedAttestation` but epoch, slot and index are not bounded and must be a bigint */ +export const IndexedAttestationBigint = new ContainerType( + { + attestingIndices: AttestingIndices, // Modified in ELECTRA + data: phase0Ssz.AttestationDataBigint, + signature: BLSSignature, + }, + {typeName: "IndexedAttestation", jsonCase: "eth2"} +); + +export const AttesterSlashing = new ContainerType( + { + attestation1: IndexedAttestationBigint, // Modified in ELECTRA + attestation2: IndexedAttestationBigint, // Modified in ELECTRA + }, + {typeName: "AttesterSlashing", jsonCase: "eth2"} +); + +export const AggregateAndProof = new ContainerType( + { + aggregatorIndex: ValidatorIndex, + aggregate: Attestation, // Modified in ELECTRA + selectionProof: BLSSignature, + }, + {typeName: "AggregateAndProof", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const SignedAggregateAndProof = new ContainerType( + { + message: AggregateAndProof, // Modified in ELECTRA + signature: BLSSignature, + }, + {typeName: "SignedAggregateAndProof", jsonCase: "eth2"} +); export const DepositReceipt = new ContainerType( { @@ -60,7 +147,15 @@ export const ExecutionPayloadHeader = new ContainerType( // We have to preserve Fields ordering while changing the type of ExecutionPayload export const BeaconBlockBody = new ContainerType( { - ...altairSsz.BeaconBlockBody.fields, + randaoReveal: phase0Ssz.BeaconBlockBody.fields.randaoReveal, + eth1Data: phase0Ssz.BeaconBlockBody.fields.eth1Data, + graffiti: phase0Ssz.BeaconBlockBody.fields.graffiti, + proposerSlashings: phase0Ssz.BeaconBlockBody.fields.proposerSlashings, + attesterSlashings: new ListCompositeType(AttesterSlashing, MAX_ATTESTER_SLASHINGS_ELECTRA), // Modified in ELECTRA + attestations: new ListCompositeType(Attestation, MAX_ATTESTATIONS_ELECTRA), // Modified in ELECTRA + deposits: phase0Ssz.BeaconBlockBody.fields.deposits, + voluntaryExits: phase0Ssz.BeaconBlockBody.fields.voluntaryExits, + syncAggregate: altairSsz.BeaconBlockBody.fields.syncAggregate, executionPayload: ExecutionPayload, // Modified in ELECTRA blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, @@ -86,7 +181,15 @@ export const SignedBeaconBlock = new ContainerType( export const BlindedBeaconBlockBody = new ContainerType( { - ...altairSsz.BeaconBlockBody.fields, + randaoReveal: phase0Ssz.BeaconBlockBody.fields.randaoReveal, + eth1Data: phase0Ssz.BeaconBlockBody.fields.eth1Data, + graffiti: phase0Ssz.BeaconBlockBody.fields.graffiti, + proposerSlashings: phase0Ssz.BeaconBlockBody.fields.proposerSlashings, + attesterSlashings: new ListCompositeType(AttesterSlashing, MAX_ATTESTER_SLASHINGS_ELECTRA), // Modified in ELECTRA + attestations: new ListCompositeType(Attestation, MAX_ATTESTATIONS_ELECTRA), // Modified in ELECTRA + deposits: phase0Ssz.BeaconBlockBody.fields.deposits, + voluntaryExits: phase0Ssz.BeaconBlockBody.fields.voluntaryExits, + syncAggregate: altairSsz.SyncAggregate, executionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 1b9b42217b8c..2925885b3af3 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -1,6 +1,14 @@ import {ValueOf} from "@chainsafe/ssz"; import * as ssz from "./sszTypes.js"; +export type Attestation = ValueOf; +export type IndexedAttestation = ValueOf; +export type IndexedAttestationBigint = ValueOf; +export type AttesterSlashing = ValueOf; + +export type AggregateAndProof = ValueOf; +export type SignedAggregateAndProof = ValueOf; + export type DepositReceipt = ValueOf; export type DepositReceipts = ValueOf; diff --git a/packages/types/src/sszTypes.ts b/packages/types/src/sszTypes.ts index c2044edb5e98..4399904a94bc 100644 --- a/packages/types/src/sszTypes.ts +++ b/packages/types/src/sszTypes.ts @@ -20,7 +20,7 @@ const typesByFork = { [ForkName.bellatrix]: {...phase0, ...altair, ...bellatrix}, [ForkName.capella]: {...phase0, ...altair, ...bellatrix, ...capella}, [ForkName.deneb]: {...phase0, ...altair, ...bellatrix, ...capella, ...deneb}, - [ForkName.deneb]: {...phase0, ...altair, ...bellatrix, ...capella, ...deneb, ...electra}, + [ForkName.electra]: {...phase0, ...altair, ...bellatrix, ...capella, ...deneb, ...electra}, }; /** diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index c0cc6591dd56..73cb84f5eab6 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -36,6 +36,7 @@ type TypesByFork = { BeaconState: phase0.BeaconState; SignedBeaconBlock: phase0.SignedBeaconBlock; Metadata: phase0.Metadata; + Attestation: phase0.Attestation; }; [ForkName.altair]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -53,6 +54,7 @@ type TypesByFork = { LightClientStore: altair.LightClientStore; SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; + Attestation: phase0.Attestation; }; [ForkName.bellatrix]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -78,6 +80,7 @@ type TypesByFork = { SSEPayloadAttributes: bellatrix.SSEPayloadAttributes; SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; + Attestation: phase0.Attestation; }; [ForkName.capella]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -103,6 +106,7 @@ type TypesByFork = { SSEPayloadAttributes: capella.SSEPayloadAttributes; SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; + Attestation: phase0.Attestation; }; [ForkName.deneb]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -133,6 +137,7 @@ type TypesByFork = { Contents: deneb.Contents; SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; + Attestation: phase0.Attestation; }; [ForkName.electra]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -167,6 +172,7 @@ type TypesByFork = { Contents: deneb.Contents; SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; + Attestation: electra.Attestation; }; }; @@ -225,3 +231,5 @@ export type Metadata = TypesByFork[F]["Metadata"]; export type BuilderBid = TypesByFork[F]["BuilderBid"]; export type SignedBuilderBid = TypesByFork[F]["SignedBuilderBid"]; export type SSEPayloadAttributes = TypesByFork[F]["SSEPayloadAttributes"]; + +export type Attestation = TypesByFork[F]["Attestation"]; diff --git a/packages/types/src/utils/typeguards.ts b/packages/types/src/utils/typeguards.ts index f006227e03c9..a41b45eec546 100644 --- a/packages/types/src/utils/typeguards.ts +++ b/packages/types/src/utils/typeguards.ts @@ -1,4 +1,4 @@ -import {ForkBlobs, ForkExecution} from "@lodestar/params"; +import {ForkBlobs, ForkExecution, ForkName} from "@lodestar/params"; import { BlockContents, SignedBeaconBlock, @@ -13,6 +13,7 @@ import { BlindedBeaconBlockBody, SignedBlockContents, BeaconBlock, + Attestation, } from "../types.js"; export function isExecutionPayload( @@ -66,3 +67,9 @@ export function isSignedBlockContents( ): data is SignedBlockContents { return (data as SignedBlockContents).kzgProofs !== undefined; } + +export function isElectraAttestation( + attestation: Attestation +): attestation is Attestation { + return (attestation as Attestation).committeeBits !== undefined; +} diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index ca1b36883a90..1eda005b70c3 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -225,5 +225,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Sun, 5 May 2024 00:17:57 +0530 Subject: [PATCH 027/145] chore: fix the rebase build (#6735) * chore: fix the rebase build * fix test --- packages/beacon-node/src/chain/regen/queued.ts | 2 +- .../perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 84907ea163af..57e64bd364ea 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -212,7 +212,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { */ updateUnfinalizedPubkeys(validators: UnfinalizedPubkeyIndexMap): void { let numStatesUpdated = 0; - const states = this.stateCache.getStates(); + const states = this.blockStateCache.getStates(); const cpStates = this.checkpointStateCache.getStates(); // Add finalized pubkeys to all states. diff --git a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts index 39bf1a1551c9..b8f5c30a70ea 100644 --- a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts +++ b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts @@ -7,7 +7,8 @@ import bls from "@chainsafe/bls"; import {ssz} from "@lodestar/types"; import {type CachedBeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; import {bytesToBigInt, intToBytes} from "@lodestar/utils"; -import {InMemoryCheckpointStateCache, StateContextCache} from "../../../../src/chain/stateCache/index.js"; +import {InMemoryCheckpointStateCache, BlockStateCacheImpl} from "../../../../src/chain/stateCache/index.js"; +import {BlockStateCache} from "../../../../src/chain/stateCache/types.js"; import {generateCachedElectraState} from "../../../utils/state.js"; // Benchmark date from Mon Nov 21 2023 - Intel Core i7-9750H @ 2.60Ghz @@ -22,7 +23,7 @@ describe("updateUnfinalizedPubkeys perf tests", function () { const numStateCache = 3 * 32; let checkpointStateCache: InMemoryCheckpointStateCache; - let stateCache: StateContextCache; + let stateCache: BlockStateCache; const unfinalizedPubkey2Index = generatePubkey2Index(0, Math.max.apply(null, numPubkeysToBeFinalizedCases)); const baseState = generateCachedElectraState(); @@ -36,7 +37,7 @@ describe("updateUnfinalizedPubkeys perf tests", function () { baseState.epochCtx.index2pubkey = []; checkpointStateCache = new InMemoryCheckpointStateCache({}); - stateCache = new StateContextCache({}); + stateCache = new BlockStateCacheImpl({}); for (let i = 0; i < numCheckpointStateCache; i++) { const clonedState = baseState.clone(); From 8db4adaecb852cde7bb5d7c04492167e9b092483 Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 6 May 2024 00:08:09 +0530 Subject: [PATCH 028/145] feat: upgrade 7002 exits to withdrawal request (#6736) * feat: upgrade 7002 exits to withdrawal request * fix types * fix types and references * further fix the types references and get build passing * update the process ops fn but needs to be extended by maxeb --- .../src/execution/engine/payloadIdCache.ts | 3 +- .../beacon-node/src/execution/engine/types.ts | 50 ++++++++++++------- packages/light-client/src/spec/utils.ts | 6 +-- packages/params/src/index.ts | 2 +- packages/params/src/presets/mainnet.ts | 2 +- packages/params/src/presets/minimal.ts | 2 +- packages/params/src/types.ts | 4 +- ...processExecutionLayerWithdrawalRequest.ts} | 25 +++++++--- .../src/block/processOperations.ts | 8 +-- .../src/slot/upgradeStateToElectra.ts | 2 +- .../state-transition/src/util/execution.ts | 7 +-- packages/types/src/electra/sszTypes.ts | 16 +++--- packages/types/src/electra/types.ts | 4 +- packages/validator/src/util/params.ts | 2 +- 14 files changed, 81 insertions(+), 52 deletions(-) rename packages/state-transition/src/block/{processExecutionLayerExit.ts => processExecutionLayerWithdrawalRequest.ts} (71%) diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index 960b061f12da..b5fe3d33e267 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -26,9 +26,10 @@ export type DepositReceiptV1 = { index: QUANTITY; }; -export type ExecutionLayerExitV1 = { +export type ExecutionLayerWithdrawalRequestV1 = { sourceAddress: DATA; validatorPubkey: DATA; + amount: QUANTITY; }; type FcuAttributes = {headBlockHash: DATA; finalizedBlockHash: DATA} & Omit; diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index f12f98e1a0f7..54b3a415ec98 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1, DepositReceiptV1, ExecutionLayerExitV1} from "./payloadIdCache.js"; +import {WithdrawalV1, DepositReceiptV1, ExecutionLayerWithdrawalRequestV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -126,14 +126,14 @@ export type ExecutionPayloadBodyRpc = { transactions: DATA[]; withdrawals: WithdrawalV1[] | null | undefined; depositReceipts: DepositReceiptV1[] | null | undefined; - exits: ExecutionLayerExitV1[] | null | undefined; + withdrawalRequests: ExecutionLayerWithdrawalRequestV1[] | null | undefined; }; export type ExecutionPayloadBody = { transactions: bellatrix.Transaction[]; withdrawals: capella.Withdrawals | null; depositReceipts: electra.DepositReceipts | null; - exits: electra.ExecutionLayerExits | null; + withdrawalRequests: electra.ExecutionLayerWithdrawalRequests | null; }; export type ExecutionPayloadRpc = { @@ -156,7 +156,7 @@ export type ExecutionPayloadRpc = { excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB depositReceipts?: DepositReceiptRpc[]; // ELECTRA - exits?: ExecutionLayerExitRpc[]; // ELECTRA + withdrawalRequests?: ExecutionLayerWithdrawalRequestRpc[]; // ELECTRA }; export type WithdrawalRpc = { @@ -167,7 +167,7 @@ export type WithdrawalRpc = { }; export type DepositReceiptRpc = DepositReceiptV1; -export type ExecutionLayerExitRpc = ExecutionLayerExitV1; +export type ExecutionLayerWithdrawalRequestRpc = ExecutionLayerWithdrawalRequestV1; export type VersionedHashesRpc = DATA[]; @@ -233,9 +233,9 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload // ELECTRA adds depositReceipts to the ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositReceipts, exits} = data as electra.ExecutionPayload; + const {depositReceipts, withdrawalRequests} = data as electra.ExecutionPayload; payload.depositReceipts = depositReceipts.map(serializeDepositReceipt); - payload.exits = exits.map(serializeExecutionLayerExit); + payload.withdrawalRequests = withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest); } return payload; @@ -324,7 +324,7 @@ export function parseExecutionPayload( } if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositReceipts, exits} = data; + const {depositReceipts, withdrawalRequests} = data; // Geth can also reply with null if (depositReceipts == null) { throw Error( @@ -333,12 +333,14 @@ export function parseExecutionPayload( } (executionPayload as electra.ExecutionPayload).depositReceipts = depositReceipts.map(deserializeDepositReceipt); - if (exits == null) { + if (withdrawalRequests == null) { throw Error( - `exits missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` + `withdrawalRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` ); } - (executionPayload as electra.ExecutionPayload).exits = exits.map(deserializeExecutionLayerExit); + (executionPayload as electra.ExecutionPayload).withdrawalRequests = withdrawalRequests.map( + deserializeExecutionLayerWithdrawalRequest + ); } return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; @@ -427,17 +429,23 @@ export function deserializeDepositReceipt(serialized: DepositReceiptRpc): electr } as electra.DepositReceipt; } -export function serializeExecutionLayerExit(exit: electra.ExecutionLayerExit): ExecutionLayerExitRpc { +export function serializeExecutionLayerWithdrawalRequest( + withdrawalRequest: electra.ExecutionLayerWithdrawalRequest +): ExecutionLayerWithdrawalRequestRpc { return { - sourceAddress: bytesToData(exit.sourceAddress), - validatorPubkey: bytesToData(exit.validatorPubkey), + sourceAddress: bytesToData(withdrawalRequest.sourceAddress), + validatorPubkey: bytesToData(withdrawalRequest.validatorPubkey), + amount: numToQuantity(withdrawalRequest.amount), }; } -export function deserializeExecutionLayerExit(exit: ExecutionLayerExitRpc): electra.ExecutionLayerExit { +export function deserializeExecutionLayerWithdrawalRequest( + withdrawalRequest: ExecutionLayerWithdrawalRequestRpc +): electra.ExecutionLayerWithdrawalRequest { return { - sourceAddress: dataToBytes(exit.sourceAddress, 20), - validatorPubkey: dataToBytes(exit.validatorPubkey, 48), + sourceAddress: dataToBytes(withdrawalRequest.sourceAddress, 20), + validatorPubkey: dataToBytes(withdrawalRequest.validatorPubkey, 48), + amount: quantityToNum(withdrawalRequest.amount), }; } @@ -447,7 +455,9 @@ export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, depositReceipts: data.depositReceipts ? data.depositReceipts.map(deserializeDepositReceipt) : null, - exits: data.exits ? data.exits.map(deserializeExecutionLayerExit) : null, + withdrawalRequests: data.withdrawalRequests + ? data.withdrawalRequests.map(deserializeExecutionLayerWithdrawalRequest) + : null, } : null; } @@ -458,7 +468,9 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, depositReceipts: data.depositReceipts ? data.depositReceipts.map(serializeDepositReceipt) : null, - exits: data.exits ? data.exits.map(serializeExecutionLayerExit) : null, + withdrawalRequests: data.withdrawalRequests + ? data.withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest) + : null, } : null; } diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 8e6b3456a600..1b460dd39cf2 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -117,8 +117,8 @@ export function upgradeLightClientHeader( case ForkName.electra: (upgradedHeader as LightClientHeader).execution.depositReceiptsRoot = ssz.electra.LightClientHeader.fields.execution.fields.depositReceiptsRoot.defaultValue(); - (upgradedHeader as electra.LightClientHeader).execution.exitsRoot = - ssz.electra.LightClientHeader.fields.execution.fields.exitsRoot.defaultValue(); + (upgradedHeader as electra.LightClientHeader).execution.withdrawalRequestsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.withdrawalRequestsRoot.defaultValue(); // Break if no further upgrades is required else fall through if (ForkSeq[targetFork] <= ForkSeq.electra) break; @@ -158,7 +158,7 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC if (epoch < config.ELECTRA_FORK_EPOCH) { if ( (header as LightClientHeader).execution.depositReceiptsRoot !== undefined || - (header as LightClientHeader).execution.exitsRoot !== undefined + (header as LightClientHeader).execution.withdrawalRequestsRoot !== undefined ) { return false; } diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 3e56effc4138..b261e07959d0 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -95,7 +95,7 @@ export const { KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, - MAX_EXECUTION_LAYER_EXITS, + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_ATTESTATIONS_ELECTRA, } = activePreset; diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 27cb7640b2dd..5343966a43f4 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -121,7 +121,7 @@ export const mainnetPreset: BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192, - MAX_EXECUTION_LAYER_EXITS: 16, + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16, MAX_ATTESTER_SLASHINGS_ELECTRA: 1, MAX_ATTESTATIONS_ELECTRA: 8, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 022532a49e6f..e4938d501a51 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -122,7 +122,7 @@ export const minimalPreset: BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4, - MAX_EXECUTION_LAYER_EXITS: 16, + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16, MAX_ATTESTER_SLASHINGS_ELECTRA: 1, MAX_ATTESTATIONS_ELECTRA: 8, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index 34f40a66707e..e5b85a9e2224 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -85,7 +85,7 @@ export type BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: number; - MAX_EXECUTION_LAYER_EXITS: number; + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: number; MAX_ATTESTER_SLASHINGS_ELECTRA: number; MAX_ATTESTATIONS_ELECTRA: number; }; @@ -176,7 +176,7 @@ export const beaconPresetTypes: BeaconPresetTypes = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: "number", - MAX_EXECUTION_LAYER_EXITS: "number", + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: "number", MAX_ATTESTER_SLASHINGS_ELECTRA: "number", MAX_ATTESTATIONS_ELECTRA: "number", }; diff --git a/packages/state-transition/src/block/processExecutionLayerExit.ts b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts similarity index 71% rename from packages/state-transition/src/block/processExecutionLayerExit.ts rename to packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts index 5068d9af8667..cfe719bf57f3 100644 --- a/packages/state-transition/src/block/processExecutionLayerExit.ts +++ b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts @@ -6,22 +6,33 @@ import {isActiveValidator} from "../util/index.js"; import {CachedBeaconStateElectra} from "../types.js"; import {initiateValidatorExit} from "./index.js"; +const FULL_EXIT_REQUEST_AMOUNT = 0; /** * Process execution layer exit messages and initiate exit incase they belong to a valid active validator * otherwise silent ignore. */ -export function processExecutionLayerExit(state: CachedBeaconStateElectra, exit: electra.ExecutionLayerExit): void { - const validator = isValidExecutionLayerExit(state, exit); - if (validator === null) { - return; - } +export function processExecutionLayerWithdrawalRequest( + state: CachedBeaconStateElectra, + withdrawalRequest: electra.ExecutionLayerWithdrawalRequest +): void { + const isFullExitRequest = withdrawalRequest.amount === FULL_EXIT_REQUEST_AMOUNT; + + if (isFullExitRequest) { + const validator = isValidExecutionLayerExit(state, withdrawalRequest); + if (validator === null) { + return; + } - initiateValidatorExit(state, validator); + initiateValidatorExit(state, validator); + } else { + // partial withdral request add codeblock + } } +// TODO electra : add pending withdrawal check before exit export function isValidExecutionLayerExit( state: CachedBeaconStateElectra, - exit: electra.ExecutionLayerExit + exit: electra.ExecutionLayerWithdrawalRequest ): CompositeViewDU | null { const {config, epochCtx} = state; const validatorIndex = epochCtx.getValidatorIndex(exit.validatorPubkey); diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index c4879da4aa71..228b4eef6fd3 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -8,7 +8,7 @@ import {processProposerSlashing} from "./processProposerSlashing.js"; import {processAttesterSlashing} from "./processAttesterSlashing.js"; import {processDeposit} from "./processDeposit.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; -import {processExecutionLayerExit} from "./processExecutionLayerExit.js"; +import {processExecutionLayerWithdrawalRequest} from "./processExecutionLayerWithdrawalRequest.js"; import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; import {processDepositReceipt} from "./processDepositReceipt.js"; import {ProcessBlockOpts} from "./types.js"; @@ -19,7 +19,7 @@ export { processAttestations, processDeposit, processVoluntaryExit, - processExecutionLayerExit, + processExecutionLayerWithdrawalRequest, processBlsToExecutionChange, processDepositReceipt, }; @@ -55,8 +55,8 @@ export function processOperations( processVoluntaryExit(state, voluntaryExit, opts.verifySignatures); } if (fork >= ForkSeq.electra) { - for (const elExit of (body as electra.BeaconBlockBody).executionPayload.exits) { - processExecutionLayerExit(state as CachedBeaconStateElectra, elExit); + for (const elWithdrawalRequest of (body as electra.BeaconBlockBody).executionPayload.withdrawalRequests) { + processExecutionLayerWithdrawalRequest(state as CachedBeaconStateElectra, elWithdrawalRequest); } } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 369ab19c447b..f41c37af94aa 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -20,7 +20,7 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache epoch: stateDeneb.epochCtx.epoch, }); - // latestExecutionPayloadHeader's depositReceiptsRoot and exitsRoot set to zeros by default + // latestExecutionPayloadHeader's depositReceiptsRoot and withdrawalRequestsRoot set to zeros by default // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 2975b84bf747..c7f0ec2f395c 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -174,9 +174,10 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio if (fork >= ForkSeq.electra) { (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositReceiptsRoot = ssz.electra.DepositReceipts.hashTreeRoot((payload as electra.ExecutionPayload).depositReceipts); - (bellatrixPayloadFields as electra.ExecutionPayloadHeader).exitsRoot = ssz.electra.ExecutionLayerExits.hashTreeRoot( - (payload as electra.ExecutionPayload).exits - ); + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).withdrawalRequestsRoot = + ssz.electra.ExecutionLayerWithdrawalRequests.hashTreeRoot( + (payload as electra.ExecutionPayload).withdrawalRequests + ); } return bellatrixPayloadFields; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 3404baf04110..53195f30de11 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -16,7 +16,7 @@ import { MAX_COMMITTEES_PER_SLOT, MAX_ATTESTATIONS_ELECTRA, MAX_ATTESTER_SLASHINGS_ELECTRA, - MAX_EXECUTION_LAYER_EXITS, + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, } from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; @@ -117,20 +117,24 @@ export const DepositReceipt = new ContainerType( export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); -export const ExecutionLayerExit = new ContainerType( +export const ExecutionLayerWithdrawalRequest = new ContainerType( { sourceAddress: ExecutionAddress, validatorPubkey: BLSPubkey, + amount: UintNum64, }, - {typeName: "ExecutionLayerExit", jsonCase: "eth2"} + {typeName: "ExecutionLayerWithdrawalRequest", jsonCase: "eth2"} +); +export const ExecutionLayerWithdrawalRequests = new ListCompositeType( + ExecutionLayerWithdrawalRequest, + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD ); -export const ExecutionLayerExits = new ListCompositeType(ExecutionLayerExit, MAX_EXECUTION_LAYER_EXITS); export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, depositReceipts: DepositReceipts, // New in ELECTRA - exits: ExecutionLayerExits, // New in ELECTRA + withdrawalRequests: ExecutionLayerWithdrawalRequests, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -139,7 +143,7 @@ export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, depositReceiptsRoot: Root, // New in ELECTRA - exitsRoot: Root, // New in ELECTRA + withdrawalRequestsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 2925885b3af3..d1d0109e6ae6 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -12,8 +12,8 @@ export type SignedAggregateAndProof = ValueOf; export type DepositReceipts = ValueOf; -export type ExecutionLayerExit = ValueOf; -export type ExecutionLayerExits = ValueOf; +export type ExecutionLayerWithdrawalRequest = ValueOf; +export type ExecutionLayerWithdrawalRequests = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 1eda005b70c3..ff1c8c0fdc25 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -224,7 +224,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Tue, 7 May 2024 21:18:41 +0300 Subject: [PATCH 029/145] feat: implement maxEB EIP-7251 (#6539) * feat: implement EIP-6110 (#6042) * Add immutable in the dependencies * Initial change to pubkeyCache * Added todos * Moved unfinalized cache to epochCache * Move populating finalized cache to afterProcessEpoch * Specify unfinalized cache during state cloning * Move from unfinalized to finalized cache in afterProcessEpoch * Confused myself * Clean up * Change logic * Fix cloning issue * Clean up redundant code * Add CarryoverData in epochCtx.createFromState * Fix typo * Update usage of pubkeyCache * Update pubkeyCache usage * Fix lint * Fix lint * Add 6110 to ChainConfig * Add 6110 to BeaconPreset * Define 6110 fork and container * Add V6110 api to execution engine * Update test * Add depositReceiptsRoot to process_execution_payload * State transitioning to EIP6110 * State transitioning to EIP6110 * Light client change in EIP-6110 * Update tests * produceBlock * Refactor processDeposit to match the spec * Implement processDepositReceipt * Implement 6110 fork guard for pubkeyCache * Handle changes in eth1 deposit * Update eth1 deposit test * Fix typo * Lint * Remove embarassing comments * Address comments * Modify applyDeposit signature * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/pubkeyCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Remove old code * Rename fields in epochCache and immutableData * Remove CarryoverData * Move isAfter6110 from var to method * Fix cyclic import * Fix operations spec runner * Fix for spec test * Fix spec test * state.depositReceiptsStartIndex to BigInt * getDeposit requires cached state * default depositReceiptsStartIndex value in genesis * Fix pubkeyCache bug * newUnfinalizedPubkeyIndexMap in createCachedBeaconState * Lint * Pass epochCache instead of pubkey2IndexFn in apis * Address comments * Add unit test on pubkey cache cloning * Add unfinalizedPubkeyCacheSize to metrics * Add unfinalizedPubkeyCacheSize to metrics * Clean up code * Add besu to el-interop * Add 6110 genesis file * Template for sim test * Add unit test for getEth1DepositCount * Update sim test * Update besudocker * Finish beacon api calls in sim test * Update epochCache.createFromState() * Fix bug unfinalized validators are not finalized * Add sim test to run a few blocks * Lint * Merge branch 'unstable' into 611 * Add more check to sim test * Update besu docker image instruction * Update sim test with correct tx * Address comment + cleanup * Clean up code * Properly handle promise rejection * Lint * Update packages/beacon-node/src/execution/engine/types.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update comments * Accept type undefined in ExecutionPayloadBodyRpc * Update comment and semantic * Remove if statement when adding finalized validator * Comment on repeated insert on finalized cache * rename createFromState * Add comment on getPubkey() * Stash change to reduce diffs * Stash change to reduce diffs * Lint * addFinalizedPubkey on finalized checkpoint * Update comment * Use OrderedMap for unfinalized cache * Pull out logic of deleting pubkeys for batch op * Add updateUnfinalizedPubkeys in regen * Update updateUnfinalizedPubkeys logic * Add comment * Add metrics for state context caches * Address comment * Address comment * Deprecate eth1Data polling when condition is reached * Fix conflicts * Fix sim test * Lint * Fix type * Fix test * Fix test * Lint * Update packages/light-client/src/spec/utils.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Fix spec test * Address comments * Improve cache logic on checkpoint finalized * Update sim test according to new cache logic * Update comment * Lint * Finalized pubkey cache only update once per checkpoint * Add perf test for updateUnfinalizedPubkeys * Add perf test for updateUnfinalizedPubkeys * Tweak params for perf test * Freeze besu docker image version for 6110 * Add benchmark result * Use Map instead of OrderedMap. Update benchmark * Minor optimization * Minor optimization * Add memory test for immutable.js * Update test * Reduce code duplication * Lint * Remove try/catch in updateUnfinalizedPubkeys * Introduce EpochCache metric * Add historicalValidatorLengths * Polish code * Migrate state-transition unit tests to vitest * Fix calculation of pivot index * `historicalValidatorLengths` only activate post 6110 * Update sim test * Lint * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Improve readability on historicalValidatorLengths * Update types * Fix calculation * Add eth1data poll todo * Add epochCache.getValidatorCountAtEpoch * Add todo * Add getStateIterator for state cache * Partial commit * Update perf test * updateUnfinalizedPubkeys directly modify states from regen * Update sim test. Lint * Add todo * some improvements and a fix for effectiveBalanceIncrements fork safeness * rename eip6110 to elctra * fix electra-interop.test.ts --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Co-authored-by: gajinder lint and tsc small cleanup fix rebase issue * feat: implement EIP-6110 (#6042) * Add immutable in the dependencies * Initial change to pubkeyCache * Added todos * Moved unfinalized cache to epochCache * Move populating finalized cache to afterProcessEpoch * Specify unfinalized cache during state cloning * Move from unfinalized to finalized cache in afterProcessEpoch * Confused myself * Clean up * Change logic * Fix cloning issue * Clean up redundant code * Add CarryoverData in epochCtx.createFromState * Fix typo * Update usage of pubkeyCache * Update pubkeyCache usage * Fix lint * Fix lint * Add 6110 to ChainConfig * Add 6110 to BeaconPreset * Define 6110 fork and container * Add V6110 api to execution engine * Update test * Add depositReceiptsRoot to process_execution_payload * State transitioning to EIP6110 * State transitioning to EIP6110 * Light client change in EIP-6110 * Update tests * produceBlock * Refactor processDeposit to match the spec * Implement processDepositReceipt * Implement 6110 fork guard for pubkeyCache * Handle changes in eth1 deposit * Update eth1 deposit test * Fix typo * Lint * Remove embarassing comments * Address comments * Modify applyDeposit signature * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/pubkeyCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Remove old code * Rename fields in epochCache and immutableData * Remove CarryoverData * Move isAfter6110 from var to method * Fix cyclic import * Fix operations spec runner * Fix for spec test * Fix spec test * state.depositReceiptsStartIndex to BigInt * getDeposit requires cached state * default depositReceiptsStartIndex value in genesis * Fix pubkeyCache bug * newUnfinalizedPubkeyIndexMap in createCachedBeaconState * Lint * Pass epochCache instead of pubkey2IndexFn in apis * Address comments * Add unit test on pubkey cache cloning * Add unfinalizedPubkeyCacheSize to metrics * Add unfinalizedPubkeyCacheSize to metrics * Clean up code * Add besu to el-interop * Add 6110 genesis file * Template for sim test * Add unit test for getEth1DepositCount * Update sim test * Update besudocker * Finish beacon api calls in sim test * Update epochCache.createFromState() * Fix bug unfinalized validators are not finalized * Add sim test to run a few blocks * Lint * Merge branch 'unstable' into 611 * Add more check to sim test * Update besu docker image instruction * Update sim test with correct tx * Address comment + cleanup * Clean up code * Properly handle promise rejection * Lint * Update packages/beacon-node/src/execution/engine/types.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update comments * Accept type undefined in ExecutionPayloadBodyRpc * Update comment and semantic * Remove if statement when adding finalized validator * Comment on repeated insert on finalized cache * rename createFromState * Add comment on getPubkey() * Stash change to reduce diffs * Stash change to reduce diffs * Lint * addFinalizedPubkey on finalized checkpoint * Update comment * Use OrderedMap for unfinalized cache * Pull out logic of deleting pubkeys for batch op * Add updateUnfinalizedPubkeys in regen * Update updateUnfinalizedPubkeys logic * Add comment * Add metrics for state context caches * Address comment * Address comment * Deprecate eth1Data polling when condition is reached * Fix conflicts * Fix sim test * Lint * Fix type * Fix test * Fix test * Lint * Update packages/light-client/src/spec/utils.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Fix spec test * Address comments * Improve cache logic on checkpoint finalized * Update sim test according to new cache logic * Update comment * Lint * Finalized pubkey cache only update once per checkpoint * Add perf test for updateUnfinalizedPubkeys * Add perf test for updateUnfinalizedPubkeys * Tweak params for perf test * Freeze besu docker image version for 6110 * Add benchmark result * Use Map instead of OrderedMap. Update benchmark * Minor optimization * Minor optimization * Add memory test for immutable.js * Update test * Reduce code duplication * Lint * Remove try/catch in updateUnfinalizedPubkeys * Introduce EpochCache metric * Add historicalValidatorLengths * Polish code * Migrate state-transition unit tests to vitest * Fix calculation of pivot index * `historicalValidatorLengths` only activate post 6110 * Update sim test * Lint * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Improve readability on historicalValidatorLengths * Update types * Fix calculation * Add eth1data poll todo * Add epochCache.getValidatorCountAtEpoch * Add todo * Add getStateIterator for state cache * Partial commit * Update perf test * updateUnfinalizedPubkeys directly modify states from regen * Update sim test. Lint * Add todo * some improvements and a fix for effectiveBalanceIncrements fork safeness * rename eip6110 to elctra * fix electra-interop.test.ts --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Co-authored-by: gajinder lint and tsc small cleanup * Add presets * Update config * Add necessary containers * Update presets * Update config * Add todo comments * Update constants and params * Impl new process withdrawal * Add withdrawaRequests to payload * Add processConsolidation * Add process withdraw request * Update deposit and withdrawal flow * epoch processing * Implement churn limits * Lint * lint * Update state-transition utils * processExecutionLayerWithdrawRequest * processConsolidation * queueExcessActiveBalance * isValidDepositSignature * Add jsdoc and timer for new processEpoch functions * Lint * Update maxEB * update voluntary exit * Fix config * Update initiateValidatorExit * Remove churn limit in processRegistryUpdates * Fix conflict * Add MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD * Reflect latest spec changes * rebase fixes, fixes, improvements and cleanup lint * Upgrade ssz version * Use sliceFrom() * cleanup as per specs feedback subarry * simplify * fix withdrawals * remove slice * fix the slashing quotient determination in slashvalidator --------- Co-authored-by: harkamal --- .../chain/produceBlock/produceBlockBody.ts | 2 + .../test/sim/electra-interop.test.ts | 2 +- .../test/spec/presets/operations.test.ts | 4 +- .../test/unit/executionEngine/http.test.ts | 8 +- .../config/src/chainConfig/configs/mainnet.ts | 6 + .../config/src/chainConfig/configs/minimal.ts | 6 + packages/config/src/chainConfig/types.ts | 4 + packages/params/src/index.ts | 14 +- packages/params/src/presets/mainnet.ts | 11 ++ packages/params/src/presets/minimal.ts | 13 +- packages/params/src/types.ts | 18 +++ packages/state-transition/src/block/index.ts | 2 + .../src/block/initiateValidatorExit.ts | 39 ++++-- .../src/block/processConsolidation.ts | 111 +++++++++++++++ .../src/block/processDeposit.ts | 98 ++++++++++---- .../processExecutionLayerWithdrawalRequest.ts | 126 +++++++++++------- .../src/block/processOperations.ts | 25 ++-- .../src/block/processVoluntaryExit.ts | 30 ++++- .../src/block/processWithdrawals.ts | 87 +++++++++--- .../src/block/slashValidator.ts | 15 ++- .../state-transition/src/cache/epochCache.ts | 2 + .../src/cache/epochTransitionCache.ts | 8 +- packages/state-transition/src/epoch/index.ts | 32 ++++- .../epoch/processEffectiveBalanceUpdates.ts | 22 ++- .../epoch/processPendingBalanceDeposits.ts | 35 +++++ .../src/epoch/processPendingConsolidations.ts | 45 +++++++ .../src/epoch/processRegistryUpdates.ts | 14 +- .../src/signatureSets/consolidation.ts | 51 +++++++ .../src/signatureSets/index.ts | 16 ++- .../src/slot/upgradeStateToElectra.ts | 24 +++- packages/state-transition/src/util/electra.ts | 103 ++++++++++++++ packages/state-transition/src/util/epoch.ts | 56 ++++++++ packages/state-transition/src/util/genesis.ts | 2 +- packages/state-transition/src/util/index.ts | 1 + .../state-transition/src/util/validator.ts | 55 +++++++- .../test/perf/analyzeEpochs.ts | 3 + .../perf/block/processWithdrawals.test.ts | 5 +- .../unit/block/processWithdrawals.test.ts | 4 +- packages/types/src/electra/sszTypes.ts | 60 ++++++++- packages/types/src/electra/types.ts | 7 + packages/types/src/phase0/sszTypes.ts | 2 +- packages/validator/src/util/params.ts | 11 ++ .../test/unit/utils/interopConfigs.ts | 34 +++++ 43 files changed, 1056 insertions(+), 157 deletions(-) create mode 100644 packages/state-transition/src/block/processConsolidation.ts create mode 100644 packages/state-transition/src/epoch/processPendingBalanceDeposits.ts create mode 100644 packages/state-transition/src/epoch/processPendingConsolidations.ts create mode 100644 packages/state-transition/src/signatureSets/consolidation.ts create mode 100644 packages/state-transition/src/util/electra.ts diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index d1b410610a43..ba560d5a7ff0 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -559,7 +559,9 @@ function preparePayloadAttributes( }; if (ForkSeq[fork] >= ForkSeq.capella) { + // withdrawals logic is now fork aware as it changes on electra fork post capella (payloadAttributes as capella.SSEPayloadAttributes["payloadAttributes"]).withdrawals = getExpectedWithdrawals( + ForkSeq[fork], prepareState as CachedBeaconStateCapella ).withdrawals; } diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index ab3f1ae6b2c1..29483b249c85 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -173,7 +173,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { blockHash: dataToBytes(newPayloadBlockHash, 32), receiptsRoot: dataToBytes("0x79ee3424eb720a3ad4b1c5a372bb8160580cbe4d893778660f34213c685627a9", 32), blobGasUsed: 0n, - exits: [], + withdrawalRequests: [], }; const parentBeaconBlockRoot = dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32); const payloadResult = await executionEngine.notifyNewPayload( diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index cc032bec6fa2..9b8214a88c0a 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -11,7 +11,7 @@ import { import * as blockFns from "@lodestar/state-transition/block"; import {ssz, phase0, altair, bellatrix, capella, electra, sszTypesFor} from "@lodestar/types"; import {InputType} from "@lodestar/spec-test-util"; -import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; +import {ACTIVE_PRESET, ForkName, ForkSeq} from "@lodestar/params"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; @@ -88,7 +88,7 @@ const operationFns: Record> = }, withdrawals: (state, testCase: {execution_payload: capella.ExecutionPayload}) => { - blockFns.processWithdrawals(state as CachedBeaconStateCapella, testCase.execution_payload); + blockFns.processWithdrawals(ForkSeq.capella, state as CachedBeaconStateCapella, testCase.execution_payload); }, }; diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index d58eefa10368..842f39a5ff90 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -194,7 +194,7 @@ describe("ExecutionEngine / http", () => { }, ], depositReceipts: null, // depositReceipts is null pre-electra - exits: null, + withdrawalRequests: null, }, null, // null returned for missing blocks { @@ -204,7 +204,7 @@ describe("ExecutionEngine / http", () => { ], withdrawals: null, // withdrawals is null pre-capella depositReceipts: null, // depositReceipts is null pre-electra - exits: null, + withdrawalRequests: null, }, ], }; @@ -253,7 +253,7 @@ describe("ExecutionEngine / http", () => { }, ], depositReceipts: null, // depositReceipts is null pre-electra - exits: null, + withdrawalRequests: null, }, null, // null returned for missing blocks { @@ -263,7 +263,7 @@ describe("ExecutionEngine / http", () => { ], withdrawals: null, // withdrawals is null pre-capella depositReceipts: null, // depositReceipts is null pre-electra - exits: null, + withdrawalRequests: null, }, ], }; diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 6d1fd9b75952..741ddc99f8cd 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -102,4 +102,10 @@ export const chainConfig: ChainConfig = { // Deneb // `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096, + + // Electra + // 2**8 * 10**9 (= 256,000,000,000) + MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 256000000000, + // 2*7 * 10**9 (= 128,000,000,000) + MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000, }; diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index 44a28ca36ec7..26f49cc3e47d 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -99,4 +99,10 @@ export const chainConfig: ChainConfig = { // Deneb // `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096, + + // Electra + // 2**7 * 10**9 (= 128,000,000,000) + MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 128000000000, + // 2**6 * 10**9 (= 64,000,000,000) + MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000, }; diff --git a/packages/config/src/chainConfig/types.ts b/packages/config/src/chainConfig/types.ts index 234a08558be5..05fff02f2eaf 100644 --- a/packages/config/src/chainConfig/types.ts +++ b/packages/config/src/chainConfig/types.ts @@ -58,6 +58,8 @@ export type ChainConfig = { MIN_PER_EPOCH_CHURN_LIMIT: number; MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: number; CHURN_LIMIT_QUOTIENT: number; + MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: number; + MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: number; // Fork choice PROPOSER_SCORE_BOOST: number; @@ -120,6 +122,8 @@ export const chainConfigTypes: SpecTypes = { MIN_PER_EPOCH_CHURN_LIMIT: "number", MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: "number", CHURN_LIMIT_QUOTIENT: "number", + MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: "number", + MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: "number", // Fork choice PROPOSER_SCORE_BOOST: "number", diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index b261e07959d0..c1d3f1bc1981 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -94,10 +94,20 @@ export const { MAX_BLOBS_PER_BLOCK, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, + MAX_EFFECTIVE_BALANCE_ELECTRA, + MIN_ACTIVATION_BALANCE, + PENDING_BALANCE_DEPOSITS_LIMIT, + PENDING_PARTIAL_WITHDRAWALS_LIMIT, + PENDING_CONSOLIDATIONS_LIMIT, + MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA, + MAX_CONSOLIDATIONS, + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_ATTESTATIONS_ELECTRA, + MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, + WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA, } = activePreset; //////////// @@ -119,6 +129,7 @@ export const JUSTIFICATION_BITS_LENGTH = 4; // Since the prefixes are just 1 byte, we define and use them as number export const BLS_WITHDRAWAL_PREFIX = 0; export const ETH1_ADDRESS_WITHDRAWAL_PREFIX = 1; +export const COMPOUNDING_WITHDRAWAL_PREFIX = 2; // Domain types @@ -133,7 +144,7 @@ export const DOMAIN_SYNC_COMMITTEE = Uint8Array.from([7, 0, 0, 0]); export const DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF = Uint8Array.from([8, 0, 0, 0]); export const DOMAIN_CONTRIBUTION_AND_PROOF = Uint8Array.from([9, 0, 0, 0]); export const DOMAIN_BLS_TO_EXECUTION_CHANGE = Uint8Array.from([10, 0, 0, 0]); -export const DOMAIN_BLOB_SIDECAR = Uint8Array.from([11, 0, 0, 0]); +export const DOMAIN_CONSOLIDATION = Uint8Array.from([11, 0, 0, 0]); // Application specific domains @@ -252,3 +263,4 @@ export const BLOBSIDECAR_FIXED_SIZE = ACTIVE_PRESET === PresetName.minimal ? 131 // Electra Misc export const UNSET_DEPOSIT_RECEIPTS_START_INDEX = 2n ** 64n - 1n; +export const FULL_EXIT_REQUEST_AMOUNT = 0; diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 5343966a43f4..2495f7ef97a1 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -124,4 +124,15 @@ export const mainnetPreset: BeaconPreset = { MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16, MAX_ATTESTER_SLASHINGS_ELECTRA: 1, MAX_ATTESTATIONS_ELECTRA: 8, + MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 8, + // 2**11 * 10**9 (= 2,048,000,000,000) Gwei + MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000, + // 2**16 (= 65536) + MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: 4096, + MIN_ACTIVATION_BALANCE: 32000000000, + PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, + PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728, + PENDING_CONSOLIDATIONS_LIMIT: 262144, + MAX_CONSOLIDATIONS: 1, + WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index e4938d501a51..8e71407965d7 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -122,7 +122,18 @@ export const minimalPreset: BeaconPreset = { // ELECTRA MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4, - MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16, + MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2, MAX_ATTESTER_SLASHINGS_ELECTRA: 1, MAX_ATTESTATIONS_ELECTRA: 8, + MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 1, + // 2**11 * 10**9 (= 2,048,000,000,000) Gwei + MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000, + // 2**16 (= 65536) + MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: 4096, + MIN_ACTIVATION_BALANCE: 32000000000, + PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, + PENDING_PARTIAL_WITHDRAWALS_LIMIT: 64, + PENDING_CONSOLIDATIONS_LIMIT: 64, + MAX_CONSOLIDATIONS: 1, + WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index e5b85a9e2224..dffd98518006 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -88,6 +88,15 @@ export type BeaconPreset = { MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: number; MAX_ATTESTER_SLASHINGS_ELECTRA: number; MAX_ATTESTATIONS_ELECTRA: number; + MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: number; + MAX_EFFECTIVE_BALANCE_ELECTRA: number; + MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: number; + MIN_ACTIVATION_BALANCE: number; + PENDING_BALANCE_DEPOSITS_LIMIT: number; + PENDING_PARTIAL_WITHDRAWALS_LIMIT: number; + PENDING_CONSOLIDATIONS_LIMIT: number; + MAX_CONSOLIDATIONS: number; + WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: number; }; /** @@ -179,6 +188,15 @@ export const beaconPresetTypes: BeaconPresetTypes = { MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: "number", MAX_ATTESTER_SLASHINGS_ELECTRA: "number", MAX_ATTESTATIONS_ELECTRA: "number", + MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: "number", + MAX_EFFECTIVE_BALANCE_ELECTRA: "number", + MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: "number", + MIN_ACTIVATION_BALANCE: "number", + PENDING_BALANCE_DEPOSITS_LIMIT: "number", + PENDING_PARTIAL_WITHDRAWALS_LIMIT: "number", + PENDING_CONSOLIDATIONS_LIMIT: "number", + MAX_CONSOLIDATIONS: "number", + WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: "number", }; type BeaconPresetTypes = { diff --git a/packages/state-transition/src/block/index.ts b/packages/state-transition/src/block/index.ts index fdfc9e903518..3857511292c8 100644 --- a/packages/state-transition/src/block/index.ts +++ b/packages/state-transition/src/block/index.ts @@ -47,10 +47,12 @@ export function processBlock( // https://github.com/ethereum/consensus-specs/blob/b62c9e877990242d63aa17a2a59a49bc649a2f2e/specs/eip4844/beacon-chain.md#disabling-withdrawals if (fork >= ForkSeq.capella) { processWithdrawals( + fork, state as CachedBeaconStateCapella, fullOrBlindedPayload as capella.FullOrBlindedExecutionPayload ); } + processExecutionPayload(fork, state as CachedBeaconStateBellatrix, block.body, externalData); } diff --git a/packages/state-transition/src/block/initiateValidatorExit.ts b/packages/state-transition/src/block/initiateValidatorExit.ts index e34d4dda7002..d1420daef84c 100644 --- a/packages/state-transition/src/block/initiateValidatorExit.ts +++ b/packages/state-transition/src/block/initiateValidatorExit.ts @@ -1,7 +1,8 @@ import {CompositeViewDU} from "@chainsafe/ssz"; -import {FAR_FUTURE_EPOCH} from "@lodestar/params"; +import {FAR_FUTURE_EPOCH, ForkSeq} from "@lodestar/params"; import {ssz} from "@lodestar/types"; -import {CachedBeaconStateAllForks} from "../types.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateElectra} from "../types.js"; +import {computeExitEpochAndUpdateChurn} from "../util/epoch.js"; /** * Initiate the exit of the validator with index ``index``. @@ -24,6 +25,7 @@ import {CachedBeaconStateAllForks} from "../types.js"; * Forcing consumers to pass the SubTree of `validator` directly mitigates this issue. */ export function initiateValidatorExit( + fork: ForkSeq, state: CachedBeaconStateAllForks, validator: CompositeViewDU ): void { @@ -34,18 +36,27 @@ export function initiateValidatorExit( return; } - // Limits the number of validators that can exit on each epoch. - // Expects all state.validators to follow this rule, i.e. no validator.exitEpoch is greater than exitQueueEpoch. - // If there the churnLimit is reached at this current exitQueueEpoch, advance epoch and reset churn. - if (epochCtx.exitQueueChurn >= epochCtx.churnLimit) { - epochCtx.exitQueueEpoch += 1; - epochCtx.exitQueueChurn = 1; // = 1 to account for this validator with exitQueueEpoch + if (fork < ForkSeq.electra) { + // Limits the number of validators that can exit on each epoch. + // Expects all state.validators to follow this rule, i.e. no validator.exitEpoch is greater than exitQueueEpoch. + // If there the churnLimit is reached at this current exitQueueEpoch, advance epoch and reset churn. + if (epochCtx.exitQueueChurn >= epochCtx.churnLimit) { + epochCtx.exitQueueEpoch += 1; + epochCtx.exitQueueChurn = 1; // = 1 to account for this validator with exitQueueEpoch + } else { + // Add this validator to the current exitQueueEpoch churn + epochCtx.exitQueueChurn += 1; + } + + // set validator exit epoch + validator.exitEpoch = epochCtx.exitQueueEpoch; } else { - // Add this validator to the current exitQueueEpoch churn - epochCtx.exitQueueChurn += 1; + // set validator exit epoch + // Note we don't use epochCtx.exitQueueChurn and exitQueueEpoch anymore + validator.exitEpoch = computeExitEpochAndUpdateChurn( + state as CachedBeaconStateElectra, + BigInt(validator.effectiveBalance) + ); } - - // set validator exit epoch and withdrawable epoch - validator.exitEpoch = epochCtx.exitQueueEpoch; - validator.withdrawableEpoch = epochCtx.exitQueueEpoch + config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; + validator.withdrawableEpoch = validator.exitEpoch + config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; } diff --git a/packages/state-transition/src/block/processConsolidation.ts b/packages/state-transition/src/block/processConsolidation.ts new file mode 100644 index 000000000000..846b0e2521b4 --- /dev/null +++ b/packages/state-transition/src/block/processConsolidation.ts @@ -0,0 +1,111 @@ +import {toHexString} from "@chainsafe/ssz"; +import {electra, ssz} from "@lodestar/types"; +import {FAR_FUTURE_EPOCH, MIN_ACTIVATION_BALANCE, PENDING_CONSOLIDATIONS_LIMIT} from "@lodestar/params"; +import {verifyConsolidationSignature} from "../signatureSets/index.js"; + +import {CachedBeaconStateElectra} from "../types.js"; +import {getConsolidationChurnLimit, isActiveValidator} from "../util/validator.js"; +import {hasExecutionWithdrawalCredential} from "../util/electra.js"; +import {computeConsolidationEpochAndUpdateChurn} from "../util/epoch.js"; + +export function processConsolidation( + state: CachedBeaconStateElectra, + signedConsolidation: electra.SignedConsolidation +): void { + assertValidConsolidation(state, signedConsolidation); + + // Initiate source validator exit and append pending consolidation + const {sourceIndex, targetIndex} = signedConsolidation.message; + const sourceValidator = state.validators.get(sourceIndex); + + const exitEpoch = computeConsolidationEpochAndUpdateChurn(state, BigInt(sourceValidator.effectiveBalance)); + sourceValidator.exitEpoch = exitEpoch; + sourceValidator.withdrawableEpoch = exitEpoch + state.config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; + + const pendingConsolidation = ssz.electra.PendingConsolidation.toViewDU({ + sourceIndex, + targetIndex, + }); + state.pendingConsolidations.push(pendingConsolidation); +} + +function assertValidConsolidation( + state: CachedBeaconStateElectra, + signedConsolidation: electra.SignedConsolidation +): void { + // If the pending consolidations queue is full, no consolidations are allowed in the block + if (state.pendingConsolidations.length >= PENDING_CONSOLIDATIONS_LIMIT) { + throw new Error("Pending consolidation queue is full"); + } + + // If there is too little available consolidation churn limit, no consolidations are allowed in the block + // assert get_consolidation_churn_limit(state) > MIN_ACTIVATION_BALANCE + if (getConsolidationChurnLimit(state) <= MIN_ACTIVATION_BALANCE) { + throw new Error(`Consolidation churn limit too low. consolidationChurnLimit=${getConsolidationChurnLimit(state)}`); + } + + const consolidation = signedConsolidation.message; + const {sourceIndex, targetIndex} = consolidation; + + // Verify that source != target, so a consolidation cannot be used as an exit. + if (sourceIndex === targetIndex) { + throw new Error( + `Consolidation source and target index cannot be the same: sourceIndex=${sourceIndex} targetIndex=${targetIndex}` + ); + } + + const sourceValidator = state.validators.getReadonly(sourceIndex); + const targetValidator = state.validators.getReadonly(targetIndex); + const currentEpoch = state.epochCtx.epoch; + + // Verify the source and the target are active + if (!isActiveValidator(sourceValidator, currentEpoch)) { + throw new Error(`Consolidation source validator is not active: sourceIndex=${sourceIndex}`); + } + + if (!isActiveValidator(targetValidator, currentEpoch)) { + throw new Error(`Consolidation target validator is not active: targetIndex=${targetIndex}`); + } + + // Verify exits for source and target have not been initiated + if (sourceValidator.exitEpoch !== FAR_FUTURE_EPOCH) { + throw new Error(`Consolidation source validator has initialized exit: sourceIndex=${sourceIndex}`); + } + if (targetValidator.exitEpoch !== FAR_FUTURE_EPOCH) { + throw new Error(`Consolidation target validator has initialized exit: targetIndex=${targetIndex}`); + } + + // Consolidations must specify an epoch when they become valid; they are not valid before then + if (currentEpoch < consolidation.epoch) { + throw new Error( + `Consolidation epoch is after the current epoch: consolidationEpoch=${consolidation.epoch} currentEpoch=${currentEpoch}` + ); + } + + // Verify the source and the target have Execution layer withdrawal credentials + if (!hasExecutionWithdrawalCredential(sourceValidator.withdrawalCredentials)) { + throw new Error( + `Consolidation source validator does not have execution withdrawal credentials: sourceIndex=${sourceIndex}` + ); + } + if (!hasExecutionWithdrawalCredential(targetValidator.withdrawalCredentials)) { + throw new Error( + `Consolidation target validator does not have execution withdrawal credentials: targetIndex=${targetIndex}` + ); + } + + // Verify the same withdrawal address + const sourceWithdrawalAddress = toHexString(sourceValidator.withdrawalCredentials.subarray(12)); + const targetWithdrawalAddress = toHexString(targetValidator.withdrawalCredentials.subarray(12)); + + if (sourceWithdrawalAddress !== targetWithdrawalAddress) { + throw new Error( + `Consolidation source and target withdrawal address are different: source: ${sourceWithdrawalAddress} target: ${targetWithdrawalAddress}` + ); + } + + // Verify consolidation is signed by the source and the target + if (!verifyConsolidationSignature(state, signedConsolidation)) { + throw new Error("Consolidation not valid"); + } +} diff --git a/packages/state-transition/src/block/processDeposit.ts b/packages/state-transition/src/block/processDeposit.ts index 5a2a892a5c9b..65b3cbfb10fd 100644 --- a/packages/state-transition/src/block/processDeposit.ts +++ b/packages/state-transition/src/block/processDeposit.ts @@ -13,9 +13,17 @@ import { import {DepositData} from "@lodestar/types/lib/phase0/types.js"; import {DepositReceipt} from "@lodestar/types/lib/electra/types.js"; +import {BeaconConfig} from "@lodestar/config"; import {ZERO_HASH} from "../constants/index.js"; -import {computeDomain, computeSigningRoot, increaseBalance} from "../util/index.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; +import { + computeDomain, + computeSigningRoot, + hasCompoundingWithdrawalCredential, + hasEth1WithdrawalCredential, + increaseBalance, + switchToCompoundingValidator, +} from "../util/index.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStateElectra} from "../types.js"; /** * Process a Deposit operation. Potentially adds a new validator to the registry. Mutates the validators and balances @@ -58,29 +66,29 @@ export function applyDeposit( const cachedIndex = epochCtx.getValidatorIndex(pubkey); if (cachedIndex === undefined || !Number.isSafeInteger(cachedIndex) || cachedIndex >= validators.length) { - // verify the deposit signature (proof of posession) which is not checked by the deposit contract - const depositMessage = { - pubkey, - withdrawalCredentials, - amount, - }; - // fork-agnostic domain since deposits are valid across forks - const domain = computeDomain(DOMAIN_DEPOSIT, config.GENESIS_FORK_VERSION, ZERO_HASH); - const signingRoot = computeSigningRoot(ssz.phase0.DepositMessage, depositMessage, domain); - try { - // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed - const publicKey = PublicKey.fromBytes(pubkey, true); - const signature = Signature.fromBytes(deposit.data.signature, true); - if (!verify(signingRoot, publicKey, signature)) { - return; - } - } catch (e) { - return; // Catch all BLS errors: failed key validation, failed signature validation, invalid signature + if (isValidDepositSignature(config, pubkey, withdrawalCredentials, amount, deposit.signature)) { + addValidatorToRegistry(fork, state, pubkey, withdrawalCredentials, amount); } - addValidatorToRegistry(fork, state, pubkey, withdrawalCredentials, amount); } else { - // increase balance by deposit amount - increaseBalance(state, cachedIndex, amount); + if (fork < ForkSeq.electra) { + // increase balance by deposit amount right away pre-electra + increaseBalance(state, cachedIndex, amount); + } else if (fork >= ForkSeq.electra) { + const stateElectra = state as CachedBeaconStateElectra; + const pendingBalanceDeposit = ssz.electra.PendingBalanceDeposit.toViewDU({ + index: cachedIndex, + amount: BigInt(amount), + }); + stateElectra.pendingBalanceDeposits.push(pendingBalanceDeposit); + + if ( + hasCompoundingWithdrawalCredential(withdrawalCredentials) && + hasEth1WithdrawalCredential(validators.getReadonly(cachedIndex).withdrawalCredentials) && + isValidDepositSignature(config, pubkey, withdrawalCredentials, amount, deposit.signature) + ) { + switchToCompoundingValidator(stateElectra, cachedIndex); + } + } } } @@ -93,7 +101,8 @@ function addValidatorToRegistry( ): void { const {validators, epochCtx} = state; // add validator and balance entries - const effectiveBalance = Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); + const effectiveBalance = + fork < ForkSeq.electra ? Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE) : 0; validators.push( ssz.phase0.Validator.toViewDU({ pubkey, @@ -106,9 +115,9 @@ function addValidatorToRegistry( slashed: false, }) ); - state.balances.push(amount); const validatorIndex = validators.length - 1; + // TODO Electra: Review this // Updating here is better than updating at once on epoch transition // - Simplify genesis fn applyDeposits(): effectiveBalanceIncrements is populated immediately // - Keep related code together to reduce risk of breaking this cache @@ -128,4 +137,43 @@ function addValidatorToRegistry( stateAltair.previousEpochParticipation.push(0); stateAltair.currentEpochParticipation.push(0); } + + if (fork < ForkSeq.electra) { + state.balances.push(amount); + } else if (fork >= ForkSeq.electra) { + state.balances.push(0); + const stateElectra = state as CachedBeaconStateElectra; + const pendingBalanceDeposit = ssz.electra.PendingBalanceDeposit.toViewDU({ + index: validatorIndex, + amount: BigInt(amount), + }); + stateElectra.pendingBalanceDeposits.push(pendingBalanceDeposit); + } +} + +function isValidDepositSignature( + config: BeaconConfig, + pubkey: Uint8Array, + withdrawalCredentials: Uint8Array, + amount: number, + depositSignature: Uint8Array +): boolean { + // verify the deposit signature (proof of posession) which is not checked by the deposit contract + const depositMessage = { + pubkey, + withdrawalCredentials, + amount, + }; + // fork-agnostic domain since deposits are valid across forks + const domain = computeDomain(DOMAIN_DEPOSIT, config.GENESIS_FORK_VERSION, ZERO_HASH); + const signingRoot = computeSigningRoot(ssz.phase0.DepositMessage, depositMessage, domain); + try { + // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed + const publicKey = PublicKey.fromBytes(pubkey, true); + const signature = Signature.fromBytes(depositSignature, true); + + return verify(signingRoot, publicKey, signature) + } catch (e) { + return false; // Catch all BLS errors: failed key validation, failed signature validation, invalid signature + } } diff --git a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts index cfe719bf57f3..e16fad105aaa 100644 --- a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts @@ -1,67 +1,99 @@ -import {CompositeViewDU} from "@chainsafe/ssz"; -import {electra, ssz} from "@lodestar/types"; -import {ETH1_ADDRESS_WITHDRAWAL_PREFIX, FAR_FUTURE_EPOCH} from "@lodestar/params"; +import {toHexString} from "@chainsafe/ssz"; +import {electra, phase0, ssz} from "@lodestar/types"; +import { + FAR_FUTURE_EPOCH, + MIN_ACTIVATION_BALANCE, + PENDING_PARTIAL_WITHDRAWALS_LIMIT, + FULL_EXIT_REQUEST_AMOUNT, + ForkSeq, +} from "@lodestar/params"; -import {isActiveValidator} from "../util/index.js"; import {CachedBeaconStateElectra} from "../types.js"; -import {initiateValidatorExit} from "./index.js"; +import {hasCompoundingWithdrawalCredential, hasExecutionWithdrawalCredential} from "../util/electra.js"; +import {getPendingBalanceToWithdraw, isActiveValidator} from "../util/validator.js"; +import {computeExitEpochAndUpdateChurn} from "../util/epoch.js"; +import {initiateValidatorExit} from "./initiateValidatorExit.js"; -const FULL_EXIT_REQUEST_AMOUNT = 0; -/** - * Process execution layer exit messages and initiate exit incase they belong to a valid active validator - * otherwise silent ignore. - */ export function processExecutionLayerWithdrawalRequest( + fork: ForkSeq, state: CachedBeaconStateElectra, - withdrawalRequest: electra.ExecutionLayerWithdrawalRequest + executionLayerWithdrawalRequest: electra.ExecutionLayerWithdrawalRequest ): void { - const isFullExitRequest = withdrawalRequest.amount === FULL_EXIT_REQUEST_AMOUNT; + const amount = Number(executionLayerWithdrawalRequest.amount); + const {pendingPartialWithdrawals, validators, epochCtx} = state; + // no need to use unfinalized pubkey cache from 6110 as validator won't be active anyway + const {pubkey2index, config} = epochCtx; + const isFullExitRequest = amount === FULL_EXIT_REQUEST_AMOUNT; - if (isFullExitRequest) { - const validator = isValidExecutionLayerExit(state, withdrawalRequest); - if (validator === null) { - return; - } - - initiateValidatorExit(state, validator); - } else { - // partial withdral request add codeblock + // If partial withdrawal queue is full, only full exits are processed + if (pendingPartialWithdrawals.length >= PENDING_PARTIAL_WITHDRAWALS_LIMIT && !isFullExitRequest) { + return; } -} -// TODO electra : add pending withdrawal check before exit -export function isValidExecutionLayerExit( - state: CachedBeaconStateElectra, - exit: electra.ExecutionLayerWithdrawalRequest -): CompositeViewDU | null { - const {config, epochCtx} = state; - const validatorIndex = epochCtx.getValidatorIndex(exit.validatorPubkey); - const validator = validatorIndex !== undefined ? state.validators.getReadonly(validatorIndex) : undefined; - if (validator === undefined) { - return null; + // bail out if validator is not in beacon state + // note that we don't need to check for 6110 unfinalized vals as they won't be eligible for withdraw/exit anyway + const validatorIndex = pubkey2index.get(executionLayerWithdrawalRequest.validatorPubkey); + if (validatorIndex === undefined) { + return; } - const {withdrawalCredentials} = validator; - if (withdrawalCredentials[0] !== ETH1_ADDRESS_WITHDRAWAL_PREFIX) { - return null; + const validator = validators.getReadonly(validatorIndex); + if (!isValidatorEligibleForWithdrawOrExit(validator, executionLayerWithdrawalRequest.sourceAddress, state)) { + return; } - const executionAddress = withdrawalCredentials.subarray(12, 32); - if (Buffer.compare(executionAddress, exit.sourceAddress) !== 0) { - return null; + // TODO Electra: Consider caching pendingPartialWithdrawals + const pendingBalanceToWithdraw = getPendingBalanceToWithdraw(state, validatorIndex); + const validatorBalance = state.balances.get(validatorIndex); + + if (isFullExitRequest) { + // only exit validator if it has no pending withdrawals in the queue + if (pendingBalanceToWithdraw === 0) { + initiateValidatorExit(fork, state, validator); + } + return; } - const currentEpoch = epochCtx.epoch; + // partial withdrawal request + const hasSufficientEffectiveBalance = validator.effectiveBalance >= MIN_ACTIVATION_BALANCE; + const hasExcessBalance = validatorBalance > MIN_ACTIVATION_BALANCE + pendingBalanceToWithdraw; + + // Only allow partial withdrawals with compounding withdrawal credentials if ( - // verify the validator is active + hasCompoundingWithdrawalCredential(validator.withdrawalCredentials) && + hasSufficientEffectiveBalance && + hasExcessBalance + ) { + const amountToWithdraw = BigInt( + Math.min(validatorBalance - MIN_ACTIVATION_BALANCE - pendingBalanceToWithdraw, amount) + ); + const exitQueueEpoch = computeExitEpochAndUpdateChurn(state, amountToWithdraw); + const withdrawableEpoch = exitQueueEpoch + config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; + + const pendingPartialWithdrawal = ssz.electra.PartialWithdrawal.toViewDU({ + index: validatorIndex, + amount: amountToWithdraw, + withdrawableEpoch, + }); + state.pendingPartialWithdrawals.push(pendingPartialWithdrawal); + } +} + +function isValidatorEligibleForWithdrawOrExit( + validator: phase0.Validator, + sourceAddress: Uint8Array, + state: CachedBeaconStateElectra +): boolean { + const {withdrawalCredentials} = validator; + const addressStr = toHexString(withdrawalCredentials.subarray(12)); + const sourceAddressStr = toHexString(sourceAddress); + const {epoch: currentEpoch, config} = state.epochCtx; + + return ( + hasExecutionWithdrawalCredential(withdrawalCredentials) && + addressStr === sourceAddressStr && isActiveValidator(validator, currentEpoch) && - // verify exit has not been initiated validator.exitEpoch === FAR_FUTURE_EPOCH && - // verify the validator had been active long enough currentEpoch >= validator.activationEpoch + config.SHARD_COMMITTEE_PERIOD - ) { - return validator; - } else { - return null; - } + ); } diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 228b4eef6fd3..8d65762931e7 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -8,10 +8,11 @@ import {processProposerSlashing} from "./processProposerSlashing.js"; import {processAttesterSlashing} from "./processAttesterSlashing.js"; import {processDeposit} from "./processDeposit.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; -import {processExecutionLayerWithdrawalRequest} from "./processExecutionLayerWithdrawalRequest.js"; import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; +import {processExecutionLayerWithdrawalRequest} from "./processExecutionLayerWithdrawalRequest.js"; import {processDepositReceipt} from "./processDepositReceipt.js"; import {ProcessBlockOpts} from "./types.js"; +import {processConsolidation} from "./processConsolidation.js"; export { processProposerSlashing, @@ -52,12 +53,7 @@ export function processOperations( } for (const voluntaryExit of body.voluntaryExits) { - processVoluntaryExit(state, voluntaryExit, opts.verifySignatures); - } - if (fork >= ForkSeq.electra) { - for (const elWithdrawalRequest of (body as electra.BeaconBlockBody).executionPayload.withdrawalRequests) { - processExecutionLayerWithdrawalRequest(state as CachedBeaconStateElectra, elWithdrawalRequest); - } + processVoluntaryExit(fork, state, voluntaryExit, opts.verifySignatures); } if (fork >= ForkSeq.capella) { @@ -67,8 +63,19 @@ export function processOperations( } if (fork >= ForkSeq.electra) { - for (const depositReceipt of (body as electra.BeaconBlockBody).executionPayload.depositReceipts) { - processDepositReceipt(fork, state as CachedBeaconStateElectra, depositReceipt); + const stateElectra = state as CachedBeaconStateElectra; + const bodyElectra = body as electra.BeaconBlockBody; + + for (const elWithdrawalRequest of bodyElectra.executionPayload.withdrawalRequests) { + processExecutionLayerWithdrawalRequest(fork, state as CachedBeaconStateElectra, elWithdrawalRequest); + } + + for (const depositReceipt of bodyElectra.executionPayload.depositReceipts) { + processDepositReceipt(fork, stateElectra, depositReceipt); + } + + for (const consolidation of bodyElectra.consolidations) { + processConsolidation(stateElectra, consolidation); } } } diff --git a/packages/state-transition/src/block/processVoluntaryExit.ts b/packages/state-transition/src/block/processVoluntaryExit.ts index 80982623a447..b08aa7800884 100644 --- a/packages/state-transition/src/block/processVoluntaryExit.ts +++ b/packages/state-transition/src/block/processVoluntaryExit.ts @@ -1,7 +1,7 @@ -import {FAR_FUTURE_EPOCH} from "@lodestar/params"; +import {FAR_FUTURE_EPOCH, ForkSeq} from "@lodestar/params"; import {phase0} from "@lodestar/types"; -import {isActiveValidator} from "../util/index.js"; -import {CachedBeaconStateAllForks} from "../types.js"; +import {getPendingBalanceToWithdraw, isActiveValidator} from "../util/index.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateElectra} from "../types.js"; import {verifyVoluntaryExitSignature} from "../signatureSets/index.js"; import {initiateValidatorExit} from "./index.js"; @@ -11,16 +11,21 @@ import {initiateValidatorExit} from "./index.js"; * PERF: Work depends on number of VoluntaryExit per block. On regular networks the average is 0 / block. */ export function processVoluntaryExit( + fork: ForkSeq, state: CachedBeaconStateAllForks, signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature = true ): void { - if (!isValidVoluntaryExit(state, signedVoluntaryExit, verifySignature)) { - throw Error("Invalid voluntary exit"); + const isValidExit = + fork >= ForkSeq.electra + ? isValidVoluntaryExitElectra(state as CachedBeaconStateElectra, signedVoluntaryExit, verifySignature) + : isValidVoluntaryExit(state, signedVoluntaryExit, verifySignature); + if (!isValidExit) { + throw Error(`Invalid voluntary exit at forkSeq=${fork}`); } const validator = state.validators.get(signedVoluntaryExit.message.validatorIndex); - initiateValidatorExit(state, validator); + initiateValidatorExit(fork, state, validator); } export function isValidVoluntaryExit( @@ -46,3 +51,16 @@ export function isValidVoluntaryExit( (!verifySignature || verifyVoluntaryExitSignature(state, signedVoluntaryExit)) ); } + +function isValidVoluntaryExitElectra( + state: CachedBeaconStateElectra, + signedVoluntaryExit: phase0.SignedVoluntaryExit, + verifySignature = true +): boolean { + // only exit validator if it has no pending withdrawals in the queue (post-Electra only) + if (getPendingBalanceToWithdraw(state, signedVoluntaryExit.message.validatorIndex) === 0) { + return isValidVoluntaryExit(state, signedVoluntaryExit, verifySignature); + } + + return false; +} diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index 9ae4c570e013..50113ef1741a 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -1,20 +1,30 @@ import {byteArrayEquals} from "@chainsafe/ssz"; import {ssz, capella} from "@lodestar/types"; import { - MAX_EFFECTIVE_BALANCE, MAX_WITHDRAWALS_PER_PAYLOAD, MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP, + ForkSeq, + MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, + FAR_FUTURE_EPOCH, + MIN_ACTIVATION_BALANCE, } from "@lodestar/params"; import {toRootHex} from "@lodestar/utils"; -import {CachedBeaconStateCapella} from "../types.js"; -import {decreaseBalance, hasEth1WithdrawalCredential, isCapellaPayloadHeader} from "../util/index.js"; +import {CachedBeaconStateCapella, CachedBeaconStateElectra} from "../types.js"; +import { + decreaseBalance, + getValidatorMaxEffectiveBalance, + isCapellaPayloadHeader, + isFullyWithdrawableValidator, + isPartiallyWithdrawableValidator, +} from "../util/index.js"; export function processWithdrawals( - state: CachedBeaconStateCapella, + fork: ForkSeq, + state: CachedBeaconStateCapella | CachedBeaconStateElectra, payload: capella.FullOrBlindedExecutionPayload ): void { - const {withdrawals: expectedWithdrawals} = getExpectedWithdrawals(state); + const {withdrawals: expectedWithdrawals, partialWithdrawalsCount} = getExpectedWithdrawals(fork, state); const numWithdrawals = expectedWithdrawals.length; if (isCapellaPayloadHeader(payload)) { @@ -44,6 +54,11 @@ export function processWithdrawals( decreaseBalance(state, withdrawal.validatorIndex, Number(withdrawal.amount)); } + if (fork >= ForkSeq.electra) { + const stateElectra = state as CachedBeaconStateElectra; + stateElectra.pendingPartialWithdrawals = stateElectra.pendingPartialWithdrawals.sliceFrom(partialWithdrawalsCount); + } + // Update the nextWithdrawalIndex if (expectedWithdrawals.length > 0) { const latestWithdrawal = expectedWithdrawals[expectedWithdrawals.length - 1]; @@ -63,46 +78,80 @@ export function processWithdrawals( } } -export function getExpectedWithdrawals(state: CachedBeaconStateCapella): { +export function getExpectedWithdrawals( + fork: ForkSeq, + state: CachedBeaconStateCapella | CachedBeaconStateElectra +): { withdrawals: capella.Withdrawal[]; sampledValidators: number; + partialWithdrawalsCount: number; } { const epoch = state.epochCtx.epoch; let withdrawalIndex = state.nextWithdrawalIndex; const {validators, balances, nextWithdrawalValidatorIndex} = state; - const bound = Math.min(validators.length, MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP); - - let n = 0; const withdrawals: capella.Withdrawal[] = []; + + if (fork >= ForkSeq.electra) { + const stateElectra = state as CachedBeaconStateElectra; + + for (const withdrawal of stateElectra.pendingPartialWithdrawals.getAllReadonly()) { + if (withdrawal.withdrawableEpoch > epoch || withdrawals.length === MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP) { + break; + } + + const validator = validators.getReadonly(withdrawal.index); + + if ( + validator.exitEpoch === FAR_FUTURE_EPOCH && + validator.effectiveBalance >= MIN_ACTIVATION_BALANCE && + balances.get(withdrawalIndex) > MIN_ACTIVATION_BALANCE + ) { + const balanceOverMinActivationBalance = BigInt(balances.get(withdrawalIndex) - MIN_ACTIVATION_BALANCE); + const withdrawableBalance = + balanceOverMinActivationBalance < withdrawal.amount ? balanceOverMinActivationBalance : withdrawal.amount; + withdrawals.push({ + index: withdrawalIndex, + validatorIndex: withdrawal.index, + address: validator.withdrawalCredentials.subarray(12), + amount: withdrawableBalance, + }); + withdrawalIndex++; + } + } + } + + const partialWithdrawalsCount = withdrawals.length; + const bound = Math.min(validators.length, MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP); + let n = 0; // Just run a bounded loop max iterating over all withdrawals // however breaks out once we have MAX_WITHDRAWALS_PER_PAYLOAD for (n = 0; n < bound; n++) { // Get next validator in turn const validatorIndex = (nextWithdrawalValidatorIndex + n) % validators.length; - // It's most likely for validators to not have set eth1 credentials, than having 0 balance const validator = validators.getReadonly(validatorIndex); - if (!hasEth1WithdrawalCredential(validator.withdrawalCredentials)) { + const balance = balances.get(validatorIndex); + // early skip for balance = 0 as its now more likely that validator has exited/slahed with + // balance zero than not have withdrawal credentials set + if (balance === 0) { continue; } - const balance = balances.get(validatorIndex); - - if (balance > 0 && validator.withdrawableEpoch <= epoch) { + if (isFullyWithdrawableValidator(fork, validator, balance, epoch)) { withdrawals.push({ index: withdrawalIndex, validatorIndex, - address: validator.withdrawalCredentials.slice(12), + address: validator.withdrawalCredentials.subarray(12), amount: BigInt(balance), }); withdrawalIndex++; - } else if (validator.effectiveBalance === MAX_EFFECTIVE_BALANCE && balance > MAX_EFFECTIVE_BALANCE) { + } else if (isPartiallyWithdrawableValidator(fork, validator, balance)) { withdrawals.push({ index: withdrawalIndex, validatorIndex, - address: validator.withdrawalCredentials.slice(12), - amount: BigInt(balance - MAX_EFFECTIVE_BALANCE), + address: validator.withdrawalCredentials.subarray(12), + amount: BigInt(balance - getValidatorMaxEffectiveBalance(validator.withdrawalCredentials)), }); withdrawalIndex++; } @@ -113,5 +162,5 @@ export function getExpectedWithdrawals(state: CachedBeaconStateCapella): { } } - return {withdrawals, sampledValidators: n}; + return {withdrawals, sampledValidators: n, partialWithdrawalsCount}; } diff --git a/packages/state-transition/src/block/slashValidator.ts b/packages/state-transition/src/block/slashValidator.ts index 9f3eb2947644..c4b7d5f848ea 100644 --- a/packages/state-transition/src/block/slashValidator.ts +++ b/packages/state-transition/src/block/slashValidator.ts @@ -6,11 +6,13 @@ import { MIN_SLASHING_PENALTY_QUOTIENT, MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR, MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX, + MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA, PROPOSER_REWARD_QUOTIENT, PROPOSER_WEIGHT, TIMELY_TARGET_FLAG_INDEX, WEIGHT_DENOMINATOR, WHISTLEBLOWER_REWARD_QUOTIENT, + WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA, } from "@lodestar/params"; import {decreaseBalance, increaseBalance} from "../util/index.js"; @@ -31,7 +33,7 @@ export function slashValidator( const validator = state.validators.get(slashedIndex); // TODO: Bellatrix initiateValidatorExit validators.update() with the one below - initiateValidatorExit(state, validator); + initiateValidatorExit(fork, state, validator); validator.slashed = true; validator.withdrawableEpoch = Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR); @@ -41,7 +43,7 @@ export function slashValidator( // state.slashings is initially a Gwei (BigInt) vector, however since Nov 2023 it's converted to UintNum64 (number) vector in the state transition because: // - state.slashings[nextEpoch % EPOCHS_PER_SLASHINGS_VECTOR] is reset per epoch in processSlashingsReset() // - max slashed validators per epoch is SLOTS_PER_EPOCH * MAX_ATTESTER_SLASHINGS * MAX_VALIDATORS_PER_COMMITTEE which is 32 * 2 * 2048 = 131072 on mainnet - // - with that and 32_000_000_000 MAX_EFFECTIVE_BALANCE, it still fits in a number given that Math.floor(Number.MAX_SAFE_INTEGER / 32_000_000_000) = 281474 + // - with that and 32_000_000_000 MAX_EFFECTIVE_BALANCE or 2048_000_000_000 MAX_EFFECTIVE_BALANCE_ELECTRA, it still fits in a number given that Math.floor(Number.MAX_SAFE_INTEGER / 32_000_000_000) = 281474 // - we don't need to compute the total slashings from state.slashings, it's handled by totalSlashingsByIncrement in EpochCache const slashingIndex = epoch % EPOCHS_PER_SLASHINGS_VECTOR; state.slashings.set(slashingIndex, (state.slashings.get(slashingIndex) ?? 0) + effectiveBalance); @@ -52,11 +54,16 @@ export function slashValidator( ? MIN_SLASHING_PENALTY_QUOTIENT : fork === ForkSeq.altair ? MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR - : MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX; + : fork < ForkSeq.electra // no change from bellatrix to deneb + ? MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX + : MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA; decreaseBalance(state, slashedIndex, Math.floor(effectiveBalance / minSlashingPenaltyQuotient)); // apply proposer and whistleblower rewards - const whistleblowerReward = Math.floor(effectiveBalance / WHISTLEBLOWER_REWARD_QUOTIENT); + const whistleblowerReward = + fork < ForkSeq.electra + ? Math.floor(effectiveBalance / WHISTLEBLOWER_REWARD_QUOTIENT) + : Math.floor(effectiveBalance / WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA); const proposerReward = fork === ForkSeq.phase0 ? Math.floor(whistleblowerReward / PROPOSER_REWARD_QUOTIENT) diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index a7433e848a44..538f00dd91c7 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -180,6 +180,7 @@ export class EpochCache { * initiateValidatorExit(). This value may vary on each fork of the state. * * NOTE: Changes block to block + * NOTE: No longer used by initiateValidatorExit post-electra */ exitQueueEpoch: Epoch; /** @@ -187,6 +188,7 @@ export class EpochCache { * initiateValidatorExit(). This value may vary on each fork of the state. * * NOTE: Changes block to block + * NOTE: No longer used by initiateValidatorExit post-electra */ exitQueueChurn: number; diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index e6f84de6c62e..280524ec7beb 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,6 +1,6 @@ import {Epoch, ValidatorIndex} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; -import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; +import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; import { hasMarkers, @@ -78,7 +78,7 @@ export interface EpochTransitionCache { /** * Indices of validators that just joined and will be eligible for the active queue. * ``` - * v.activationEligibilityEpoch === FAR_FUTURE_EPOCH && v.effectiveBalance === MAX_EFFECTIVE_BALANCE + * v.activationEligibilityEpoch === FAR_FUTURE_EPOCH && v.effectiveBalance >= MAX_EFFECTIVE_BALANCE * ``` * All validators in indicesEligibleForActivationQueue get activationEligibilityEpoch set. So it can only include * validators that have just joined the registry through a valid full deposit(s). @@ -297,12 +297,12 @@ export function beforeProcessEpoch( // def is_eligible_for_activation_queue(validator: Validator) -> bool: // return ( // validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH - // and validator.effective_balance == MAX_EFFECTIVE_BALANCE + // and validator.effective_balance >= MAX_EFFECTIVE_BALANCE # [Modified in Electra] // ) // ``` if ( validator.activationEligibilityEpoch === FAR_FUTURE_EPOCH && - validator.effectiveBalance === MAX_EFFECTIVE_BALANCE + validator.effectiveBalance >= MIN_ACTIVATION_BALANCE ) { indicesEligibleForActivationQueue.push(i); } diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index b55ebe291fb9..6e736fdae2cc 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -11,6 +11,7 @@ import { CachedBeaconStateAltair, CachedBeaconStatePhase0, EpochTransitionCache, + CachedBeaconStateElectra, } from "../types.js"; import {BeaconStateTransitionMetrics} from "../metrics.js"; import {processEffectiveBalanceUpdates} from "./processEffectiveBalanceUpdates.js"; @@ -27,6 +28,8 @@ import {processRewardsAndPenalties} from "./processRewardsAndPenalties.js"; import {processSlashings} from "./processSlashings.js"; import {processSlashingsReset} from "./processSlashingsReset.js"; import {processSyncCommitteeUpdates} from "./processSyncCommitteeUpdates.js"; +import {processPendingBalanceDeposits} from "./processPendingBalanceDeposits.js"; +import {processPendingConsolidations} from "./processPendingConsolidations.js"; // For spec tests export {getRewardsAndPenalties} from "./processRewardsAndPenalties.js"; @@ -45,6 +48,8 @@ export { processParticipationFlagUpdates, processSyncCommitteeUpdates, processHistoricalSummariesUpdate, + processPendingBalanceDeposits, + processPendingConsolidations, }; export {computeUnrealizedCheckpoints} from "./computeUnrealizedCheckpoints.js"; @@ -65,6 +70,8 @@ export enum EpochTransitionStep { processEffectiveBalanceUpdates = "processEffectiveBalanceUpdates", processParticipationFlagUpdates = "processParticipationFlagUpdates", processSyncCommitteeUpdates = "processSyncCommitteeUpdates", + processPendingBalanceDeposits = "processPendingBalanceDeposits", + processPendingConsolidations = "processPendingConsolidations", } export function processEpoch( @@ -76,7 +83,7 @@ export function processEpoch( // state.slashings is initially a Gwei (BigInt) vector, however since Nov 2023 it's converted to UintNum64 (number) vector in the state transition because: // - state.slashings[nextEpoch % EPOCHS_PER_SLASHINGS_VECTOR] is reset per epoch in processSlashingsReset() // - max slashed validators per epoch is SLOTS_PER_EPOCH * MAX_ATTESTER_SLASHINGS * MAX_VALIDATORS_PER_COMMITTEE which is 32 * 2 * 2048 = 131072 on mainnet - // - with that and 32_000_000_000 MAX_EFFECTIVE_BALANCE, it still fits in a number given that Math.floor(Number.MAX_SAFE_INTEGER / 32_000_000_000) = 281474 + // - with that and 32_000_000_000 MAX_EFFECTIVE_BALANCE or 2048_000_000_000 MAX_EFFECTIVE_BALANCE_ELECTRA, it still fits in a number given that Math.floor(Number.MAX_SAFE_INTEGER / 32_000_000_000) = 281474 if (maxValidatorsPerStateSlashing > maxSafeValidators) { throw new Error("Lodestar does not support this network, parameters don't fit number value inside state.slashings"); } @@ -100,7 +107,7 @@ export function processEpoch( // processRewardsAndPenalties(state, cache); { const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.processRegistryUpdates}); - processRegistryUpdates(state, cache); + processRegistryUpdates(fork, state, cache); timer?.(); } @@ -120,11 +127,30 @@ export function processEpoch( processEth1DataReset(state, cache); + if (fork >= ForkSeq.electra) { + const stateElectra = state as CachedBeaconStateElectra; + { + const timer = metrics?.epochTransitionStepTime.startTimer({ + step: EpochTransitionStep.processPendingBalanceDeposits, + }); + processPendingBalanceDeposits(stateElectra); + timer?.(); + } + + { + const timer = metrics?.epochTransitionStepTime.startTimer({ + step: EpochTransitionStep.processPendingConsolidations, + }); + processPendingConsolidations(stateElectra); + timer?.(); + } + } + { const timer = metrics?.epochTransitionStepTime.startTimer({ step: EpochTransitionStep.processEffectiveBalanceUpdates, }); - processEffectiveBalanceUpdates(state, cache); + processEffectiveBalanceUpdates(fork, state, cache); timer?.(); } diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index 5f1df35b7215..e83d2545ef50 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -5,9 +5,12 @@ import { HYSTERESIS_QUOTIENT, HYSTERESIS_UPWARD_MULTIPLIER, MAX_EFFECTIVE_BALANCE, + MAX_EFFECTIVE_BALANCE_ELECTRA, + MIN_ACTIVATION_BALANCE, TIMELY_TARGET_FLAG_INDEX, } from "@lodestar/params"; import {EpochTransitionCache, CachedBeaconStateAllForks, BeaconStateAltair} from "../types.js"; +import {hasCompoundingWithdrawalCredential} from "../util/electra.js"; /** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; @@ -21,7 +24,11 @@ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; * - On normal mainnet conditions 0 validators change their effective balance * - In case of big innactivity event a medium portion of validators may have their effectiveBalance updated */ -export function processEffectiveBalanceUpdates(state: CachedBeaconStateAllForks, cache: EpochTransitionCache): void { +export function processEffectiveBalanceUpdates( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + cache: EpochTransitionCache +): void { const HYSTERESIS_INCREMENT = EFFECTIVE_BALANCE_INCREMENT / HYSTERESIS_QUOTIENT; const DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER; const UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER; @@ -42,6 +49,7 @@ export function processEffectiveBalanceUpdates(state: CachedBeaconStateAllForks, // PERF: It's faster to access to get() every single element (4ms) than to convert to regular array then loop (9ms) let effectiveBalanceIncrement = effectiveBalanceIncrements[i]; let effectiveBalance = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT; + let effectiveBalanceLimit; if ( // Too big @@ -49,10 +57,20 @@ export function processEffectiveBalanceUpdates(state: CachedBeaconStateAllForks, // Too small. Check effectiveBalance < MAX_EFFECTIVE_BALANCE to prevent unnecessary updates (effectiveBalance < MAX_EFFECTIVE_BALANCE && effectiveBalance < balance - UPWARD_THRESHOLD) ) { - effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); // Update the state tree // Should happen rarely, so it's fine to update the tree const validator = validators.get(i); + + if (fork < ForkSeq.electra) { + effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; + } else { + // Electra or after + effectiveBalanceLimit = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials) + ? MAX_EFFECTIVE_BALANCE_ELECTRA + : MIN_ACTIVATION_BALANCE; + } + + effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), effectiveBalanceLimit); validator.effectiveBalance = effectiveBalance; // Also update the fast cached version const newEffectiveBalanceIncrement = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); diff --git a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts new file mode 100644 index 000000000000..95928d66df2d --- /dev/null +++ b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts @@ -0,0 +1,35 @@ +import {CachedBeaconStateElectra} from "../types.js"; +import {increaseBalance} from "../util/balance.js"; +import {getActivationExitChurnLimit} from "../util/validator.js"; + +/** + * Starting from Electra: + * Process pending balance deposits from state subject to churn limit and depsoitBalanceToConsume. + * For each eligible `deposit`, call `increaseBalance()`. + * Remove the processed deposits from `state.pendingBalanceDeposits`. + * Update `state.depositBalanceToConsume` for the next epoch + */ +export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): void { + const availableForProcessing = state.depositBalanceToConsume + BigInt(getActivationExitChurnLimit(state)); + let processedAmount = 0n; + let nextDepositIndex = 0; + + for (const deposit of state.pendingBalanceDeposits.getAllReadonly()) { + const {amount} = deposit; + if (processedAmount + amount > availableForProcessing) { + break; + } + increaseBalance(state, deposit.index, Number(amount)); + processedAmount = processedAmount + amount; + nextDepositIndex++; + } + + const remainingPendingBalanceDeposits = state.pendingBalanceDeposits.sliceFrom(nextDepositIndex); + state.pendingBalanceDeposits = remainingPendingBalanceDeposits; + + if (remainingPendingBalanceDeposits.length === 0) { + state.depositBalanceToConsume = 0n; + } else { + state.depositBalanceToConsume = availableForProcessing - processedAmount; + } +} diff --git a/packages/state-transition/src/epoch/processPendingConsolidations.ts b/packages/state-transition/src/epoch/processPendingConsolidations.ts new file mode 100644 index 000000000000..e025a454f64e --- /dev/null +++ b/packages/state-transition/src/epoch/processPendingConsolidations.ts @@ -0,0 +1,45 @@ +import {CachedBeaconStateElectra} from "../types.js"; +import {decreaseBalance, increaseBalance} from "../util/balance.js"; +import {getActiveBalance} from "../util/validator.js"; +import {switchToCompoundingValidator} from "../util/electra.js"; + +/** + * Starting from Electra: + * Process every `pendingConsolidation` in `state.pendingConsolidations`. + * Churn limit was applied when enqueueing so we don't care about the limit here + * However we only process consolidations up to current epoch + * + * For each valid `pendingConsolidation`, update withdrawal credential of target + * validator to compounding, decrease balance of source validator and increase balance + * of target validator. + * + * Dequeue all processed consolidations from `state.pendingConsolidation` + * + */ +export function processPendingConsolidations(state: CachedBeaconStateElectra): void { + let nextPendingConsolidation = 0; + + for (const pendingConsolidation of state.pendingConsolidations.getAllReadonly()) { + const {sourceIndex, targetIndex} = pendingConsolidation; + const sourceValidator = state.validators.getReadonly(sourceIndex); + + if (sourceValidator.slashed) { + nextPendingConsolidation++; + continue; + } + + if (sourceValidator.withdrawableEpoch > state.epochCtx.epoch) { + break; + } + // Churn any target excess active balance of target and raise its max + switchToCompoundingValidator(state, targetIndex); + // Move active balance to target. Excess balance is withdrawable. + const activeBalance = getActiveBalance(state, sourceIndex); + decreaseBalance(state, sourceIndex, activeBalance); + increaseBalance(state, targetIndex, activeBalance); + + nextPendingConsolidation++; + } + + state.pendingConsolidations = state.pendingConsolidations.sliceFrom(nextPendingConsolidation); +} diff --git a/packages/state-transition/src/epoch/processRegistryUpdates.ts b/packages/state-transition/src/epoch/processRegistryUpdates.ts index 905e7b567c01..d2e93632dabe 100644 --- a/packages/state-transition/src/epoch/processRegistryUpdates.ts +++ b/packages/state-transition/src/epoch/processRegistryUpdates.ts @@ -1,3 +1,4 @@ +import {ForkSeq} from "@lodestar/params"; import {computeActivationExitEpoch} from "../util/index.js"; import {initiateValidatorExit} from "../block/index.js"; import {EpochTransitionCache, CachedBeaconStateAllForks} from "../types.js"; @@ -16,7 +17,11 @@ import {EpochTransitionCache, CachedBeaconStateAllForks} from "../types.js"; * - indicesEligibleForActivationQueue: 0 * - indicesToEject: 0 */ -export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache: EpochTransitionCache): void { +export function processRegistryUpdates( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + cache: EpochTransitionCache +): void { const {epochCtx} = state; // Get the validators sub tree once for all the loop @@ -28,7 +33,7 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache: for (const index of cache.indicesToEject) { // set validator exit epoch and withdrawable epoch // TODO: Figure out a way to quickly set properties on the validators tree - initiateValidatorExit(state, validators.get(index)); + initiateValidatorExit(fork, state, validators.get(index)); } // set new activation eligibilities @@ -38,7 +43,10 @@ export function processRegistryUpdates(state: CachedBeaconStateAllForks, cache: const finalityEpoch = state.finalizedCheckpoint.epoch; // this avoids an array allocation compared to `slice(0, epochCtx.activationChurnLimit)` - const len = Math.min(cache.indicesEligibleForActivation.length, epochCtx.activationChurnLimit); + const len = + fork < ForkSeq.electra + ? Math.min(cache.indicesEligibleForActivation.length, epochCtx.activationChurnLimit) + : cache.indicesEligibleForActivation.length; const activationEpoch = computeActivationExitEpoch(cache.currentEpoch); // dequeue validators for activation up to churn limit for (let i = 0; i < len; i++) { diff --git a/packages/state-transition/src/signatureSets/consolidation.ts b/packages/state-transition/src/signatureSets/consolidation.ts new file mode 100644 index 000000000000..0aef47d417a5 --- /dev/null +++ b/packages/state-transition/src/signatureSets/consolidation.ts @@ -0,0 +1,51 @@ +import {DOMAIN_CONSOLIDATION, ForkName} from "@lodestar/params"; +import {electra, ssz} from "@lodestar/types"; + +import { + computeSigningRoot, + createAggregateSignatureSetFromComponents, + ISignatureSet, + verifySignatureSet, +} from "../util/index.js"; +import {CachedBeaconStateElectra} from "../types.js"; + +export function verifyConsolidationSignature( + state: CachedBeaconStateElectra, + signedConsolidation: electra.SignedConsolidation +): boolean { + return verifySignatureSet(getConsolidationSignatureSet(state, signedConsolidation)); +} + +/** + * Extract signatures to allow validating all block signatures at once + */ +export function getConsolidationSignatureSet( + state: CachedBeaconStateElectra, + signedConsolidation: electra.SignedConsolidation +): ISignatureSet { + const {config} = state; + const {index2pubkey} = state.epochCtx; // TODO Electra: Use 6110 pubkey cache + const {sourceIndex, targetIndex} = signedConsolidation.message; + const sourcePubkey = index2pubkey[sourceIndex]; + const targetPubkey = index2pubkey[targetIndex]; + + // signatureFork for signing domain is fixed + const signatureFork = ForkName.phase0; + const domain = config.getDomainAtFork(signatureFork, DOMAIN_CONSOLIDATION); + const signingRoot = computeSigningRoot(ssz.electra.Consolidation, signedConsolidation.message, domain); + + return createAggregateSignatureSetFromComponents( + [sourcePubkey, targetPubkey], + signingRoot, + signedConsolidation.signature + ); +} + +export function getConsolidationSignatureSets( + state: CachedBeaconStateElectra, + signedBlock: electra.SignedBeaconBlock +): ISignatureSet[] { + return signedBlock.message.body.consolidations.map((consolidation) => + getConsolidationSignatureSet(state, consolidation) + ); +} diff --git a/packages/state-transition/src/signatureSets/index.ts b/packages/state-transition/src/signatureSets/index.ts index 983e131e00e6..140b607c0bcb 100644 --- a/packages/state-transition/src/signatureSets/index.ts +++ b/packages/state-transition/src/signatureSets/index.ts @@ -1,7 +1,7 @@ import {ForkSeq} from "@lodestar/params"; -import {SignedBeaconBlock, altair, capella} from "@lodestar/types"; +import {SignedBeaconBlock, altair, capella, electra} from "@lodestar/types"; import {ISignatureSet} from "../util/index.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStateElectra} from "../types.js"; import {getSyncCommitteeSignatureSet} from "../block/processSyncCommittee.js"; import {getProposerSlashingsSignatureSets} from "./proposerSlashings.js"; import {getAttesterSlashingsSignatureSets} from "./attesterSlashings.js"; @@ -10,6 +10,7 @@ import {getBlockProposerSignatureSet} from "./proposer.js"; import {getRandaoRevealSignatureSet} from "./randao.js"; import {getVoluntaryExitsSignatureSets} from "./voluntaryExits.js"; import {getBlsToExecutionChangeSignatureSets} from "./blsToExecutionChange.js"; +import {getConsolidationSignatureSets} from "./consolidation.js"; export * from "./attesterSlashings.js"; export * from "./indexedAttestation.js"; @@ -18,6 +19,7 @@ export * from "./proposerSlashings.js"; export * from "./randao.js"; export * from "./voluntaryExits.js"; export * from "./blsToExecutionChange.js"; +export * from "./consolidation.js"; /** * Includes all signatures on the block (except the deposit signatures) for verification. @@ -69,5 +71,15 @@ export function getBlockSignatureSets( } } + if (fork >= ForkSeq.electra) { + const consolidationSignatureSets = getConsolidationSignatureSets( + state as CachedBeaconStateElectra, + signedBlock as electra.SignedBeaconBlock + ); + if (consolidationSignatureSets.length > 0) { + signatureSets.push(...consolidationSignatureSets); + } + } + return signatureSets; } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index f41c37af94aa..760595c3a86b 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -1,7 +1,12 @@ import {ssz} from "@lodestar/types"; -import {UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {FAR_FUTURE_EPOCH, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; import {CachedBeaconStateDeneb} from "../types.js"; import {CachedBeaconStateElectra, getCachedBeaconState} from "../cache/stateCache.js"; +import { + hasCompoundingWithdrawalCredential, + queueEntireBalanceAndResetValidator, + queueExcessActiveBalance, +} from "../util/electra.js"; /** * Upgrade a state from Capella to Deneb. @@ -24,6 +29,23 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + const validatorsArr = stateElectra.validators.getAllReadonly(); + + for (let i = 0; i < validatorsArr.length; i++) { + const validator = validatorsArr[i]; + + // [EIP-7251]: add validators that are not yet active to pending balance deposits + if (validator.activationEligibilityEpoch === FAR_FUTURE_EPOCH) { + queueEntireBalanceAndResetValidator(stateElectra, i); + } + + // [EIP-7251]: Ensure early adopters of compounding credentials go through the activation churn + const withdrawalCredential = validator.withdrawalCredentials; + if (hasCompoundingWithdrawalCredential(withdrawalCredential)) { + queueExcessActiveBalance(stateElectra, i); + } + } + // Commit new added fields ViewDU to the root node stateElectra.commit(); // Clear cache to ensure the cache of deneb fields is not used by new ELECTRA fields diff --git a/packages/state-transition/src/util/electra.ts b/packages/state-transition/src/util/electra.ts new file mode 100644 index 000000000000..00c5bb183eb4 --- /dev/null +++ b/packages/state-transition/src/util/electra.ts @@ -0,0 +1,103 @@ +import { + COMPOUNDING_WITHDRAWAL_PREFIX, + FAR_FUTURE_EPOCH, + ForkSeq, + MAX_EFFECTIVE_BALANCE, + MIN_ACTIVATION_BALANCE, +} from "@lodestar/params"; +import {ValidatorIndex, phase0, ssz} from "@lodestar/types"; +import {CachedBeaconStateElectra} from "../types.js"; +import {getValidatorMaxEffectiveBalance} from "./validator.js"; +import {hasEth1WithdrawalCredential} from "./capella.js"; + +type ValidatorInfo = Pick; + +export function hasCompoundingWithdrawalCredential(withdrawalCredentials: Uint8Array): boolean { + return withdrawalCredentials[0] === COMPOUNDING_WITHDRAWAL_PREFIX; +} + +export function hasExecutionWithdrawalCredential(withdrawalCredentials: Uint8Array): boolean { + return ( + hasCompoundingWithdrawalCredential(withdrawalCredentials) || hasEth1WithdrawalCredential(withdrawalCredentials) + ); +} + +export function isFullyWithdrawableValidator( + fork: ForkSeq, + validatorCredential: ValidatorInfo, + balance: number, + epoch: number +): boolean { + const {withdrawableEpoch, withdrawalCredentials: withdrawalCredential} = validatorCredential; + + if (fork < ForkSeq.capella) { + throw new Error(`isFullyWithdrawableValidator not supported at forkSeq=${fork} < ForkSeq.capella`); + } + const hasWithdrawableCredentials = + fork >= ForkSeq.electra + ? hasExecutionWithdrawalCredential(withdrawalCredential) + : hasEth1WithdrawalCredential(withdrawalCredential); + + return hasWithdrawableCredentials && withdrawableEpoch <= epoch && balance > 0; +} + +export function isPartiallyWithdrawableValidator( + fork: ForkSeq, + validatorCredential: ValidatorInfo, + balance: number +): boolean { + const {effectiveBalance, withdrawalCredentials: withdrawalCredential} = validatorCredential; + + if (fork < ForkSeq.capella) { + throw new Error(`isPartiallyWithdrawableValidator not supported at forkSeq=${fork} < ForkSeq.capella`); + } + const hasWithdrawableCredentials = + fork >= ForkSeq.electra + ? hasExecutionWithdrawalCredential(withdrawalCredential) + : hasEth1WithdrawalCredential(withdrawalCredential); + + const validatorMaxEffectiveBalance = + fork >= ForkSeq.electra ? getValidatorMaxEffectiveBalance(withdrawalCredential) : MAX_EFFECTIVE_BALANCE; + const hasMaxEffectiveBalance = effectiveBalance === validatorMaxEffectiveBalance; + const hasExcessBalance = balance > validatorMaxEffectiveBalance; + + return hasWithdrawableCredentials && hasMaxEffectiveBalance && hasExcessBalance; +} + +export function switchToCompoundingValidator(state: CachedBeaconStateElectra, index: ValidatorIndex): void { + const validator = state.validators.get(index); + + if (hasEth1WithdrawalCredential(validator.withdrawalCredentials)) { + validator.withdrawalCredentials[0] = COMPOUNDING_WITHDRAWAL_PREFIX; + queueExcessActiveBalance(state, index); + } +} + +export function queueExcessActiveBalance(state: CachedBeaconStateElectra, index: ValidatorIndex): void { + const balance = state.balances.get(index); + if (balance > MIN_ACTIVATION_BALANCE) { + const excessBalance = balance - MIN_ACTIVATION_BALANCE; + state.balances.set(index, MIN_ACTIVATION_BALANCE); + + const pendingBalanceDeposit = ssz.electra.PendingBalanceDeposit.toViewDU({ + index, + amount: BigInt(excessBalance), + }); + state.pendingBalanceDeposits.push(pendingBalanceDeposit); + } +} + +export function queueEntireBalanceAndResetValidator(state: CachedBeaconStateElectra, index: ValidatorIndex): void { + const balance = state.balances.get(index); + state.balances.set(index, 0); + + const validator = state.validators.get(index); + validator.effectiveBalance = 0; + validator.activationEligibilityEpoch = FAR_FUTURE_EPOCH; + + const pendingBalanceDeposit = ssz.electra.PendingBalanceDeposit.toViewDU({ + index, + amount: BigInt(balance), + }); + state.pendingBalanceDeposits.push(pendingBalanceDeposit); +} diff --git a/packages/state-transition/src/util/epoch.ts b/packages/state-transition/src/util/epoch.ts index ba182f627de0..3ac77556e3cb 100644 --- a/packages/state-transition/src/util/epoch.ts +++ b/packages/state-transition/src/util/epoch.ts @@ -1,5 +1,7 @@ import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, GENESIS_EPOCH, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {BeaconState, Epoch, Slot, SyncPeriod} from "@lodestar/types"; +import {CachedBeaconStateElectra} from "../types.js"; +import {getActivationExitChurnLimit, getConsolidationChurnLimit} from "./validator.js"; /** * Return the epoch number at the given slot. @@ -39,6 +41,60 @@ export function computeActivationExitEpoch(epoch: Epoch): Epoch { return epoch + 1 + MAX_SEED_LOOKAHEAD; } +export function computeExitEpochAndUpdateChurn(state: CachedBeaconStateElectra, exitBalance: Gwei): number { + let earliestExitEpoch = Math.max(state.earliestExitEpoch, computeActivationExitEpoch(state.epochCtx.epoch)); + const perEpochChurn = getActivationExitChurnLimit(state); + + // New epoch for exits. + let exitBalanceToConsume = + state.earliestExitEpoch < earliestExitEpoch ? perEpochChurn : Number(state.exitBalanceToConsume); + + // Exit doesn't fit in the current earliest epoch. + if (exitBalance > exitBalanceToConsume) { + const balanceToProcess = Number(exitBalance) - exitBalanceToConsume; + const additionalEpochs = Math.floor((balanceToProcess - 1) / (perEpochChurn + 1)); + earliestExitEpoch += additionalEpochs; + exitBalanceToConsume += additionalEpochs * perEpochChurn; + } + + // Consume the balance and update state variables. + state.exitBalanceToConsume = BigInt(exitBalanceToConsume) - exitBalance; + state.earliestExitEpoch = earliestExitEpoch; + + return state.earliestExitEpoch; +} + +export function computeConsolidationEpochAndUpdateChurn( + state: CachedBeaconStateElectra, + consolidationBalance: Gwei +): number { + let earliestConsolidationEpoch = Math.max( + state.earliestConsolidationEpoch, + computeActivationExitEpoch(state.epochCtx.epoch) + ); + const perEpochConsolidationChurn = getConsolidationChurnLimit(state); + + // New epoch for consolidations + let consolidationBalanceToConsume = + state.earliestConsolidationEpoch < earliestConsolidationEpoch + ? perEpochConsolidationChurn + : Number(state.consolidationBalanceToConsume); + + // Consolidation doesn't fit in the current earliest epoch. + if (consolidationBalance > consolidationBalanceToConsume) { + const balanceToProcess = Number(consolidationBalance) - consolidationBalanceToConsume; + const additionalEpochs = Math.floor((balanceToProcess - 1) / (perEpochConsolidationChurn + 1)); + earliestConsolidationEpoch += additionalEpochs; + consolidationBalanceToConsume += additionalEpochs * perEpochConsolidationChurn; + } + + // Consume the balance and update state variables. + state.consolidationBalanceToConsume = BigInt(consolidationBalanceToConsume) - consolidationBalance; + state.earliestConsolidationEpoch = earliestConsolidationEpoch; + + return state.earliestConsolidationEpoch; +} + /** * Return the current epoch of the given state. */ diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 6c700436d7f6..537edc7f976f 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -185,7 +185,7 @@ export function applyDeposits( validator.effectiveBalance = effectiveBalance; epochCtx.effectiveBalanceIncrementsSet(i, effectiveBalance); - if (validator.effectiveBalance === MAX_EFFECTIVE_BALANCE) { + if (validator.effectiveBalance >= MAX_EFFECTIVE_BALANCE) { validator.activationEligibilityEpoch = GENESIS_EPOCH; validator.activationEpoch = GENESIS_EPOCH; activatedValidatorCount++; diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index ba998f65b254..6a839fbe103d 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -24,3 +24,4 @@ export * from "./syncCommittee.js"; export * from "./validator.js"; export * from "./weakSubjectivity.js"; export * from "./deposit.js"; +export * from "./electra.js"; diff --git a/packages/state-transition/src/util/validator.ts b/packages/state-transition/src/util/validator.ts index 99f1e6fa0b19..0c8c74b2cb03 100644 --- a/packages/state-transition/src/util/validator.ts +++ b/packages/state-transition/src/util/validator.ts @@ -1,8 +1,14 @@ import {Epoch, phase0, ValidatorIndex} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; import {ChainForkConfig} from "@lodestar/config"; -import {ForkSeq} from "@lodestar/params"; -import {BeaconStateAllForks} from "../types.js"; +import { + EFFECTIVE_BALANCE_INCREMENT, + ForkSeq, + MAX_EFFECTIVE_BALANCE_ELECTRA, + MIN_ACTIVATION_BALANCE, +} from "@lodestar/params"; +import {BeaconStateAllForks, CachedBeaconStateElectra} from "../types.js"; +import {hasCompoundingWithdrawalCredential} from "./electra.js"; /** * Check if [[validator]] is active @@ -47,3 +53,48 @@ export function getActivationChurnLimit(config: ChainForkConfig, fork: ForkSeq, export function getChurnLimit(config: ChainForkConfig, activeValidatorCount: number): number { return Math.max(config.MIN_PER_EPOCH_CHURN_LIMIT, intDiv(activeValidatorCount, config.CHURN_LIMIT_QUOTIENT)); } + +/** + * Get combined churn limit of activation-exit and consolidation + */ +export function getBalanceChurnLimit(state: CachedBeaconStateElectra): number { + const churnLimitByTotalActiveBalance = Math.floor( + (state.epochCtx.totalActiveBalanceIncrements / state.config.CHURN_LIMIT_QUOTIENT) * EFFECTIVE_BALANCE_INCREMENT + ); // TODO Electra: verify calculation + + const churn = Math.max(churnLimitByTotalActiveBalance, state.config.MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA); + + return churn - (churn % EFFECTIVE_BALANCE_INCREMENT); +} + +export function getActivationExitChurnLimit(state: CachedBeaconStateElectra): number { + return Math.min(state.config.MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT, getBalanceChurnLimit(state)); +} + +export function getConsolidationChurnLimit(state: CachedBeaconStateElectra): number { + return getBalanceChurnLimit(state) - getActivationExitChurnLimit(state); +} + +export function getValidatorMaxEffectiveBalance(withdrawalCredentials: Uint8Array): number { + // Compounding withdrawal credential only available since Electra + if (hasCompoundingWithdrawalCredential(withdrawalCredentials)) { + return MAX_EFFECTIVE_BALANCE_ELECTRA; + } else { + return MIN_ACTIVATION_BALANCE; + } +} + +export function getActiveBalance(state: CachedBeaconStateElectra, validatorIndex: ValidatorIndex): number { + const validatorMaxEffectiveBalance = getValidatorMaxEffectiveBalance( + state.validators.getReadonly(validatorIndex).withdrawalCredentials + ); + + return Math.min(state.balances.get(validatorIndex), validatorMaxEffectiveBalance); +} + +export function getPendingBalanceToWithdraw(state: CachedBeaconStateElectra, validatorIndex: ValidatorIndex): number { + return state.pendingPartialWithdrawals + .getAllReadonly() + .filter((item) => item.index === validatorIndex) + .reduce((total, item) => total + Number(item.amount), 0); +} diff --git a/packages/state-transition/test/perf/analyzeEpochs.ts b/packages/state-transition/test/perf/analyzeEpochs.ts index 6f61bc81abbc..deb0861427bf 100644 --- a/packages/state-transition/test/perf/analyzeEpochs.ts +++ b/packages/state-transition/test/perf/analyzeEpochs.ts @@ -152,6 +152,9 @@ async function analyzeEpochs(network: NetworkName, fromEpoch?: number): Promise< // processRegistryUpdates: function of registry updates // processSlashingsAllForks: function of process.indicesToSlash // processSlashingsReset: free + // -- electra + // processPendingBalanceDeposits: - + // processPendingConsolidations: - // -- altair // processInactivityUpdates: - // processParticipationFlagUpdates: - diff --git a/packages/state-transition/test/perf/block/processWithdrawals.test.ts b/packages/state-transition/test/perf/block/processWithdrawals.test.ts index 997f401d32ce..66d624b39bfd 100644 --- a/packages/state-transition/test/perf/block/processWithdrawals.test.ts +++ b/packages/state-transition/test/perf/block/processWithdrawals.test.ts @@ -1,4 +1,5 @@ import {itBench} from "@dapplion/benchmark"; +import {ForkSeq} from "@lodestar/params"; import {CachedBeaconStateCapella} from "../../../src/index.js"; import {getExpectedWithdrawals} from "../../../src/block/processWithdrawals.js"; import {numValidators} from "../util.js"; @@ -9,7 +10,7 @@ import {getExpectedWithdrawalsTestData, WithdrawalOpts} from "../../utils/capell // having BLS withdrawal credential prefix as that validator probe is wasted. // // Best case: -// All Validator have balances > MAX_EFFECTIVE_BALANCE and ETH1 withdrawal credential prefix set +// All Validator have balances > MAX_EFFECTIVE_BALANCE and ETH1 withdrawal credential prefix set // TODO Electra: Not true anymore // // Worst case: // All balances are low enough or withdrawal credential not set @@ -69,7 +70,7 @@ describe("getExpectedWithdrawals", () => { return opts.cache ? state : state.clone(true); }, fn: (state) => { - const {sampledValidators} = getExpectedWithdrawals(state); + const {sampledValidators} = getExpectedWithdrawals(ForkSeq.capella, state); // TODO Electra: Do test for electra if (sampledValidators !== opts.sampled) { throw Error(`Wrong sampledValidators ${sampledValidators} != ${opts.sampled}`); } diff --git a/packages/state-transition/test/unit/block/processWithdrawals.test.ts b/packages/state-transition/test/unit/block/processWithdrawals.test.ts index 2841da635472..7b708d108a7b 100644 --- a/packages/state-transition/test/unit/block/processWithdrawals.test.ts +++ b/packages/state-transition/test/unit/block/processWithdrawals.test.ts @@ -1,4 +1,5 @@ import {describe, it, expect} from "vitest"; +import {ForkSeq} from "@lodestar/params"; import {getExpectedWithdrawals} from "../../../src/block/processWithdrawals.js"; import {numValidators} from "../../perf/util.js"; import {getExpectedWithdrawalsTestData, WithdrawalOpts} from "../../utils/capella.js"; @@ -36,8 +37,9 @@ describe("getExpectedWithdrawals", () => { // Clone true to drop cache const state = beforeValue(() => getExpectedWithdrawalsTestData(vc, opts).clone(true)); + // TODO Electra: Add test for electra it(`getExpectedWithdrawals ${vc} ${caseID}`, () => { - const {sampledValidators, withdrawals} = getExpectedWithdrawals(state.value); + const {sampledValidators, withdrawals} = getExpectedWithdrawals(ForkSeq.capella, state.value); expect(sampledValidators).toBe(opts.sampled); expect(withdrawals.length).toBe(opts.withdrawals); }); diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 53195f30de11..26539dd61365 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -17,6 +17,10 @@ import { MAX_ATTESTATIONS_ELECTRA, MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, + MAX_CONSOLIDATIONS, + PENDING_BALANCE_DEPOSITS_LIMIT, + PENDING_CONSOLIDATIONS_LIMIT, + PENDING_PARTIAL_WITHDRAWALS_LIMIT, } from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; @@ -26,6 +30,8 @@ import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; const { + Epoch, + Gwei, UintNum64, Slot, Root, @@ -148,6 +154,23 @@ export const ExecutionPayloadHeader = new ContainerType( {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); +export const Consolidation = new ContainerType( + { + sourceIndex: ValidatorIndex, + targetIndex: ValidatorIndex, + epoch: Epoch, + }, + {typeName: "Consolidation", jsonCase: "eth2"} +); + +export const SignedConsolidation = new ContainerType( + { + message: Consolidation, + signature: BLSSignature, + }, + {typeName: "SignedConsolidation", jsonCase: "eth2"} +); + // We have to preserve Fields ordering while changing the type of ExecutionPayload export const BeaconBlockBody = new ContainerType( { @@ -163,6 +186,7 @@ export const BeaconBlockBody = new ContainerType( executionPayload: ExecutionPayload, // Modified in ELECTRA blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, + consolidations: new ListCompositeType(SignedConsolidation, MAX_CONSOLIDATIONS), // [New in Electra] }, {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -243,8 +267,32 @@ export const ExecutionPayloadAndBlobsBundle = new ContainerType( {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} ); -// We don't spread deneb.BeaconState fields since we need to replace -// latestExecutionPayloadHeader and we cannot keep order doing that +export const PendingBalanceDeposit = new ContainerType( + { + index: ValidatorIndex, + amount: Gwei, + }, + {typeName: "PendingBalanceDeposit", jsonCase: "eth2"} +); + +export const PartialWithdrawal = new ContainerType( + { + index: ValidatorIndex, + amount: Gwei, + withdrawableEpoch: Epoch, + }, + {typeName: "PartialWithdrawal", jsonCase: "eth2"} +); + +export const PendingConsolidation = new ContainerType( + { + sourceIndex: ValidatorIndex, + targetIndex: ValidatorIndex, + }, + {typeName: "PendingConsolidation", jsonCase: "eth2"} +); + +// In EIP-7251, we spread deneb fields as new fields are appended at the end export const BeaconState = new ContainerType( { genesisTime: UintNum64, @@ -288,6 +336,14 @@ export const BeaconState = new ContainerType( // Deep history valid from Capella onwards historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, depositReceiptsStartIndex: UintBn64, // New in ELECTRA + depositBalanceToConsume: Gwei, // [New in Electra] + exitBalanceToConsume: Gwei, // [New in Electra] + earliestExitEpoch: Epoch, // [New in Electra] + consolidationBalanceToConsume: Gwei, // [New in Electra] + earliestConsolidationEpoch: Epoch, // [New in Electra] + pendingBalanceDeposits: new ListCompositeType(PendingBalanceDeposit, PENDING_BALANCE_DEPOSITS_LIMIT), // [New in Electra] + pendingPartialWithdrawals: new ListCompositeType(PartialWithdrawal, PENDING_PARTIAL_WITHDRAWALS_LIMIT), // [New in Electra] + pendingConsolidations: new ListCompositeType(PendingConsolidation, PENDING_CONSOLIDATIONS_LIMIT), // [New in Electra] }, {typeName: "BeaconState", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index d1d0109e6ae6..4de209b48960 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -42,3 +42,10 @@ export type LightClientUpdate = ValueOf; export type LightClientFinalityUpdate = ValueOf; export type LightClientOptimisticUpdate = ValueOf; export type LightClientStore = ValueOf; + +export type Consolidation = ValueOf; +export type SignedConsolidation = ValueOf; + +export type PendingBalanceDeposit = ValueOf; +export type PartialWithdrawal = ValueOf; +export type PendingConsolidation = ValueOf; diff --git a/packages/types/src/phase0/sszTypes.ts b/packages/types/src/phase0/sszTypes.ts index 9eb2a13e5fae..4a04701b789d 100644 --- a/packages/types/src/phase0/sszTypes.ts +++ b/packages/types/src/phase0/sszTypes.ts @@ -236,7 +236,7 @@ export const RandaoMixes = new VectorCompositeType(Bytes32, EPOCHS_PER_HISTORICA * This is initially a Gwei (BigInt) vector, however since Nov 2023 it's converted to UintNum64 (number) vector in the state transition because: * - state.slashings[nextEpoch % EPOCHS_PER_SLASHINGS_VECTOR] is reset per epoch in processSlashingsReset() * - max slashed validators per epoch is SLOTS_PER_EPOCH * MAX_ATTESTER_SLASHINGS * MAX_VALIDATORS_PER_COMMITTEE which is 32 * 2 * 2048 = 131072 on mainnet - * - with that and 32_000_000_000 MAX_EFFECTIVE_BALANCE, it still fits in a number given that Math.floor(Number.MAX_SAFE_INTEGER / 32_000_000_000) = 281474 + * - with that and 32_000_000_000 MAX_EFFECTIVE_BALANCE or 2048_000_000_000 MAX_EFFECTIVE_BALANCE_ELECTRA, it still fits in a number given that Math.floor(Number.MAX_SAFE_INTEGER / 32_000_000_000) = 281474 * - we don't need to compute the total slashings from state.slashings, it's handled by totalSlashingsByIncrement in EpochCache */ export const Slashings = new VectorBasicType(UintNum64, EPOCHS_PER_SLASHINGS_VECTOR); diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index ff1c8c0fdc25..3a497555361a 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -227,5 +227,16 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Tue, 7 May 2024 22:11:51 +0300 Subject: [PATCH 030/145] feat: beacon node process electra attestations EIP-7549 (#6738) * Process attestations in block * Fix check-types * Address comments --- .../src/chain/blocks/importBlock.ts | 4 +- .../test/spec/presets/fork_choice.test.ts | 5 +- .../src/block/processAttestationPhase0.ts | 60 +++++++++---- .../src/block/processAttestations.ts | 4 +- .../src/block/processAttestationsAltair.ts | 7 +- .../state-transition/src/cache/epochCache.ts | 90 ++++++++++++++++--- .../src/signatureSets/indexedAttestation.ts | 6 +- 7 files changed, 139 insertions(+), 37 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 360a3f8f5db9..de5ecf607d95 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -69,6 +69,7 @@ export async function importBlock( const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; const blockDelaySec = (fullyVerifiedBlock.seenTimestampSec - postState.genesisTime) % this.config.SECONDS_PER_SLOT; const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000); + const fork = this.config.getForkSeq(blockSlot); // this is just a type assertion since blockinput with dataPromise type will not end up here if (blockInput.type === BlockInputType.dataPromise) { @@ -120,7 +121,8 @@ export async function importBlock( for (const attestation of attestations) { try { - const indexedAttestation = postState.epochCtx.getIndexedAttestation(attestation); + // TODO Electra: figure out how to reuse the attesting indices computed from state transition + const indexedAttestation = postState.epochCtx.getIndexedAttestation(fork, attestation); const {target, beaconBlockRoot} = attestation.data; const attDataRoot = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(indexedAttestation.data)); diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index 92862c6cb03b..01adcc77500b 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -137,7 +137,10 @@ const forkChoiceTest = if (!attestation) throw Error(`No attestation ${step.attestation}`); const headState = chain.getHeadState(); const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation.data)); - chain.forkChoice.onAttestation(headState.epochCtx.getIndexedAttestation(attestation), attDataRootHex); + chain.forkChoice.onAttestation( + headState.epochCtx.getIndexedAttestation(ForkSeq[fork], attestation), + attDataRootHex + ); } // attester slashing step diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index 95fc575003ee..c7c6b3b531a8 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -1,7 +1,7 @@ -import {Slot, phase0, ssz} from "@lodestar/types"; - -import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, ForkSeq} from "@lodestar/params"; import {toRootHex} from "@lodestar/utils"; +import {Slot, allForks, electra, phase0, ssz} from "@lodestar/types"; +import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, ForkSeq} from "@lodestar/params"; +import {assert} from "@lodestar/utils"; import {computeEpochAtSlot} from "../util/index.js"; import {CachedBeaconStatePhase0, CachedBeaconStateAllForks} from "../types.js"; import {isValidIndexedAttestation} from "./index.js"; @@ -51,7 +51,7 @@ export function processAttestationPhase0( state.previousEpochAttestations.push(pendingAttestation); } - if (!isValidIndexedAttestation(state, epochCtx.getIndexedAttestation(attestation), verifySignature)) { + if (!isValidIndexedAttestation(state, epochCtx.getIndexedAttestation(ForkSeq.phase0, attestation), verifySignature)) { throw new Error("Attestation is not valid"); } } @@ -59,19 +59,14 @@ export function processAttestationPhase0( export function validateAttestation( fork: ForkSeq, state: CachedBeaconStateAllForks, - attestation: phase0.Attestation + attestation: allForks.Attestation ): void { const {epochCtx} = state; const slot = state.slot; const data = attestation.data; const computedEpoch = computeEpochAtSlot(data.slot); const committeeCount = epochCtx.getCommitteeCountPerSlot(computedEpoch); - if (!(data.index < committeeCount)) { - throw new Error( - "Attestation committee index not within current committee count: " + - `committeeIndex=${data.index} committeeCount=${committeeCount}` - ); - } + if (!(data.target.epoch === epochCtx.previousShuffling.epoch || data.target.epoch === epochCtx.epoch)) { throw new Error( "Attestation target epoch not in previous or current epoch: " + @@ -93,12 +88,45 @@ export function validateAttestation( ); } - const committee = epochCtx.getBeaconCommittee(data.slot, data.index); - if (attestation.aggregationBits.bitLen !== committee.length) { - throw new Error( - "Attestation aggregation bits length does not match committee length: " + - `aggregationBitsLength=${attestation.aggregationBits.bitLen} committeeLength=${committee.length}` + if (fork >= ForkSeq.electra) { + assert.equal(data.index, 0, `AttestationData.index must be zero: index=${data.index}`); + const attestationElectra = attestation as electra.Attestation; + const committeeBitsLength = attestationElectra.committeeBits.bitLen; + + if (committeeBitsLength > committeeCount) { + throw new Error( + `Attestation committee bits length are longer than number of committees: committeeBitsLength=${committeeBitsLength} numCommittees=${committeeCount}` + ); + } + + // TODO Electra: this should be obsolete soon when the spec switches to committeeIndices + const committeeIndices = attestationElectra.committeeBits.getTrueBitIndexes(); + + // Get total number of attestation participant of every committee specified + const participantCount = committeeIndices + .map((committeeIndex) => epochCtx.getBeaconCommittee(data.slot, committeeIndex).length) + .reduce((acc, committeeSize) => acc + committeeSize, 0); + + assert.equal( + attestationElectra.aggregationBits.bitLen, + participantCount, + `Attestation aggregation bits length does not match total number of committee participant aggregationBitsLength=${attestation.aggregationBits.bitLen} participantCount=${participantCount}` ); + } else { + if (!(data.index < committeeCount)) { + throw new Error( + "Attestation committee index not within current committee count: " + + `committeeIndex=${data.index} committeeCount=${committeeCount}` + ); + } + + const committee = epochCtx.getBeaconCommittee(data.slot, data.index); + if (attestation.aggregationBits.bitLen !== committee.length) { + throw new Error( + "Attestation aggregation bits length does not match committee length: " + + `aggregationBitsLength=${attestation.aggregationBits.bitLen} committeeLength=${committee.length}` + ); + } } } diff --git a/packages/state-transition/src/block/processAttestations.ts b/packages/state-transition/src/block/processAttestations.ts index 2b132fa22e0b..991ba5621905 100644 --- a/packages/state-transition/src/block/processAttestations.ts +++ b/packages/state-transition/src/block/processAttestations.ts @@ -1,4 +1,4 @@ -import {phase0} from "@lodestar/types"; +import {allForks} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../types.js"; import {processAttestationPhase0} from "./processAttestationPhase0.js"; @@ -10,7 +10,7 @@ import {processAttestationsAltair} from "./processAttestationsAltair.js"; export function processAttestations( fork: ForkSeq, state: CachedBeaconStateAllForks, - attestations: phase0.Attestation[], + attestations: allForks.Attestation[], verifySignatures = true ): void { if (fork === ForkSeq.phase0) { diff --git a/packages/state-transition/src/block/processAttestationsAltair.ts b/packages/state-transition/src/block/processAttestationsAltair.ts index e37629712194..48d304c08133 100644 --- a/packages/state-transition/src/block/processAttestationsAltair.ts +++ b/packages/state-transition/src/block/processAttestationsAltair.ts @@ -1,5 +1,5 @@ import {byteArrayEquals} from "@chainsafe/ssz"; -import {Epoch, phase0} from "@lodestar/types"; +import {Epoch, allForks, phase0} from "@lodestar/types"; import {intSqrt} from "@lodestar/utils"; import { @@ -32,7 +32,7 @@ const SLOTS_PER_EPOCH_SQRT = intSqrt(SLOTS_PER_EPOCH); export function processAttestationsAltair( fork: ForkSeq, state: CachedBeaconStateAltair, - attestations: phase0.Attestation[], + attestations: allForks.Attestation[], verifySignature = true ): void { const {epochCtx} = state; @@ -49,8 +49,7 @@ export function processAttestationsAltair( validateAttestation(fork, state, attestation); // Retrieve the validator indices from the attestation participation bitfield - const committeeIndices = epochCtx.getBeaconCommittee(data.slot, data.index); - const attestingIndices = attestation.aggregationBits.intersectValues(committeeIndices); + const attestingIndices = epochCtx.getAttestingIndices(fork, attestation); // this check is done last because its the most expensive (if signature verification is toggled on) // TODO: Why should we verify an indexed attestation that we just created? If it's just for the signature diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 538f00dd91c7..71c3b3d15f1a 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,7 +1,17 @@ import {PublicKey} from "@chainsafe/blst"; import * as immutable from "immutable"; import {fromHexString} from "@chainsafe/ssz"; -import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod} from "@lodestar/types"; +import { + BLSSignature, + CommitteeIndex, + Epoch, + Slot, + ValidatorIndex, + phase0, + SyncPeriod, + allForks, + electra, +} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainConfig} from "@lodestar/config"; import { ATTESTATION_SUBNET_COUNT, @@ -651,15 +661,47 @@ export class EpochCache { * Return the beacon committee at slot for index. */ getBeaconCommittee(slot: Slot, index: CommitteeIndex): Uint32Array { + return this.getBeaconCommittees(slot, [index]); + } + + /** + * Return a single Uint32Array representing concatted committees of indices + */ + getBeaconCommittees(slot: Slot, indices: CommitteeIndex[]): Uint32Array { + if (indices.length === 0) { + throw new Error("Attempt to get committees without providing CommitteeIndex"); + } + const slotCommittees = this.getShufflingAtSlot(slot).committees[slot % SLOTS_PER_EPOCH]; - if (index >= slotCommittees.length) { - throw new EpochCacheError({ - code: EpochCacheErrorCode.COMMITTEE_INDEX_OUT_OF_RANGE, - index, - maxIndex: slotCommittees.length, - }); + const committees = []; + + for (const index of indices) { + if (index >= slotCommittees.length) { + throw new EpochCacheError({ + code: EpochCacheErrorCode.COMMITTEE_INDEX_OUT_OF_RANGE, + index, + maxIndex: slotCommittees.length, + }); + } + committees.push(slotCommittees[index]); + } + + // Early return if only one index + if (committees.length === 1) { + return committees[0]; + } + + // Create a new Uint32Array to flatten `committees` + const totalLength = committees.reduce((acc, curr) => acc + curr.length, 0); + const result = new Uint32Array(totalLength); + + let offset = 0; + for (const committee of committees) { + result.set(committee, offset); + offset += committee.length; } - return slotCommittees[index]; + + return result; } getCommitteeCountPerSlot(epoch: Epoch): number { @@ -745,10 +787,9 @@ export class EpochCache { /** * Return the indexed attestation corresponding to ``attestation``. */ - getIndexedAttestation(attestation: phase0.Attestation): phase0.IndexedAttestation { - const {aggregationBits, data} = attestation; - const committeeIndices = this.getBeaconCommittee(data.slot, data.index); - const attestingIndices = aggregationBits.intersectValues(committeeIndices); + getIndexedAttestation(fork: ForkSeq, attestation: allForks.Attestation): allForks.IndexedAttestation { + const {data} = attestation; + const attestingIndices = this.getAttestingIndices(fork, attestation); // sort in-place attestingIndices.sort((a, b) => a - b); @@ -759,6 +800,31 @@ export class EpochCache { }; } + /** + * Return indices of validators who attestested in `attestation` + */ + getAttestingIndices(fork: ForkSeq, attestation: allForks.Attestation): number[] { + if (fork < ForkSeq.electra) { + const {aggregationBits, data} = attestation; + const validatorIndices = this.getBeaconCommittee(data.slot, data.index); + + return aggregationBits.intersectValues(validatorIndices); + } else { + const {aggregationBits, committeeBits, data} = attestation as electra.Attestation; + + // There is a naming conflict on the term `committeeIndices` + // In Lodestar it usually means a list of validator indices of participants in a committee + // In the spec it means a list of committee indices according to committeeBits + // This `committeeIndices` refers to the latter + // TODO Electra: resolve the naming conflicts + const committeeIndices = committeeBits.getTrueBitIndexes(); + + const validatorIndices = this.getBeaconCommittees(data.slot, committeeIndices); + + return aggregationBits.intersectValues(validatorIndices); + } + } + getCommitteeAssignments( epoch: Epoch, requestedValidatorIndices: ValidatorIndex[] diff --git a/packages/state-transition/src/signatureSets/indexedAttestation.ts b/packages/state-transition/src/signatureSets/indexedAttestation.ts index 9ae6627d0b56..d3decaf6406f 100644 --- a/packages/state-transition/src/signatureSets/indexedAttestation.ts +++ b/packages/state-transition/src/signatureSets/indexedAttestation.ts @@ -41,7 +41,11 @@ export function getAttestationsSignatureSets( state: CachedBeaconStateAllForks, signedBlock: SignedBeaconBlock ): ISignatureSet[] { + // TODO: figure how to get attesting indices of an attestation once per block processing return signedBlock.message.body.attestations.map((attestation) => - getIndexedAttestationSignatureSet(state, state.epochCtx.getIndexedAttestation(attestation)) + getIndexedAttestationSignatureSet( + state, + state.epochCtx.getIndexedAttestation(state.epochCtx.config.getForkSeq(signedBlock.message.slot), attestation) + ) ); } From 8e1535058a525345103fd1ce22353aa9d8a8218e Mon Sep 17 00:00:00 2001 From: g11tech Date: Wed, 8 May 2024 16:01:26 +0530 Subject: [PATCH 031/145] feat: handle the EL payload sending data in deposit requests instead of deposit receipts (#6746) --- .../beacon-node/src/execution/engine/types.ts | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 54b3a415ec98..f0701d48c5fc 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -125,7 +125,9 @@ type ExecutionPayloadResponse = ExecutionPayloadRpc | ExecutionPayloadRpcWithVal export type ExecutionPayloadBodyRpc = { transactions: DATA[]; withdrawals: WithdrawalV1[] | null | undefined; - depositReceipts: DepositReceiptV1[] | null | undefined; + // currently there is a discepancy between EL and CL field name references for deposit requests + // its likely CL receipt will be renamed to requests + depositRequests: DepositReceiptV1[] | null | undefined; withdrawalRequests: ExecutionLayerWithdrawalRequestV1[] | null | undefined; }; @@ -155,7 +157,7 @@ export type ExecutionPayloadRpc = { blobGasUsed?: QUANTITY; // DENEB excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB - depositReceipts?: DepositReceiptRpc[]; // ELECTRA + depositRequests?: DepositReceiptRpc[]; // ELECTRA withdrawalRequests?: ExecutionLayerWithdrawalRequestRpc[]; // ELECTRA }; @@ -231,10 +233,10 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload payload.excessBlobGas = numToQuantity(excessBlobGas); } - // ELECTRA adds depositReceipts to the ExecutionPayload + // ELECTRA adds depositReceipts/depositRequests to the ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { const {depositReceipts, withdrawalRequests} = data as electra.ExecutionPayload; - payload.depositReceipts = depositReceipts.map(serializeDepositReceipt); + payload.depositRequests = depositReceipts.map(serializeDepositReceipt); payload.withdrawalRequests = withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest); } @@ -324,14 +326,15 @@ export function parseExecutionPayload( } if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositReceipts, withdrawalRequests} = data; + // electra adds depositRequests/depositReceipts + const {depositRequests, withdrawalRequests} = data; // Geth can also reply with null - if (depositReceipts == null) { + if (depositRequests == null) { throw Error( - `depositReceipts missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` + `depositRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` ); } - (executionPayload as electra.ExecutionPayload).depositReceipts = depositReceipts.map(deserializeDepositReceipt); + (executionPayload as electra.ExecutionPayload).depositReceipts = depositRequests.map(deserializeDepositReceipt); if (withdrawalRequests == null) { throw Error( @@ -454,7 +457,7 @@ export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | ? { transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, - depositReceipts: data.depositReceipts ? data.depositReceipts.map(deserializeDepositReceipt) : null, + depositReceipts: data.depositRequests ? data.depositRequests.map(deserializeDepositReceipt) : null, withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(deserializeExecutionLayerWithdrawalRequest) : null, @@ -467,7 +470,7 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) ? { transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, - depositReceipts: data.depositReceipts ? data.depositReceipts.map(serializeDepositReceipt) : null, + depositRequests: data.depositReceipts ? data.depositReceipts.map(serializeDepositReceipt) : null, withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest) : null, From 84cf3962de6b1cca0789c1070ce2ce3002199c91 Mon Sep 17 00:00:00 2001 From: NC Date: Wed, 8 May 2024 14:00:11 +0300 Subject: [PATCH 032/145] feat: implement EIP-7549 (#6689) * initial commit * lint * Add getAttestingIndices and update getIndexedAttestation * Update gossip validation * Update attestation gossip validation * aggregateAndProof validation * clean up * Validator * Misc * Fix the build erros * feat: get attestations for electra block (#6732) * feat: getAttestationsForBlock() for electra * chore: fix lint * fix: MAX_ATTESTATIONS_PER_GROUP_ELECTRA and address PR comments * chore: unit test aggregateConsolidation * Fix rebase mistake * Address my own comment :) --------- Co-authored-by: Navie Chan * Fix check-types * Address comments --------- Co-authored-by: Nazar Hussain Co-authored-by: tuyennhv Co-authored-by: gajinder --- packages/api/src/beacon/routes/beacon/pool.ts | 66 +++- packages/api/src/beacon/routes/events.ts | 8 +- packages/api/src/beacon/routes/validator.ts | 93 +++-- .../src/api/impl/beacon/pool/index.ts | 4 +- .../src/api/impl/validator/index.ts | 4 +- .../src/chain/blocks/importBlock.ts | 10 +- .../src/chain/errors/attestationError.ts | 12 +- .../opPools/aggregatedAttestationPool.ts | 348 ++++++++++++++---- .../src/chain/opPools/attestationPool.ts | 72 +++- .../beacon-node/src/chain/opPools/opPool.ts | 8 +- .../src/chain/validation/aggregateAndProof.ts | 33 +- .../src/chain/validation/attestation.ts | 50 ++- .../src/chain/validation/attesterSlashing.ts | 2 +- .../src/metrics/validatorMonitor.ts | 2 +- .../src/network/gossip/interface.ts | 10 +- .../beacon-node/src/network/gossip/topic.ts | 11 +- packages/beacon-node/src/network/interface.ts | 2 +- packages/beacon-node/src/network/network.ts | 2 +- .../src/network/processor/gossipHandlers.ts | 17 +- .../test/unit/api/impl/events/events.test.ts | 6 +- .../opPools/aggregatedAttestationPool.test.ts | 87 ++++- .../fork-choice/src/forkChoice/interface.ts | 2 +- .../src/signatureSets/attesterSlashings.ts | 6 +- .../src/signatureSets/index.ts | 6 +- .../validator/src/services/attestation.ts | 6 +- .../validator/src/services/validatorStore.ts | 37 +- packages/validator/src/validator.ts | 1 + .../test/unit/services/attestation.test.ts | 3 + 28 files changed, 718 insertions(+), 190 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/pool.ts b/packages/api/src/beacon/routes/beacon/pool.ts index f957390131fe..3c390aa41481 100644 --- a/packages/api/src/beacon/routes/beacon/pool.ts +++ b/packages/api/src/beacon/routes/beacon/pool.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; +import {ForkSeq} from "@lodestar/params"; import {phase0, capella, CommitteeIndex, Slot, ssz} from "@lodestar/types"; import {Schema, Endpoint, RouteDefinitions} from "../../../utils/index.js"; import { @@ -12,18 +13,24 @@ import { EmptyRequest, EmptyResponseCodec, EmptyResponseData, + WithVersion, } from "../../../utils/codecs.js"; +import {MetaHeader, VersionCodec, VersionMeta} from "../../../utils/metadata.js"; +import {toForkName} from "../../../utils/fork.js"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes -const AttestationListType = ArrayOf(ssz.phase0.Attestation); +const AttestationListTypePhase0 = ArrayOf(ssz.phase0.Attestation); +const AttestationListTypeElectra = ArrayOf(ssz.electra.Attestation); const AttesterSlashingListType = ArrayOf(ssz.phase0.AttesterSlashing); const ProposerSlashingListType = ArrayOf(ssz.phase0.ProposerSlashing); const SignedVoluntaryExitListType = ArrayOf(ssz.phase0.SignedVoluntaryExit); const SignedBLSToExecutionChangeListType = ArrayOf(ssz.capella.SignedBLSToExecutionChange); const SyncCommitteeMessageListType = ArrayOf(ssz.altair.SyncCommitteeMessage); -type AttestationList = ValueOf; +type AttestationListPhase0 = ValueOf; +type AttestationListElectra = ValueOf; +type AttestationList = AttestationListPhase0 | AttestationListElectra; type AttesterSlashingList = ValueOf; type ProposerSlashingList = ValueOf; type SignedVoluntaryExitList = ValueOf; @@ -40,7 +47,7 @@ export type Endpoints = { {slot?: Slot; committeeIndex?: CommitteeIndex}, {query: {slot?: number; committee_index?: number}}, AttestationList, - EmptyMeta + VersionMeta >; /** @@ -106,7 +113,7 @@ export type Endpoints = { submitPoolAttestations: Endpoint< "POST", {signedAttestations: AttestationList}, - {body: unknown}, + {body: unknown; headers: {[MetaHeader.Version]: string}}, EmptyResponseData, EmptyMeta >; @@ -172,7 +179,7 @@ export type Endpoints = { >; }; -export function getDefinitions(_config: ChainForkConfig): RouteDefinitions { +export function getDefinitions(config: ChainForkConfig): RouteDefinitions { return { getPoolAttestations: { url: "/eth/v1/beacon/pool/attestations", @@ -183,8 +190,10 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions + ForkSeq[fork] >= ForkSeq.electra ? AttestationListTypeElectra : AttestationListTypePhase0 + ), + meta: VersionCodec, }, }, getPoolAttesterSlashings: { @@ -227,12 +236,47 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({body: AttestationListType.toJson(signedAttestations)}), - parseReqJson: ({body}) => ({signedAttestations: AttestationListType.fromJson(body)}), - writeReqSsz: ({signedAttestations}) => ({body: AttestationListType.serialize(signedAttestations)}), - parseReqSsz: ({body}) => ({signedAttestations: AttestationListType.deserialize(body)}), + writeReqJson: ({signedAttestations}) => { + const fork = config.getForkName(signedAttestations[0].data.slot); + return { + body: + ForkSeq[fork] >= ForkSeq.electra + ? AttestationListTypeElectra.toJson(signedAttestations as AttestationListElectra) + : AttestationListTypePhase0.toJson(signedAttestations as AttestationListPhase0), + headers: {[MetaHeader.Version]: fork}, + }; + }, + parseReqJson: ({body, headers}) => { + const fork = toForkName(headers[MetaHeader.Version]); + return { + signedAttestations: + ForkSeq[fork] >= ForkSeq.electra + ? AttestationListTypeElectra.fromJson(body) + : AttestationListTypePhase0.fromJson(body), + }; + }, + writeReqSsz: ({signedAttestations}) => { + const fork = config.getForkName(signedAttestations[0].data.slot); + return { + body: + ForkSeq[fork] >= ForkSeq.electra + ? AttestationListTypeElectra.serialize(signedAttestations as AttestationListElectra) + : AttestationListTypePhase0.serialize(signedAttestations as AttestationListPhase0), + headers: {[MetaHeader.Version]: fork}, + }; + }, + parseReqSsz: ({body, headers}) => { + const fork = toForkName(headers[MetaHeader.Version]); + return { + signedAttestations: + ForkSeq[fork] >= ForkSeq.electra + ? AttestationListTypeElectra.deserialize(body) + : AttestationListTypePhase0.deserialize(body), + }; + }, schema: { body: Schema.ObjectArray, + headers: {[MetaHeader.Version]: Schema.String}, }, }, resp: EmptyResponseCodec, diff --git a/packages/api/src/beacon/routes/events.ts b/packages/api/src/beacon/routes/events.ts index 23be5e7c2288..04131750dbf2 100644 --- a/packages/api/src/beacon/routes/events.ts +++ b/packages/api/src/beacon/routes/events.ts @@ -104,10 +104,10 @@ export type EventData = { block: RootHex; executionOptimistic: boolean; }; - [EventType.attestation]: phase0.Attestation; + [EventType.attestation]: {version: ForkName; data: allForks.Attestation}; [EventType.voluntaryExit]: phase0.SignedVoluntaryExit; [EventType.proposerSlashing]: phase0.ProposerSlashing; - [EventType.attesterSlashing]: phase0.AttesterSlashing; + [EventType.attesterSlashing]: {version: ForkName; data: allForks.AttesterSlashing}; [EventType.blsToExecutionChange]: capella.SignedBLSToExecutionChange; [EventType.finalizedCheckpoint]: { block: RootHex; @@ -225,10 +225,10 @@ export function getTypeByEvent(): {[K in EventType]: TypeJson} { {jsonCase: "eth2"} ), - [EventType.attestation]: ssz.phase0.Attestation, + [EventType.attestation]: WithVersion((fork) => (ssz.allForks[fork] as allForks.AllForksSSZTypes).Attestation), [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit, [EventType.proposerSlashing]: ssz.phase0.ProposerSlashing, - [EventType.attesterSlashing]: ssz.phase0.AttesterSlashing, + [EventType.attesterSlashing]: WithVersion((fork) => ssz.allForks[fork].AttesterSlashing), [EventType.blsToExecutionChange]: ssz.capella.SignedBLSToExecutionChange, [EventType.finalizedCheckpoint]: new ContainerType( diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 33161ec789e5..f37ec7b24488 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ContainerType, fromHexString, toHexString, Type, ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {isForkBlobs} from "@lodestar/params"; +import {isForkBlobs, ForkSeq} from "@lodestar/params"; import { altair, BLSSignature, @@ -208,7 +208,8 @@ export const ValidatorIndicesType = ArrayOf(ssz.ValidatorIndex); export const AttesterDutyListType = ArrayOf(AttesterDutyType); export const ProposerDutyListType = ArrayOf(ProposerDutyType); export const SyncDutyListType = ArrayOf(SyncDutyType); -export const SignedAggregateAndProofListType = ArrayOf(ssz.phase0.SignedAggregateAndProof); +export const SignedAggregateAndProofListPhase0Type = ArrayOf(ssz.phase0.SignedAggregateAndProof); +export const SignedAggregateAndProofListElectaType = ArrayOf(ssz.electra.SignedAggregateAndProof); export const SignedContributionAndProofListType = ArrayOf(ssz.altair.SignedContributionAndProof); export const BeaconCommitteeSubscriptionListType = ArrayOf(BeaconCommitteeSubscriptionType); export const SyncCommitteeSubscriptionListType = ArrayOf(SyncCommitteeSubscriptionType); @@ -225,7 +226,9 @@ export type ProposerDuty = ValueOf; export type ProposerDutyList = ValueOf; export type SyncDuty = ValueOf; export type SyncDutyList = ValueOf; -export type SignedAggregateAndProofList = ValueOf; +export type SignedAggregateAndProofListPhase0 = ValueOf; +export type SignedAggregateAndProofListElecta = ValueOf; +export type SignedAggregateAndProofList = SignedAggregateAndProofListPhase0 | SignedAggregateAndProofListElecta; export type SignedContributionAndProofList = ValueOf; export type BeaconCommitteeSubscription = ValueOf; export type BeaconCommitteeSubscriptionList = ValueOf; @@ -406,10 +409,11 @@ export type Endpoints = { /** HashTreeRoot of AttestationData that validator want's aggregated */ attestationDataRoot: Root; slot: Slot; + index: number; }, - {query: {attestation_data_root: string; slot: number}}, - phase0.Attestation, - EmptyMeta + {query: {attestation_data_root: string; slot: number; index: number}}, + allForks.Attestation, + VersionMeta >; /** @@ -419,7 +423,7 @@ export type Endpoints = { publishAggregateAndProofs: Endpoint< "POST", {signedAggregateAndProofs: SignedAggregateAndProofList}, - {body: unknown}, + {body: unknown; headers: {[MetaHeader.Version]: string}}, EmptyResponseData, EmptyMeta >; @@ -536,7 +540,7 @@ export type Endpoints = { >; }; -export function getDefinitions(_config: ChainForkConfig): RouteDefinitions { +export function getDefinitions(config: ChainForkConfig): RouteDefinitions { return { getAttesterDuties: { url: "/eth/v1/validator/duties/attester/{epoch}", @@ -801,33 +805,78 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({ - query: {attestation_data_root: toHexString(attestationDataRoot), slot}, + writeReq: ({attestationDataRoot, slot, index}) => ({ + query: {attestation_data_root: toHexString(attestationDataRoot), slot, index}, + }), + parseReq: ({query}) => ({ + attestationDataRoot: fromHexString(query.attestation_data_root), + slot: query.slot, + index: query.slot, }), - parseReq: ({query}) => ({attestationDataRoot: fromHexString(query.attestation_data_root), slot: query.slot}), schema: { - query: {attestation_data_root: Schema.StringRequired, slot: Schema.UintRequired}, + query: {attestation_data_root: Schema.StringRequired, slot: Schema.UintRequired, index: Schema.UintRequired}, }, }, resp: { - data: ssz.phase0.Attestation, - meta: EmptyMetaCodec, + data: WithVersion((fork) => + ForkSeq[fork] >= ForkSeq.electra ? ssz.electra.Attestation : ssz.phase0.Attestation + ), + meta: VersionCodec, }, }, publishAggregateAndProofs: { url: "/eth/v1/validator/aggregate_and_proofs", method: "POST", req: { - writeReqJson: ({signedAggregateAndProofs}) => ({ - body: SignedAggregateAndProofListType.toJson(signedAggregateAndProofs), - }), - parseReqJson: ({body}) => ({signedAggregateAndProofs: SignedAggregateAndProofListType.fromJson(body)}), - writeReqSsz: ({signedAggregateAndProofs}) => ({ - body: SignedAggregateAndProofListType.serialize(signedAggregateAndProofs), - }), - parseReqSsz: ({body}) => ({signedAggregateAndProofs: SignedAggregateAndProofListType.deserialize(body)}), + writeReqJson: ({signedAggregateAndProofs}) => { + const fork = config.getForkName(signedAggregateAndProofs[0].message.aggregate.data.slot); + return { + body: + ForkSeq[fork] >= ForkSeq.electra + ? SignedAggregateAndProofListElectaType.toJson( + signedAggregateAndProofs as SignedAggregateAndProofListElecta + ) + : SignedAggregateAndProofListPhase0Type.toJson( + signedAggregateAndProofs as SignedAggregateAndProofListPhase0 + ), + headers: {[MetaHeader.Version]: fork}, + }; + }, + parseReqJson: ({body, headers}) => { + const fork = toForkName(headers[MetaHeader.Version]); + return { + signedAggregateAndProofs: + ForkSeq[fork] >= ForkSeq.electra + ? SignedAggregateAndProofListElectaType.fromJson(body) + : SignedAggregateAndProofListPhase0Type.fromJson(body), + }; + }, + writeReqSsz: ({signedAggregateAndProofs}) => { + const fork = config.getForkName(signedAggregateAndProofs[0].message.aggregate.data.slot); + return { + body: + ForkSeq[fork] >= ForkSeq.electra + ? SignedAggregateAndProofListElectaType.serialize( + signedAggregateAndProofs as SignedAggregateAndProofListElecta + ) + : SignedAggregateAndProofListPhase0Type.serialize( + signedAggregateAndProofs as SignedAggregateAndProofListPhase0 + ), + headers: {[MetaHeader.Version]: fork}, + }; + }, + parseReqSsz: ({body, headers}) => { + const fork = toForkName(headers[MetaHeader.Version]); + return { + signedAggregateAndProofs: + ForkSeq[fork] >= ForkSeq.electra + ? SignedAggregateAndProofListElectaType.deserialize(body) + : SignedAggregateAndProofListPhase0Type.deserialize(body), + }; + }, schema: { body: Schema.ObjectArray, + headers: {[MetaHeader.Version]: Schema.String}, }, }, resp: EmptyResponseCodec, diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 8372b84db3b1..77f6b24f28af 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -1,7 +1,7 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; import {Epoch, ssz} from "@lodestar/types"; -import {SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; +import {ForkName, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; import {validateApiAttestation} from "../../../../chain/validation/index.js"; import {validateApiAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js"; import {validateApiProposerSlashing} from "../../../../chain/validation/proposerSlashing.js"; @@ -78,7 +78,7 @@ export function getBeaconPoolApi({ metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } - chain.emitter.emit(routes.events.EventType.attestation, attestation); + chain.emitter.emit(routes.events.EventType.attestation, {data: attestation, version: ForkName.phase0}); const sentPeers = await network.publishBeaconAttestation(attestation, subnet); metrics?.onPoolSubmitUnaggregatedAttestation(seenTimestampSec, indexedAttestation, subnet, sentPeers); diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index ca0fe1ac95a2..63f860e7a193 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -814,6 +814,7 @@ export function getValidatorApi( const attEpoch = computeEpochAtSlot(slot); const headBlockRootHex = chain.forkChoice.getHead().blockRoot; const headBlockRoot = fromHex(headBlockRootHex); + const fork = config.getForkSeq(slot); const beaconBlockRoot = slot >= headSlot @@ -845,7 +846,7 @@ export function getValidatorApi( return { data: { slot, - index: committeeIndex, + index: fork >= ForkSeq.electra ? 0 : committeeIndex, beaconBlockRoot, source: attEpochState.currentJustifiedCheckpoint, target: {epoch: attEpoch, root: targetRoot}, @@ -1087,6 +1088,7 @@ export function getValidatorApi( return { data: aggregate, + version: config.getForkName(slot), }; }, diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index de5ecf607d95..fa862a2fb31e 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -409,12 +409,18 @@ export async function importBlock( } if (this.emitter.listenerCount(routes.events.EventType.attestation)) { for (const attestation of block.message.body.attestations) { - this.emitter.emit(routes.events.EventType.attestation, attestation); + this.emitter.emit(routes.events.EventType.attestation, { + version: this.config.getForkName(blockSlot), + data: attestation, + }); } } if (this.emitter.listenerCount(routes.events.EventType.attesterSlashing)) { for (const attesterSlashing of block.message.body.attesterSlashings) { - this.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing); + this.emitter.emit(routes.events.EventType.attesterSlashing, { + version: this.config.getForkName(blockSlot), + data: attesterSlashing, + }); } } if (this.emitter.listenerCount(routes.events.EventType.proposerSlashing)) { diff --git a/packages/beacon-node/src/chain/errors/attestationError.ts b/packages/beacon-node/src/chain/errors/attestationError.ts index fa0635bc0115..d53a6f99682d 100644 --- a/packages/beacon-node/src/chain/errors/attestationError.ts +++ b/packages/beacon-node/src/chain/errors/attestationError.ts @@ -127,6 +127,14 @@ export enum AttestationErrorCode { INVALID_SERIALIZED_BYTES = "ATTESTATION_ERROR_INVALID_SERIALIZED_BYTES", /** Too many skipped slots. */ TOO_MANY_SKIPPED_SLOTS = "ATTESTATION_ERROR_TOO_MANY_SKIPPED_SLOTS", + /** + * Electra: The aggregated attestation doesn't have only one committee bit set. + */ + NOT_EXACTLY_ONE_COMMITTEE_BIT_SET = "ATTESTATION_ERROR_NOT_EXACTLY_ONE_COMMITTEE_BIT_SET", + /** + * Electra: Invalid attestationData index: is non-zero + */ + NON_ZERO_ATTESTATION_DATA_INDEX = "ATTESTATION_ERROR_NON_ZERO_ATTESTATION_DATA_INDEX", } export type AttestationErrorType = @@ -160,7 +168,9 @@ export type AttestationErrorType = | {code: AttestationErrorCode.INVALID_AGGREGATOR} | {code: AttestationErrorCode.INVALID_INDEXED_ATTESTATION} | {code: AttestationErrorCode.INVALID_SERIALIZED_BYTES} - | {code: AttestationErrorCode.TOO_MANY_SKIPPED_SLOTS; headBlockSlot: Slot; attestationSlot: Slot}; + | {code: AttestationErrorCode.TOO_MANY_SKIPPED_SLOTS; headBlockSlot: Slot; attestationSlot: Slot} + | {code: AttestationErrorCode.NOT_EXACTLY_ONE_COMMITTEE_BIT_SET} + | {code: AttestationErrorCode.NON_ZERO_ATTESTATION_DATA_INDEX}; export class AttestationError extends GossipActionError { getMetadata(): Record { diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 8277732ea1d5..d5368bcfe4fe 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -1,6 +1,26 @@ import {aggregateSignatures} from "@chainsafe/blst"; -import {ForkName, ForkSeq, MAX_ATTESTATIONS, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {phase0, Epoch, Slot, ssz, ValidatorIndex, RootHex} from "@lodestar/types"; +import {Signature} from "@chainsafe/bls/types"; +import {BitArray, toHexString} from "@chainsafe/ssz"; +import { + ForkName, + ForkSeq, + MAX_ATTESTATIONS, + MAX_ATTESTATIONS_ELECTRA, + MAX_COMMITTEES_PER_SLOT, + MIN_ATTESTATION_INCLUSION_DELAY, + SLOTS_PER_EPOCH, +} from "@lodestar/params"; +import { + phase0, + Epoch, + Slot, + ssz, + ValidatorIndex, + RootHex, + allForks, + electra, + isElectraAttestation, +} from "@lodestar/types"; import { CachedBeaconStateAllForks, CachedBeaconStatePhase0, @@ -19,13 +39,24 @@ type DataRootHex = string; type CommitteeIndex = number; -type AttestationWithScore = {attestation: phase0.Attestation; score: number}; +// for pre-electra +type AttestationWithScore = {attestation: allForks.Attestation; score: number}; +/** + * for electra, this is to consolidate aggregated attestations of the same attestation data into a single attestation to be included in block + * note that this is local definition in this file and it's NOT validator consolidation + */ +export type AttestationsConsolidation = { + byCommittee: Map; + attData: phase0.AttestationData; + totalNotSeenCount: number; + score: number; +}; /** - * This function returns not seen participation for a given epoch and committee. + * This function returns not seen participation for a given epoch and slot and committe index. * Return null if all validators are seen or no info to check. */ -type GetNotSeenValidatorsFn = (epoch: Epoch, committee: Uint32Array) => Set | null; +type GetNotSeenValidatorsFn = (epoch: Epoch, slot: Slot, committeeIndex: number) => Set | null; type ValidateAttestationDataFn = (attData: phase0.AttestationData) => boolean; @@ -38,14 +69,21 @@ type ValidateAttestationDataFn = (attData: phase0.AttestationData) => boolean; const MAX_RETAINED_ATTESTATIONS_PER_GROUP = 4; /** - * On mainnet, each slot has 64 committees, and each block has 128 attestations max so in average + * Pre-electra, each slot has 64 committees, and each block has 128 attestations max so in average * we get 2 attestation per groups. * Starting from Jan 2024, we have a performance issue getting attestations for a block. Based on the - * fact that lot of groups will have only 1 attestation since it's full of participation increase this number + * fact that lot of groups will have only 1 full participation attestation, increase this number * a bit higher than average. This also help decrease number of slots to search for attestations. */ const MAX_ATTESTATIONS_PER_GROUP = 3; +/** + * For electra, each block has up to 8 aggregated attestations, assuming there are 3 for the "best" + * attestation data, there are still 5 for other attestation data so this constant is still good. + * We should separate to 2 constant based on conditions of different networks + */ +const MAX_ATTESTATIONS_PER_GROUP_ELECTRA = 3; + /** * Maintain a pool of aggregated attestations. Attestations can be retrieved for inclusion in a block * or api. The returned attestations are aggregated to maximise the number of validators that can be @@ -53,20 +91,25 @@ const MAX_ATTESTATIONS_PER_GROUP = 3; * Note that we want to remove attestations with attesters that were included in the chain. */ export class AggregatedAttestationPool { - private readonly attestationGroupByDataHashByIndexBySlot = new MapDef< + /** + * post electra, different committees could have the same AttData and we have to consolidate attestations of the same + * data to be included in block, so we should group by data before index + * // TODO: make sure it does not affect performance for pre electra forks + */ + private readonly attestationGroupByIndexByDataHexBySlot = new MapDef< Slot, - Map> - >(() => new Map>()); + Map> + >(() => new Map>()); private lowestPermissibleSlot = 0; /** For metrics to track size of the pool */ getAttestationCount(): {attestationCount: number; attestationDataCount: number} { let attestationCount = 0; let attestationDataCount = 0; - for (const attestationGroupByDataByIndex of this.attestationGroupByDataHashByIndexBySlot.values()) { - for (const attestationGroupByData of attestationGroupByDataByIndex.values()) { - attestationDataCount += attestationGroupByData.size; - for (const attestationGroup of attestationGroupByData.values()) { + for (const attestationGroupByIndexByDataHex of this.attestationGroupByIndexByDataHexBySlot.values()) { + for (const attestationGroupByIndex of attestationGroupByIndexByDataHex.values()) { + attestationDataCount += attestationGroupByIndex.size; + for (const attestationGroup of attestationGroupByIndex.values()) { attestationCount += attestationGroup.getAttestationCount(); } } @@ -75,7 +118,7 @@ export class AggregatedAttestationPool { } add( - attestation: phase0.Attestation, + attestation: allForks.Attestation, dataRootHex: RootHex, attestingIndicesCount: number, committee: Uint32Array @@ -88,16 +131,24 @@ export class AggregatedAttestationPool { return InsertOutcome.Old; } - const attestationGroupByDataHashByIndex = this.attestationGroupByDataHashByIndexBySlot.getOrDefault(slot); - let attestationGroupByDataHash = attestationGroupByDataHashByIndex.get(attestation.data.index); - if (!attestationGroupByDataHash) { - attestationGroupByDataHash = new Map(); - attestationGroupByDataHashByIndex.set(attestation.data.index, attestationGroupByDataHash); + const attestationGroupByIndexByDataHash = this.attestationGroupByIndexByDataHexBySlot.getOrDefault(slot); + let attestationGroupByIndex = attestationGroupByIndexByDataHash.get(dataRootHex); + if (!attestationGroupByIndex) { + attestationGroupByIndex = new Map(); + attestationGroupByIndexByDataHash.set(dataRootHex, attestationGroupByIndex); + } + const committeeIndex = isElectraAttestation(attestation) + ? // this attestation is added to pool after validation + attestation.committeeBits.getSingleTrueBit() + : attestation.data.index; + if (committeeIndex === null) { + // this should not happen because attestation should be validated before reaching this + throw Error(`Invalid attestation slot=${slot} committeeIndex=${committeeIndex}`); } - let attestationGroup = attestationGroupByDataHash.get(dataRootHex); + let attestationGroup = attestationGroupByIndex.get(committeeIndex); if (!attestationGroup) { attestationGroup = new MatchingDataAttestationGroup(committee, attestation.data); - attestationGroupByDataHash.set(dataRootHex, attestationGroup); + attestationGroupByIndex.set(committeeIndex, attestationGroup); } return attestationGroup.add({ @@ -109,14 +160,25 @@ export class AggregatedAttestationPool { /** Remove attestations which are too old to be included in a block. */ prune(clockSlot: Slot): void { // Only retain SLOTS_PER_EPOCH slots - pruneBySlot(this.attestationGroupByDataHashByIndexBySlot, clockSlot, SLOTS_PER_EPOCH); + pruneBySlot(this.attestationGroupByIndexByDataHexBySlot, clockSlot, SLOTS_PER_EPOCH); this.lowestPermissibleSlot = Math.max(clockSlot - SLOTS_PER_EPOCH, 0); } + getAttestationsForBlock( + fork: ForkName, + forkChoice: IForkChoice, + state: CachedBeaconStateAllForks + ): allForks.Attestation[] { + const forkSeq = ForkSeq[fork]; + return forkSeq >= ForkSeq.electra + ? this.getAttestationsForBlockElectra(fork, forkChoice, state) + : this.getAttestationsForBlockPreElectra(fork, forkChoice, state); + } + /** - * Get attestations to be included in a block. Returns $MAX_ATTESTATIONS items + * Get attestations to be included in a block pre-electra. Returns up to $MAX_ATTESTATIONS items */ - getAttestationsForBlock( + getAttestationsForBlockPreElectra( fork: ForkName, forkChoice: IForkChoice, state: CachedBeaconStateAllForks @@ -130,14 +192,14 @@ export class AggregatedAttestationPool { const attestationsByScore: AttestationWithScore[] = []; - const slots = Array.from(this.attestationGroupByDataHashByIndexBySlot.keys()).sort((a, b) => b - a); + const slots = Array.from(this.attestationGroupByIndexByDataHexBySlot.keys()).sort((a, b) => b - a); let minScore = Number.MAX_SAFE_INTEGER; let slotCount = 0; slot: for (const slot of slots) { slotCount++; - const attestationGroupByDataHashByIndex = this.attestationGroupByDataHashByIndexBySlot.get(slot); + const attestationGroupByIndexByDataHash = this.attestationGroupByIndexByDataHexBySlot.get(slot); // should not happen - if (!attestationGroupByDataHashByIndex) { + if (!attestationGroupByIndexByDataHash) { throw Error(`No aggregated attestation pool for slot=${slot}`); } @@ -158,35 +220,25 @@ export class AggregatedAttestationPool { } const slotDelta = stateSlot - slot; - const shuffling = state.epochCtx.getShufflingAtEpoch(epoch); - const slotCommittees = shuffling.committees[slot % SLOTS_PER_EPOCH]; - for (const [committeeIndex, attestationGroupByData] of attestationGroupByDataHashByIndex.entries()) { - // all attestations will be validated against the state in next step so we can get committee from the state - // this is an improvement to save the notSeenValidatorsFn call for the same slot/index instead of the same attestation data - if (committeeIndex > slotCommittees.length) { - // invalid index, should not happen - continue; - } - - const committee = slotCommittees[committeeIndex]; - const notSeenAttestingIndices = notSeenValidatorsFn(epoch, committee); - if (notSeenAttestingIndices === null || notSeenAttestingIndices.size === 0) { - continue; - } + for (const attestationGroupByIndex of attestationGroupByIndexByDataHash.values()) { + for (const [committeeIndex, attestationGroup] of attestationGroupByIndex.entries()) { + const notSeenAttestingIndices = notSeenValidatorsFn(epoch, slot, committeeIndex); + if (notSeenAttestingIndices === null || notSeenAttestingIndices.size === 0) { + continue; + } - if ( - slotCount > 2 && - attestationsByScore.length >= MAX_ATTESTATIONS && - notSeenAttestingIndices.size / slotDelta < minScore - ) { - // after 2 slots, there are a good chance that we have 2 * MAX_ATTESTATIONS attestations and break the for loop early - // if not, we may have to scan all slots in the pool - // if we have enough attestations and the max possible score is lower than scores of `attestationsByScore`, we should skip - // otherwise it takes time to check attestation, add it and remove it later after the sort by score - continue; - } + if ( + slotCount > 2 && + attestationsByScore.length >= MAX_ATTESTATIONS && + notSeenAttestingIndices.size / slotDelta < minScore + ) { + // after 2 slots, there are a good chance that we have 2 * MAX_ATTESTATIONS attestations and break the for loop early + // if not, we may have to scan all slots in the pool + // if we have enough attestations and the max possible score is lower than scores of `attestationsByScore`, we should skip + // otherwise it takes time to check attestation, add it and remove it later after the sort by score + continue; + } - for (const attestationGroup of attestationGroupByData.values()) { if (!validateAttestationDataFn(attestationGroup.data)) { continue; } @@ -199,6 +251,7 @@ export class AggregatedAttestationPool { // IF they have to be validated, do it only with one attestation per group since same data // The committeeCountPerSlot can be precomputed once per slot for (const {attestation, notSeenAttesterCount} of attestationGroup.getAttestationsForBlock( + fork, notSeenAttestingIndices )) { const score = notSeenAttesterCount / slotDelta; @@ -231,23 +284,134 @@ export class AggregatedAttestationPool { return attestationsForBlock; } + /** + * Get attestations to be included in an electra block. Returns up to $MAX_ATTESTATIONS_ELECTRA items + */ + getAttestationsForBlockElectra( + fork: ForkName, + forkChoice: IForkChoice, + state: CachedBeaconStateAllForks + ): electra.Attestation[] { + const stateSlot = state.slot; + const stateEpoch = state.epochCtx.epoch; + const statePrevEpoch = stateEpoch - 1; + + const notSeenValidatorsFn = getNotSeenValidatorsFn(state); + const validateAttestationDataFn = getValidateAttestationDataFn(forkChoice, state); + + const slots = Array.from(this.attestationGroupByIndexByDataHexBySlot.keys()).sort((a, b) => b - a); + const consolidations: AttestationsConsolidation[] = []; + let minScore = Number.MAX_SAFE_INTEGER; + let slotCount = 0; + slot: for (const slot of slots) { + slotCount++; + const attestationGroupByIndexByDataHash = this.attestationGroupByIndexByDataHexBySlot.get(slot); + // should not happen + if (!attestationGroupByIndexByDataHash) { + throw Error(`No aggregated attestation pool for slot=${slot}`); + } + + const epoch = computeEpochAtSlot(slot); + // validateAttestation condition: Attestation target epoch not in previous or current epoch + if (!(epoch === stateEpoch || epoch === statePrevEpoch)) { + continue; // Invalid attestations + } + // validateAttestation condition: Attestation slot not within inclusion window + if (!(slot + MIN_ATTESTATION_INCLUSION_DELAY <= stateSlot)) { + continue; // Invalid attestations + } + + const slotDelta = stateSlot - slot; + // CommitteeIndex 0 1 2 ... Consolidation + // Attestations att00 --- att10 --- att20 --- 0 (att 00 10 20) + // att01 --- - --- att21 --- 1 (att 01 __ 21) + // - --- - --- att22 --- 2 (att __ __ 22) + for (const attestationGroupByIndex of attestationGroupByIndexByDataHash.values()) { + // sameAttDataCons could be up to MAX_ATTESTATIONS_PER_GROUP_ELECTRA + const sameAttDataCons: AttestationsConsolidation[] = []; + for (const [committeeIndex, attestationGroup] of attestationGroupByIndex.entries()) { + const notSeenAttestingIndices = notSeenValidatorsFn(epoch, slot, committeeIndex); + if (notSeenAttestingIndices === null || notSeenAttestingIndices.size === 0) { + continue; + } + + if ( + slotCount > 2 && + consolidations.length >= MAX_ATTESTATIONS_ELECTRA && + notSeenAttestingIndices.size / slotDelta < minScore + ) { + // after 2 slots, there are a good chance that we have 2 * MAX_ATTESTATIONS_ELECTRA attestations and break the for loop early + // if not, we may have to scan all slots in the pool + // if we have enough attestations and the max possible score is lower than scores of `attestationsByScore`, we should skip + // otherwise it takes time to check attestation, add it and remove it later after the sort by score + continue; + } + + if (!validateAttestationDataFn(attestationGroup.data)) { + continue; + } + + // TODO: Is it necessary to validateAttestation for: + // - Attestation committee index not within current committee count + // - Attestation aggregation bits length does not match committee length + // + // These properties should not change after being validate in gossip + // IF they have to be validated, do it only with one attestation per group since same data + // The committeeCountPerSlot can be precomputed once per slot + for (const [i, attestationNonParticipation] of attestationGroup + .getAttestationsForBlock(fork, notSeenAttestingIndices) + .entries()) { + if (sameAttDataCons[i] === undefined) { + sameAttDataCons[i] = { + byCommittee: new Map(), + attData: attestationNonParticipation.attestation.data, + totalNotSeenCount: 0, + // only update score after we have full data + score: 0, + }; + } + sameAttDataCons[i].byCommittee.set(committeeIndex, attestationNonParticipation); + sameAttDataCons[i].totalNotSeenCount += attestationNonParticipation.notSeenAttesterCount; + } + for (const consolidation of sameAttDataCons) { + const score = consolidation.totalNotSeenCount / slotDelta; + if (score < minScore) { + minScore = score; + } + consolidations.push({...consolidation, score}); + // Stop accumulating attestations there are enough that may have good scoring + if (consolidations.length >= MAX_ATTESTATIONS_ELECTRA * 2) { + break slot; + } + } + } + } + } + + const sortedConsolidationsByScore = consolidations + .sort((a, b) => b.score - a.score) + .slice(0, MAX_ATTESTATIONS_ELECTRA); + // on chain aggregation is expensive, only do it after all + return sortedConsolidationsByScore.map(aggregateConsolidation); + } + /** * Get all attestations optionally filtered by `attestation.data.slot` * @param bySlot slot to filter, `bySlot === attestation.data.slot` */ - getAll(bySlot?: Slot): phase0.Attestation[] { - let attestationGroupsArr: Map[]; + getAll(bySlot?: Slot): allForks.Attestation[] { + let attestationGroupsArr: Map[]; if (bySlot === undefined) { - attestationGroupsArr = Array.from(this.attestationGroupByDataHashByIndexBySlot.values()).flatMap((byIndex) => + attestationGroupsArr = Array.from(this.attestationGroupByIndexByDataHexBySlot.values()).flatMap((byIndex) => Array.from(byIndex.values()) ); } else { - const attestationGroupsByIndex = this.attestationGroupByDataHashByIndexBySlot.get(bySlot); + const attestationGroupsByIndex = this.attestationGroupByIndexByDataHexBySlot.get(bySlot); if (!attestationGroupsByIndex) throw Error(`No attestations for slot ${bySlot}`); attestationGroupsArr = Array.from(attestationGroupsByIndex.values()); } - const attestations: phase0.Attestation[] = []; + const attestations: allForks.Attestation[] = []; for (const attestationGroups of attestationGroupsArr) { for (const attestationGroup of attestationGroups.values()) { attestations.push(...attestationGroup.getAttestations()); @@ -258,12 +422,12 @@ export class AggregatedAttestationPool { } interface AttestationWithIndex { - attestation: phase0.Attestation; + attestation: allForks.Attestation; trueBitsCount: number; } type AttestationNonParticipant = { - attestation: phase0.Attestation; + attestation: allForks.Attestation; // this is <= attestingIndices.count since some attesters may be seen by the chain // this is only updated and used in removeBySeenValidators function notSeenAttesterCount: number; @@ -345,8 +509,9 @@ export class MatchingDataAttestationGroup { * @param notSeenAttestingIndices not seen attestting indices, i.e. indices in the same committee * @returns an array of AttestationNonParticipant */ - getAttestationsForBlock(notSeenAttestingIndices: Set): AttestationNonParticipant[] { + getAttestationsForBlock(fork: ForkName, notSeenAttestingIndices: Set): AttestationNonParticipant[] { const attestations: AttestationNonParticipant[] = []; + const forkSeq = ForkSeq[fork]; for (const {attestation} of this.attestations) { let notSeenAttesterCount = 0; const {aggregationBits} = attestation; @@ -356,22 +521,22 @@ export class MatchingDataAttestationGroup { } } - if (notSeenAttesterCount > 0) { + // if fork >= electra, should return electra-only attestations + if (notSeenAttesterCount > 0 && (forkSeq < ForkSeq.electra || isElectraAttestation(attestation))) { attestations.push({attestation, notSeenAttesterCount}); } } - if (attestations.length <= MAX_ATTESTATIONS_PER_GROUP) { + const maxAttestation = forkSeq >= ForkSeq.electra ? MAX_ATTESTATIONS_PER_GROUP_ELECTRA : MAX_ATTESTATIONS_PER_GROUP; + if (attestations.length <= maxAttestation) { return attestations; } else { - return attestations - .sort((a, b) => b.notSeenAttesterCount - a.notSeenAttesterCount) - .slice(0, MAX_ATTESTATIONS_PER_GROUP); + return attestations.sort((a, b) => b.notSeenAttesterCount - a.notSeenAttesterCount).slice(0, maxAttestation); } } /** Get attestations for API. */ - getAttestations(): phase0.Attestation[] { + getAttestations(): allForks.Attestation[] { return this.attestations.map((attestation) => attestation.attestation); } } @@ -385,6 +550,34 @@ export function aggregateInto(attestation1: AttestationWithIndex, attestation2: attestation1.attestation.signature = aggregateSignatures([signature1, signature2]).toBytes(); } +/** + * Electra and after: Block proposer consolidates attestations with the same + * attestation data from different committee into a single attestation + * https://github.com/ethereum/consensus-specs/blob/aba6345776aa876dad368cab27fbbb23fae20455/specs/_features/eip7549/validator.md?plain=1#L39 + */ +export function aggregateConsolidation({byCommittee, attData}: AttestationsConsolidation): electra.Attestation { + const committeeBits = BitArray.fromBitLen(MAX_COMMITTEES_PER_SLOT); + // TODO: can we improve this? + let aggregationBits: boolean[] = []; + const signatures: Signature[] = []; + const sortedCommittees = Array.from(byCommittee.keys()).sort((a, b) => a - b); + for (const committeeIndex of sortedCommittees) { + const attestationNonParticipation = byCommittee.get(committeeIndex); + if (attestationNonParticipation !== undefined) { + const {attestation} = attestationNonParticipation; + committeeBits.set(committeeIndex, true); + aggregationBits = [...aggregationBits, ...attestation.aggregationBits.toBoolArray()]; + signatures.push(signatureFromBytesNoCheck(attestation.signature)); + } + } + return { + aggregationBits: BitArray.fromBoolArray(aggregationBits), + data: attData, + committeeBits, + signature: aggregateSignatures(signatures).toBytes(), + }; +} + /** * Pre-compute participation from a CachedBeaconStateAllForks, for use to check if an attestation's committee * has already attested or not. @@ -407,12 +600,13 @@ export function getNotSeenValidatorsFn(state: CachedBeaconStateAllForks): GetNot state ); - return (epoch: Epoch, committee: Uint32Array) => { + return (epoch: Epoch, slot: Slot, committeeIndex: number) => { const participants = epoch === stateEpoch ? currentEpochParticipants : epoch === stateEpoch - 1 ? previousEpochParticipants : null; if (participants === null) { return null; } + const committee = state.epochCtx.getBeaconCommittee(slot, committeeIndex); const notSeenAttestingIndices = new Set(); for (const [i, validatorIndex] of committee.entries()) { @@ -434,22 +628,32 @@ export function getNotSeenValidatorsFn(state: CachedBeaconStateAllForks): GetNot const previousParticipation = altairState.previousEpochParticipation.getAll(); const currentParticipation = altairState.currentEpochParticipation.getAll(); const stateEpoch = computeEpochAtSlot(state.slot); + // this function could be called multiple times with same slot + committeeIndex + const cachedNotSeenValidators = new Map>(); - return (epoch: Epoch, committee: Uint32Array) => { + return (epoch: Epoch, slot: Slot, committeeIndex: number) => { const participationStatus = epoch === stateEpoch ? currentParticipation : epoch === stateEpoch - 1 ? previousParticipation : null; if (participationStatus === null) { return null; } + const cacheKey = slot + "_" + committeeIndex; + let notSeenAttestingIndices = cachedNotSeenValidators.get(cacheKey); + if (notSeenAttestingIndices != null) { + // if all validators are seen then return null, we don't need to check for any attestations of same committee again + return notSeenAttestingIndices.size === 0 ? null : notSeenAttestingIndices; + } - const notSeenAttestingIndices = new Set(); + const committee = state.epochCtx.getBeaconCommittee(slot, committeeIndex); + notSeenAttestingIndices = new Set(); for (const [i, validatorIndex] of committee.entries()) { // no need to check flagIsTimelySource as if validator is not seen, it's participation status is 0 if (participationStatus[validatorIndex] === 0) { notSeenAttestingIndices.add(i); } } + cachedNotSeenValidators.set(cacheKey, notSeenAttestingIndices); // if all validators are seen then return null, we don't need to check for any attestations of same committee again return notSeenAttestingIndices.size === 0 ? null : notSeenAttestingIndices; }; diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index 2b511598f9a4..9db73cf4a013 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -1,6 +1,6 @@ import {BitArray} from "@chainsafe/ssz"; import {Signature, aggregateSignatures} from "@chainsafe/blst"; -import {phase0, Slot, RootHex} from "@lodestar/types"; +import {Slot, RootHex, allForks} from "@lodestar/types"; import {MapDef} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; @@ -22,12 +22,16 @@ const SLOTS_RETAINED = 3; */ const MAX_ATTESTATIONS_PER_SLOT = 16_384; -type AggregateFast = { - data: phase0.Attestation["data"]; +type AggregateFastPhase0 = { + data: allForks.Attestation["data"]; aggregationBits: BitArray; signature: Signature; }; +type AggregateFastElectra = AggregateFastPhase0 & {committeeBits: BitArray}; + +type AggregateFast = AggregateFastPhase0 | AggregateFastElectra; + /** Hex string of DataRoot `TODO` */ type DataRootHex = string; @@ -92,7 +96,7 @@ export class AttestationPool { * - Valid committeeIndex * - Valid data */ - add(attestation: phase0.Attestation, attDataRootHex: RootHex): InsertOutcome { + add(attestation: allForks.Attestation, attDataRootHex: RootHex): InsertOutcome { const slot = attestation.data.slot; const lowestPermissibleSlot = this.lowestPermissibleSlot; @@ -127,7 +131,7 @@ export class AttestationPool { /** * For validator API to get an aggregate */ - getAggregate(slot: Slot, dataRootHex: RootHex): phase0.Attestation | null { + getAggregate(slot: Slot, dataRootHex: RootHex): allForks.Attestation | null { const aggregate = this.attestationByRootBySlot.get(slot)?.get(dataRootHex); if (!aggregate) { // TODO: Add metric for missing aggregates @@ -151,8 +155,8 @@ export class AttestationPool { * Get all attestations optionally filtered by `attestation.data.slot` * @param bySlot slot to filter, `bySlot === attestation.data.slot` */ - getAll(bySlot?: Slot): phase0.Attestation[] { - const attestations: phase0.Attestation[] = []; + getAll(bySlot?: Slot): allForks.Attestation[] { + const attestations: allForks.Attestation[] = []; const aggregateByRoots = bySlot === undefined @@ -177,7 +181,7 @@ export class AttestationPool { /** * Aggregate a new contribution into `aggregate` mutating it */ -function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0.Attestation): InsertOutcome { +function aggregateAttestationInto(aggregate: AggregateFast, attestation: allForks.Attestation): InsertOutcome { const bitIndex = attestation.aggregationBits.getSingleTrueBit(); // Should never happen, attestations are verified against this exact condition before @@ -185,6 +189,26 @@ function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0. throw Error("Invalid attestation not exactly one bit set"); } + if ("committeeBits" in attestation && !("committeeBits" in aggregate)) { + throw Error("Attempt to aggregate electra attestation into phase0 attestation"); + } + + if (!("committeeBits" in attestation) && "committeeBits" in aggregate) { + throw Error("Attempt to aggregate phase0 attestation into electra attestation"); + } + + if ("committeeBits" in attestation) { + // We assume attestation.committeeBits should already be validated in api and gossip handler and should be non-null + const attestationCommitteeIndex = attestation.committeeBits.getSingleTrueBit(); + const aggregateCommitteeIndex = (aggregate as AggregateFastElectra).committeeBits.getSingleTrueBit(); + + if (attestationCommitteeIndex !== aggregateCommitteeIndex) { + throw Error( + `Committee index mismatched: attestation ${attestationCommitteeIndex} aggregate ${aggregateCommitteeIndex}` + ); + } + } + if (aggregate.aggregationBits.get(bitIndex) === true) { return InsertOutcome.AlreadyKnown; } @@ -197,7 +221,16 @@ function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0. /** * Format `contribution` into an efficient `aggregate` to add more contributions in with aggregateContributionInto() */ -function attestationToAggregate(attestation: phase0.Attestation): AggregateFast { +function attestationToAggregate(attestation: allForks.Attestation): AggregateFast { + if ("committeeBits" in attestation) { + return { + data: attestation.data, + // clone because it will be mutated + aggregationBits: attestation.aggregationBits.clone(), + committeeBits: attestation.committeeBits, + signature: signatureFromBytesNoCheck(attestation.signature), + }; + } return { data: attestation.data, // clone because it will be mutated @@ -209,10 +242,19 @@ function attestationToAggregate(attestation: phase0.Attestation): AggregateFast /** * Unwrap AggregateFast to phase0.Attestation */ -function fastToAttestation(aggFast: AggregateFast): phase0.Attestation { - return { - data: aggFast.data, - aggregationBits: aggFast.aggregationBits, - signature: aggFast.signature.toBytes(), - }; +function fastToAttestation(aggFast: AggregateFast): allForks.Attestation { + if ("committeeBits" in aggFast) { + return { + data: aggFast.data, + aggregationBits: aggFast.aggregationBits, + committeeBits: aggFast.committeeBits, + signature: aggFast.signature.toBytes(), + }; + } else { + return { + data: aggFast.data, + aggregationBits: aggFast.aggregationBits, + signature: aggFast.signature.toBytes(), + }; + } } diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index a991a6e8630a..8eb3cebcffdb 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -14,6 +14,7 @@ import { BLS_WITHDRAWAL_PREFIX, MAX_ATTESTER_SLASHINGS, ForkSeq, + MAX_ATTESTER_SLASHINGS_ELECTRA, } from "@lodestar/params"; import {Epoch, phase0, capella, ssz, ValidatorIndex, SignedBeaconBlock} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; @@ -174,7 +175,7 @@ export class OpPool { blockType: BlockType, metrics: Metrics | null ): [ - phase0.AttesterSlashing[], + allForks.AttesterSlashing[], phase0.ProposerSlashing[], phase0.SignedVoluntaryExit[], capella.SignedBLSToExecutionChange[], @@ -208,7 +209,8 @@ export class OpPool { }); const endAttesterSlashings = stepsMetrics?.startTimer(); - const attesterSlashings: phase0.AttesterSlashing[] = []; + const attesterSlashings: allForks.AttesterSlashing[] = []; + const maxAttesterSlashing = stateFork >= ForkSeq.electra ? MAX_ATTESTER_SLASHINGS_ELECTRA : MAX_ATTESTER_SLASHINGS; attesterSlashing: for (const attesterSlashing of this.attesterSlashings.values()) { /** Indices slashable in this attester slashing */ const slashableIndices = new Set(); @@ -223,7 +225,7 @@ export class OpPool { if (isSlashableAtEpoch(validator, stateEpoch)) { slashableIndices.add(index); } - if (attesterSlashings.length >= MAX_ATTESTER_SLASHINGS) { + if (attesterSlashings.length >= maxAttesterSlashing) { break attesterSlashing; } } diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 3c32ffe1a9ec..d55b4e300232 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -1,5 +1,5 @@ -import {ForkName} from "@lodestar/params"; -import {phase0, RootHex, ssz} from "@lodestar/types"; +import {ForkName, ForkSeq} from "@lodestar/params"; +import {allForks, electra, phase0, RootHex, ssz} from "@lodestar/types"; import { computeEpochAtSlot, isAggregatorFromCommitteeLength, @@ -20,7 +20,7 @@ import { } from "./attestation.js"; export type AggregateAndProofValidationResult = { - indexedAttestation: phase0.IndexedAttestation; + indexedAttestation: allForks.IndexedAttestation; committeeIndices: Uint32Array; attDataRootHex: RootHex; }; @@ -41,7 +41,7 @@ export async function validateApiAggregateAndProof( export async function validateGossipAggregateAndProof( fork: ForkName, chain: IBeaconChain, - signedAggregateAndProof: phase0.SignedAggregateAndProof, + signedAggregateAndProof: allForks.SignedAggregateAndProof, serializedData: Uint8Array ): Promise { return validateAggregateAndProof(fork, chain, signedAggregateAndProof, serializedData); @@ -50,7 +50,7 @@ export async function validateGossipAggregateAndProof( async function validateAggregateAndProof( fork: ForkName, chain: IBeaconChain, - signedAggregateAndProof: phase0.SignedAggregateAndProof, + signedAggregateAndProof: allForks.SignedAggregateAndProof, serializedData: Uint8Array | null = null, opts: {skipValidationKnownAttesters: boolean; prioritizeBls: boolean} = { skipValidationKnownAttesters: false, @@ -74,7 +74,21 @@ async function validateAggregateAndProof( const attDataBase64 = serializedData ? getAttDataBase64FromSignedAggregateAndProofSerialized(serializedData) : null; const cachedAttData = attDataBase64 ? chain.seenAttestationDatas.get(attSlot, attDataBase64) : null; - const attIndex = attData.index; + let attIndex; + if (ForkSeq[fork] >= ForkSeq.electra) { + attIndex = (aggregate as electra.Attestation).committeeBits.getSingleTrueBit(); + // [REJECT] len(committee_indices) == 1, where committee_indices = get_committee_indices(aggregate) + if (attIndex === null) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_COMMITTEE_BIT_SET}); + } + // [REJECT] aggregate.data.index == 0 + if (attData.index === 0) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NON_ZERO_ATTESTATION_DATA_INDEX}); + } + } else { + attIndex = attData.index; + } + const attEpoch = computeEpochAtSlot(attSlot); const attTarget = attData.target; const targetEpoch = attTarget.epoch; @@ -163,11 +177,16 @@ async function validateAggregateAndProof( throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS}); } const attestingIndices = aggregate.aggregationBits.intersectValues(committeeIndices); - const indexedAttestation: phase0.IndexedAttestation = { + + const indexedAttestationContent = { attestingIndices, data: attData, signature: aggregate.signature, }; + const indexedAttestation = + ForkSeq[fork] >= ForkSeq.electra + ? (indexedAttestationContent as electra.IndexedAttestation) + : (indexedAttestationContent as phase0.IndexedAttestation); // TODO: Check this before regen // [REJECT] The attestation has participants -- that is, diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index a66b95b782e4..813f9a25fc2d 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -1,4 +1,4 @@ -import {phase0, Epoch, Root, Slot, RootHex, ssz} from "@lodestar/types"; +import {phase0, Epoch, Root, Slot, RootHex, ssz, allForks, electra} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq, DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import { @@ -20,6 +20,7 @@ import { AttDataBase64, getAggregationBitsFromAttestationSerialized, getAttDataBase64FromAttestationSerialized, + getCommitteeBitsFromAttestationSerialized, getSignatureFromAttestationSerialized, } from "../../util/sszBytes.js"; import {AttestationDataCacheEntry} from "../seenCache/seenAttestationData.js"; @@ -34,8 +35,8 @@ export type BatchResult = { }; export type AttestationValidationResult = { - attestation: phase0.Attestation; - indexedAttestation: phase0.IndexedAttestation; + attestation: allForks.Attestation; + indexedAttestation: allForks.IndexedAttestation; subnet: number; attDataRootHex: RootHex; }; @@ -43,7 +44,7 @@ export type AttestationValidationResult = { export type AttestationOrBytes = ApiAttestation | GossipAttestation; /** attestation from api */ -export type ApiAttestation = {attestation: phase0.Attestation; serializedData: null}; +export type ApiAttestation = {attestation: phase0.Attestation; serializedData: null}; // TODO Electra: add new attestation type /** attestation from gossip */ export type GossipAttestation = { @@ -248,7 +249,7 @@ async function validateGossipAttestationNoSignatureCheck( // Run the checks that happen before an indexed attestation is constructed. let attestationOrCache: - | {attestation: phase0.Attestation; cache: null} + | {attestation: allForks.Attestation; cache: null} | {attestation: null; cache: AttestationDataCacheEntry; serializedData: Uint8Array}; let attDataBase64: AttDataBase64 | null = null; if (attestationOrBytes.serializedData) { @@ -260,7 +261,7 @@ async function validateGossipAttestationNoSignatureCheck( attestationOrBytes.attDataBase64 ?? getAttDataBase64FromAttestationSerialized(attestationOrBytes.serializedData); const cachedAttData = attDataBase64 !== null ? chain.seenAttestationDatas.get(attSlot, attDataBase64) : null; if (cachedAttData === null) { - const attestation = sszDeserializeAttestation(attestationOrBytes.serializedData); + const attestation = sszDeserializeAttestation(fork, attestationOrBytes.serializedData); // only deserialize on the first AttestationData that's not cached attestationOrCache = {attestation, cache: null}; } else { @@ -276,11 +277,34 @@ async function validateGossipAttestationNoSignatureCheck( ? attestationOrCache.attestation.data : attestationOrCache.cache.attestationData; const attSlot = attData.slot; - const attIndex = attData.index; const attEpoch = computeEpochAtSlot(attSlot); const attTarget = attData.target; const targetEpoch = attTarget.epoch; + let attIndex; + if (ForkSeq[fork] >= ForkSeq.electra) { + const committeeBits = attestationOrCache.attestation + ? (attestationOrCache.attestation as electra.Attestation).committeeBits + : getCommitteeBitsFromAttestationSerialized(attestationOrCache.serializedData); + + if (committeeBits === null) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.INVALID_SERIALIZED_BYTES}); + } + + attIndex = committeeBits.getSingleTrueBit(); + // [REJECT] len(committee_indices) == 1, where committee_indices = get_committee_indices(aggregate) + if (attIndex === null) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_COMMITTEE_BIT_SET}); + } + + // [REJECT] aggregate.data.index == 0 + if (attData.index === 0) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NON_ZERO_ATTESTATION_DATA_INDEX}); + } + } else { + attIndex = attData.index; + } + chain.metrics?.gossipAttestation.attestationSlotToClockSlot.observe( {caller: RegenCaller.validateGossipAttestation}, chain.clock.currentSlot - attSlot @@ -452,13 +476,17 @@ async function validateGossipAttestationNoSignatureCheck( } // no signature check, leave that for step1 - const indexedAttestation: phase0.IndexedAttestation = { + const indexedAttestationContent = { attestingIndices, data: attData, signature, }; + const indexedAttestation = + ForkSeq[fork] >= ForkSeq.electra + ? (indexedAttestationContent as electra.IndexedAttestation) + : (indexedAttestationContent as phase0.IndexedAttestation); - const attestation: phase0.Attestation = attestationOrCache.attestation + const attestation: allForks.Attestation = attestationOrCache.attestation ? attestationOrCache.attestation : { aggregationBits, @@ -698,6 +726,10 @@ function verifyAttestationTargetRoot(headBlock: ProtoBlock, targetRoot: Root, at } } +/** + * Get a list of indices of validators in the given committee + * attestationIndex - Index of the committee in shuffling.committees + */ export function getCommitteeIndices( shuffling: EpochShuffling, attestationSlot: Slot, diff --git a/packages/beacon-node/src/chain/validation/attesterSlashing.ts b/packages/beacon-node/src/chain/validation/attesterSlashing.ts index 818812526fb3..11a499c9bb53 100644 --- a/packages/beacon-node/src/chain/validation/attesterSlashing.ts +++ b/packages/beacon-node/src/chain/validation/attesterSlashing.ts @@ -9,7 +9,7 @@ import {AttesterSlashingError, AttesterSlashingErrorCode, GossipAction} from ".. export async function validateApiAttesterSlashing( chain: IBeaconChain, - attesterSlashing: phase0.AttesterSlashing + attesterSlashing: phase0.AttesterSlashing // TODO Electra: Handle electra.AttesterSlashing ): Promise { const prioritizeBls = true; return validateAttesterSlashing(chain, attesterSlashing, prioritizeBls); diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index 5492f57edc02..a443bf761c93 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -13,7 +13,7 @@ import {BeaconBlock, RootHex, altair, deneb} from "@lodestar/types"; import {ChainConfig, ChainForkConfig} from "@lodestar/config"; import {ForkSeq, INTERVALS_PER_SLOT, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, Slot, ValidatorIndex} from "@lodestar/types"; -import {IndexedAttestation, SignedAggregateAndProof} from "@lodestar/types/phase0"; +import {IndexedAttestation, SignedAggregateAndProof} from "@lodestar/types/allForks"; import {GENESIS_SLOT} from "../constants/constants.js"; import {LodestarMetrics} from "./metrics/lodestar.js"; diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index 25a871b4e2a0..649bfd455387 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -80,8 +80,8 @@ export type SSZTypeOfGossipTopic = T extends {type: infer export type GossipTypeMap = { [GossipType.beacon_block]: SignedBeaconBlock; [GossipType.blob_sidecar]: deneb.BlobSidecar; - [GossipType.beacon_aggregate_and_proof]: phase0.SignedAggregateAndProof; - [GossipType.beacon_attestation]: phase0.Attestation; + [GossipType.beacon_aggregate_and_proof]: allForks.SignedAggregateAndProof; + [GossipType.beacon_attestation]: allForks.Attestation; [GossipType.voluntary_exit]: phase0.SignedVoluntaryExit; [GossipType.proposer_slashing]: phase0.ProposerSlashing; [GossipType.attester_slashing]: phase0.AttesterSlashing; @@ -95,8 +95,10 @@ export type GossipTypeMap = { export type GossipFnByType = { [GossipType.beacon_block]: (signedBlock: SignedBeaconBlock) => Promise | void; [GossipType.blob_sidecar]: (blobSidecar: deneb.BlobSidecar) => Promise | void; - [GossipType.beacon_aggregate_and_proof]: (aggregateAndProof: phase0.SignedAggregateAndProof) => Promise | void; - [GossipType.beacon_attestation]: (attestation: phase0.Attestation) => Promise | void; + [GossipType.beacon_aggregate_and_proof]: ( + aggregateAndProof: allForks.SignedAggregateAndProof + ) => Promise | void; + [GossipType.beacon_attestation]: (attestation: allForks.Attestation) => Promise | void; [GossipType.voluntary_exit]: (voluntaryExit: phase0.SignedVoluntaryExit) => Promise | void; [GossipType.proposer_slashing]: (proposerSlashing: phase0.ProposerSlashing) => Promise | void; [GossipType.attester_slashing]: (attesterSlashing: phase0.AttesterSlashing) => Promise | void; diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index 4923b71e6887..ffbad28cd936 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -1,4 +1,4 @@ -import {phase0, ssz, sszTypesFor} from "@lodestar/types"; +import {ssz, sszTypesFor} from "@lodestar/types"; import {ForkDigestContext} from "@lodestar/config"; import { ATTESTATION_SUBNET_COUNT, @@ -87,9 +87,9 @@ export function getGossipSSZType(topic: GossipTopic) { case GossipType.blob_sidecar: return ssz.deneb.BlobSidecar; case GossipType.beacon_aggregate_and_proof: - return ssz.phase0.SignedAggregateAndProof; + return ssz.allForks[topic.fork].SignedAggregateAndProof; case GossipType.beacon_attestation: - return ssz.phase0.Attestation; + return ssz.allForks[topic.fork].Attestation; case GossipType.proposer_slashing: return ssz.phase0.ProposerSlashing; case GossipType.attester_slashing: @@ -128,9 +128,10 @@ export function sszDeserialize(topic: T, serializedData: /** * Deserialize a gossip serialized data into an Attestation object. */ -export function sszDeserializeAttestation(serializedData: Uint8Array): phase0.Attestation { +export function sszDeserializeAttestation(fork: ForkName, serializedData: Uint8Array): allForks.Attestation { + const sszType = ssz.allForks[fork].Attestation; try { - return ssz.phase0.Attestation.deserialize(serializedData); + return sszType.deserialize(serializedData); } catch (e) { throw new GossipActionError(GossipAction.REJECT, {code: GossipErrorCode.INVALID_SERIALIZED_BYTES_ERROR_CODE}); } diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 5012650e229a..4fd1235fe91a 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -71,7 +71,7 @@ export interface INetwork extends INetworkCorePublic { // Gossip publishBeaconBlock(signedBlock: SignedBeaconBlock): Promise; publishBlobSidecar(blobSidecar: deneb.BlobSidecar): Promise; - publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise; + publishBeaconAggregateAndProof(aggregateAndProof: allForks.SignedAggregateAndProof): Promise; publishBeaconAttestation(attestation: phase0.Attestation, subnet: number): Promise; publishVoluntaryExit(voluntaryExit: phase0.SignedVoluntaryExit): Promise; publishBlsToExecutionChange(blsToExecutionChange: capella.SignedBLSToExecutionChange): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 52b9d85c0064..f9550d4c0ec2 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -316,7 +316,7 @@ export class Network implements INetwork { }); } - async publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise { + async publishBeaconAggregateAndProof(aggregateAndProof: allForks.SignedAggregateAndProof): Promise { const fork = this.config.getForkName(aggregateAndProof.message.aggregate.data.slot); return this.publishGossip( {type: GossipType.beacon_aggregate_and_proof, fork}, diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 6f64148b87e2..57697ba2643f 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -421,7 +421,11 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler validationResult = await validateGossipAggregateAndProof(fork, chain, signedAggregateAndProof, serializedData); } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject"); + chain.persistInvalidSszValue( + ssz.allForks[fork].SignedAggregateAndProof, + signedAggregateAndProof, + "gossip_reject" + ); } throw e; } @@ -450,7 +454,10 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } } - chain.emitter.emit(routes.events.EventType.attestation, signedAggregateAndProof.message.aggregate); + chain.emitter.emit(routes.events.EventType.attestation, { + version: fork, + data: signedAggregateAndProof.message.aggregate, + }); }, [GossipType.beacon_attestation]: async ({ gossipData, @@ -502,7 +509,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } } - chain.emitter.emit(routes.events.EventType.attestation, attestation); + chain.emitter.emit(routes.events.EventType.attestation, {version: fork, data: attestation}); }, [GossipType.attester_slashing]: async ({ @@ -522,7 +529,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler logger.error("Error adding attesterSlashing to pool", {}, e as Error); } - chain.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing); + chain.emitter.emit(routes.events.EventType.attesterSlashing, {version: topic.fork, data: attesterSlashing}); }, [GossipType.proposer_slashing]: async ({ @@ -710,7 +717,7 @@ function getBatchHandlers(modules: ValidatorFnsModules, options: GossipHandlerOp } } - chain.emitter.emit(routes.events.EventType.attestation, attestation); + chain.emitter.emit(routes.events.EventType.attestation, {version: fork, data: attestation}); } if (batchableBls) { diff --git a/packages/beacon-node/test/unit/api/impl/events/events.test.ts b/packages/beacon-node/test/unit/api/impl/events/events.test.ts index e031c3ac9958..b1f85b5e6e44 100644 --- a/packages/beacon-node/test/unit/api/impl/events/events.test.ts +++ b/packages/beacon-node/test/unit/api/impl/events/events.test.ts @@ -2,6 +2,7 @@ import {describe, it, expect, beforeEach, afterEach, vi, MockedObject} from "vit import {routes} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; +import {ForkName} from "@lodestar/params"; import {BeaconChain, ChainEventEmitter, HeadEventData} from "../../../../../src/chain/index.js"; import {getEventsApi} from "../../../../../src/api/impl/events/index.js"; import {ZERO_HASH_HEX} from "../../../../../src/constants/constants.js"; @@ -66,7 +67,10 @@ describe("Events api impl", function () { it("should ignore not sent topics", async function () { const events = getEvents([routes.events.EventType.head]); - chainEventEmmitter.emit(routes.events.EventType.attestation, ssz.phase0.Attestation.defaultValue()); + chainEventEmmitter.emit(routes.events.EventType.attestation, { + version: ForkName.phase0, + data: ssz.phase0.Attestation.defaultValue(), + }); chainEventEmmitter.emit(routes.events.EventType.head, headEventData); expect(events).toHaveLength(1); diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 800984fa84bc..57b5d6fad00f 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -2,13 +2,21 @@ import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {describe, it, expect, beforeEach, beforeAll, afterEach, vi} from "vitest"; import {SecretKey, Signature, fastAggregateVerify} from "@chainsafe/blst"; import {CachedBeaconStateAllForks, newFilledArray} from "@lodestar/state-transition"; -import {FAR_FUTURE_EPOCH, ForkName, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@lodestar/params"; +import { + FAR_FUTURE_EPOCH, + ForkName, + MAX_COMMITTEES_PER_SLOT, + MAX_EFFECTIVE_BALANCE, + SLOTS_PER_EPOCH, +} from "@lodestar/params"; import {ssz, phase0} from "@lodestar/types"; import {CachedBeaconStateAltair} from "@lodestar/state-transition/src/types.js"; import {MockedForkChoice, getMockedForkChoice} from "../../../mocks/mockedBeaconChain.js"; import { + aggregateConsolidation, AggregatedAttestationPool, aggregateInto, + AttestationsConsolidation, getNotSeenValidatorsFn, MatchingDataAttestationGroup, } from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; @@ -80,11 +88,11 @@ describe("AggregatedAttestationPool", function () { vi.clearAllMocks(); }); - it("getParticipationFn", () => { + it("getNotSeenValidatorsFn", () => { // previousEpochParticipation and currentEpochParticipation is created inside generateCachedState // 0 and 1 are fully participated const notSeenValidatorFn = getNotSeenValidatorsFn(altairState); - const participation = notSeenValidatorFn(currentEpoch, committee); + const participation = notSeenValidatorFn(currentEpoch, currentSlot, committeeIndex); // seen attesting indices are 0, 1 => not seen are 2, 3 expect(participation).toEqual( // { @@ -279,6 +287,7 @@ describe("MatchingDataAttestationGroup.getAttestationsForBlock", () => { } } const attestationsForBlock = attestationGroup.getAttestationsForBlock( + ForkName.phase0, // notSeenValidatorIndices, notSeenAttestingIndices ); @@ -320,3 +329,75 @@ describe("MatchingDataAttestationGroup aggregateInto", function () { ); }); }); + +describe("aggregateConsolidation", function () { + const sk0 = bls.SecretKey.fromBytes(Buffer.alloc(32, 1)); + const sk1 = bls.SecretKey.fromBytes(Buffer.alloc(32, 2)); + const sk2 = bls.SecretKey.fromBytes(Buffer.alloc(32, 3)); + const skArr = [sk0, sk1, sk2]; + const testCases: { + name: string; + committeeIndices: number[]; + aggregationBitsArr: Array[]; + expectedAggregationBits: Array; + expectedCommitteeBits: Array; + }[] = [ + // note that bit index starts from the right + { + name: "test case 0", + committeeIndices: [0, 1, 2], + aggregationBitsArr: [[0b111], [0b011], [0b111]], + expectedAggregationBits: [0b11011111, 0b1], + expectedCommitteeBits: [true, true, true, false], + }, + { + name: "test case 1", + committeeIndices: [2, 3, 1], + aggregationBitsArr: [[0b100], [0b010], [0b001]], + expectedAggregationBits: [0b10100001, 0b0], + expectedCommitteeBits: [false, true, true, true], + }, + ]; + for (const { + name, + committeeIndices, + aggregationBitsArr, + expectedAggregationBits, + expectedCommitteeBits, + } of testCases) { + it(name, () => { + const attData = ssz.phase0.AttestationData.defaultValue(); + const consolidation: AttestationsConsolidation = { + byCommittee: new Map(), + attData: attData, + totalNotSeenCount: 0, + score: 0, + }; + // to simplify, instead of signing the signingRoot, just sign the attData root + const sigArr = skArr.map((sk) => sk.sign(ssz.phase0.AttestationData.hashTreeRoot(attData))); + const attestationSeed = ssz.electra.Attestation.defaultValue(); + for (let i = 0; i < committeeIndices.length; i++) { + const committeeIndex = committeeIndices[i]; + const commiteeBits = BitArray.fromBoolArray( + Array.from({length: MAX_COMMITTEES_PER_SLOT}, (_, i) => i === committeeIndex) + ); + const aggAttestation = { + ...attestationSeed, + aggregationBits: new BitArray(new Uint8Array(aggregationBitsArr[i]), 3), + committeeBits: commiteeBits, + signature: sigArr[i].toBytes(), + }; + consolidation.byCommittee.set(committeeIndex, { + attestation: aggAttestation, + notSeenAttesterCount: aggregationBitsArr[i].filter((item) => item).length, + }); + } + + const finalAttestation = aggregateConsolidation(consolidation); + expect(finalAttestation.aggregationBits.uint8Array).toEqual(new Uint8Array(expectedAggregationBits)); + expect(finalAttestation.committeeBits.toBoolArray()).toEqual(expectedCommitteeBits); + expect(finalAttestation.data).toEqual(attData); + expect(finalAttestation.signature).toEqual(bls.Signature.aggregate(sigArr).toBytes()); + }); + } +}); diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index d0629c2125cc..c9d1aa627b03 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -156,7 +156,7 @@ export interface IForkChoice { * The supplied `attestation` **must** pass the `in_valid_indexed_attestation` function as it * will not be run here. */ - onAttestation(attestation: phase0.IndexedAttestation, attDataRoot: string, forceImport?: boolean): void; + onAttestation(attestation: allForks.IndexedAttestation, attDataRoot: string, forceImport?: boolean): void; /** * Register attester slashing in order not to consider their votes in `getHead` * diff --git a/packages/state-transition/src/signatureSets/attesterSlashings.ts b/packages/state-transition/src/signatureSets/attesterSlashings.ts index 256582afe368..10e7a1991e4f 100644 --- a/packages/state-transition/src/signatureSets/attesterSlashings.ts +++ b/packages/state-transition/src/signatureSets/attesterSlashings.ts @@ -1,4 +1,4 @@ -import {SignedBeaconBlock, phase0, ssz} from "@lodestar/types"; +import {SignedBeaconBlock, ssz} from "@lodestar/types"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import {computeSigningRoot, computeStartSlotAtEpoch, ISignatureSet, SignatureSetType} from "../util/index.js"; import {CachedBeaconStateAllForks} from "../types.js"; @@ -16,7 +16,7 @@ export function getAttesterSlashingsSignatureSets( /** Get signature sets from a single AttesterSlashing object */ export function getAttesterSlashingSignatureSets( state: CachedBeaconStateAllForks, - attesterSlashing: phase0.AttesterSlashing + attesterSlashing: allForks.AttesterSlashing ): ISignatureSet[] { return [attesterSlashing.attestation1, attesterSlashing.attestation2].map((attestation) => getIndexedAttestationBigintSignatureSet(state, attestation) @@ -25,7 +25,7 @@ export function getAttesterSlashingSignatureSets( export function getIndexedAttestationBigintSignatureSet( state: CachedBeaconStateAllForks, - indexedAttestation: phase0.IndexedAttestationBigint + indexedAttestation: allForks.IndexedAttestationBigint ): ISignatureSet { const slot = computeStartSlotAtEpoch(Number(indexedAttestation.data.target.epoch as bigint)); const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_ATTESTER, slot); diff --git a/packages/state-transition/src/signatureSets/index.ts b/packages/state-transition/src/signatureSets/index.ts index 140b607c0bcb..5f063235735c 100644 --- a/packages/state-transition/src/signatureSets/index.ts +++ b/packages/state-transition/src/signatureSets/index.ts @@ -33,6 +33,9 @@ export function getBlockSignatureSets( skipProposerSignature?: boolean; } ): ISignatureSet[] { + // fork based validations + const fork = state.config.getForkSeq(signedBlock.message.slot); + const signatureSets = [ getRandaoRevealSignatureSet(state, signedBlock.message), ...getProposerSlashingsSignatureSets(state, signedBlock), @@ -45,9 +48,6 @@ export function getBlockSignatureSets( signatureSets.push(getBlockProposerSignatureSet(state, signedBlock)); } - // fork based validations - const fork = state.config.getForkSeq(signedBlock.message.slot); - // Only after altair fork, validate tSyncCommitteeSignature if (fork >= ForkSeq.altair) { const syncCommitteeSignatureSet = getSyncCommitteeSignatureSet( diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 9191b251a6de..31eb9e662e8c 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,8 +1,9 @@ import {toHexString} from "@chainsafe/ssz"; -import {BLSSignature, phase0, Slot, ssz} from "@lodestar/types"; +import {allForks, BLSSignature, phase0, Slot, ssz} from "@lodestar/types"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; +import {ChainForkConfig} from "@lodestar/config"; import {IClock, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; @@ -43,6 +44,7 @@ export class AttestationService { chainHeadTracker: ChainHeaderTracker, syncingStatusTracker: SyncingStatusTracker, private readonly metrics: Metrics | null, + private readonly config: ChainForkConfig, private readonly opts?: AttestationServiceOpts ) { this.dutiesService = new AttestationDutiesService( @@ -271,7 +273,7 @@ export class AttestationService { const aggregate = res.value(); this.metrics?.numParticipantsInAggregate.observe(aggregate.aggregationBits.getTrueBitIndexes().length); - const signedAggregateAndProofs: phase0.SignedAggregateAndProof[] = []; + const signedAggregateAndProofs: allForks.SignedAggregateAndProof[] = []; await Promise.all( duties.map(async ({duty, selectionProof}) => { diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 53299463ad2f..4678f6188382 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -19,6 +19,7 @@ import { DOMAIN_SYNC_COMMITTEE, DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_APPLICATION_BUILDER, + ForkSeq, } from "@lodestar/params"; import { altair, @@ -27,6 +28,7 @@ import { BlindedBeaconBlock, BLSPubkey, BLSSignature, + electra, Epoch, phase0, Root, @@ -493,7 +495,7 @@ export class ValidatorStore { duty: routes.validator.AttesterDuty, attestationData: phase0.AttestationData, currentEpoch: Epoch - ): Promise { + ): Promise { // Make sure the target epoch is not higher than the current epoch to avoid potential attacks. if (attestationData.target.epoch > currentEpoch) { throw Error( @@ -525,21 +527,30 @@ export class ValidatorStore { data: attestationData, }; - return { - aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), - data: attestationData, - signature: await this.getSignature(duty.pubkey, signingRoot, signingSlot, signableMessage), - }; + if (this.config.getForkSeq(duty.slot) >= ForkSeq.electra) { + return { + aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), + data: attestationData, + committeeBits: BitArray.fromSingleBit(duty.committeesAtSlot, duty.committeeIndex), + signature: await this.getSignature(duty.pubkey, signingRoot, signingSlot, signableMessage), + } as electra.Attestation; + } else { + return { + aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), + data: attestationData, + signature: await this.getSignature(duty.pubkey, signingRoot, signingSlot, signableMessage), + } as phase0.Attestation; + } } async signAggregateAndProof( duty: routes.validator.AttesterDuty, selectionProof: BLSSignature, - aggregate: phase0.Attestation - ): Promise { + aggregate: allForks.Attestation + ): Promise { this.validateAttestationDuty(duty, aggregate.data); - const aggregateAndProof: phase0.AggregateAndProof = { + const aggregateAndProof: allForks.AggregateAndProof = { aggregate, aggregatorIndex: duty.validatorIndex, selectionProof, @@ -547,7 +558,10 @@ export class ValidatorStore { const signingSlot = aggregate.data.slot; const domain = this.config.getDomain(signingSlot, DOMAIN_AGGREGATE_AND_PROOF); - const signingRoot = computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof, domain); + const signingRoot = + this.config.getForkSeq(duty.slot) >= ForkSeq.electra + ? computeSigningRoot(ssz.electra.AggregateAndProof, aggregateAndProof, domain) + : computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof, domain); const signableMessage: SignableMessage = { type: SignableMessageType.AGGREGATE_AND_PROOF, @@ -788,6 +802,9 @@ export class ValidatorStore { `Inconsistent duties during signing: duty.committeeIndex ${duty.committeeIndex} != att.committeeIndex ${data.index}` ); } + if (this.config.getForkSeq(duty.slot) >= ForkSeq.electra && data.index !== 0) { + throw Error(`Attestataion data index must be 0 post electra: index ${data.index}`); + } } private assertDoppelgangerSafe(pubKey: PubkeyHex | BLSPubkey): void { diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 1732f54ababf..8ec8f3330af2 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -241,6 +241,7 @@ export class Validator { chainHeaderTracker, syncingStatusTracker, metrics, + config, { afterBlockDelaySlotFraction: opts.afterBlockDelaySlotFraction, disableAttestationGrouping: opts.disableAttestationGrouping || opts.distributed, diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index 11779ee496b3..f02b15f38853 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -3,6 +3,8 @@ import {toHexString} from "@chainsafe/ssz"; import {SecretKey} from "@chainsafe/blst"; import {ssz} from "@lodestar/types"; import {routes} from "@lodestar/api"; +import {createChainForkConfig} from "@lodestar/config"; +import {config} from "@lodestar/config/default"; import {AttestationService, AttestationServiceOpts} from "../../../src/services/attestation.js"; import {AttDutyAndProof} from "../../../src/services/attestationDuties.js"; import {ValidatorStore} from "../../../src/services/validatorStore.js"; @@ -68,6 +70,7 @@ describe("AttestationService", function () { chainHeadTracker, syncingStatusTracker, null, + createChainForkConfig(config), opts ); From 09e751077b1c8217fad31c9c220c575e147203d6 Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 8 May 2024 18:09:39 +0700 Subject: [PATCH 033/145] fix: attestation pool for electra (#6744) * feat: attestationPool to group by slot by data root by committee index for electra * fix: gossip validation and assert.notNull() util * fix: remove light-client stats.html * fix: lint and check-types --- packages/api/src/beacon/routes/validator.ts | 16 +-- .../api/test/unit/beacon/testData/events.ts | 98 ++++++++++--------- .../test/unit/beacon/testData/validator.ts | 4 +- .../src/api/impl/beacon/pool/index.ts | 3 +- .../src/api/impl/validator/index.ts | 11 ++- .../opPools/aggregatedAttestationPool.ts | 8 +- .../src/chain/opPools/attestationPool.ts | 79 ++++++--------- .../src/chain/validation/aggregateAndProof.ts | 2 +- .../src/chain/validation/attestation.ts | 4 +- .../test/perf/epoch/epochAltair.test.ts | 4 +- .../test/perf/epoch/epochCapella.test.ts | 4 +- .../test/perf/epoch/epochPhase0.test.ts | 4 +- .../processEffectiveBalanceUpdates.test.ts | 3 +- .../perf/epoch/processRegistryUpdates.test.ts | 3 +- packages/utils/src/assert.ts | 12 +++ packages/utils/test/unit/assert.test.ts | 12 ++- .../validator/src/services/attestation.ts | 8 +- .../validator/src/services/validatorStore.ts | 3 +- 18 files changed, 149 insertions(+), 129 deletions(-) diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index f37ec7b24488..8c8355f63d07 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -409,9 +409,9 @@ export type Endpoints = { /** HashTreeRoot of AttestationData that validator want's aggregated */ attestationDataRoot: Root; slot: Slot; - index: number; + committeeIndex: number; }, - {query: {attestation_data_root: string; slot: number; index: number}}, + {query: {attestation_data_root: string; slot: number; committeeIndex: number}}, allForks.Attestation, VersionMeta >; @@ -805,16 +805,20 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - query: {attestation_data_root: toHexString(attestationDataRoot), slot, index}, + writeReq: ({attestationDataRoot, slot, committeeIndex}) => ({ + query: {attestation_data_root: toHexString(attestationDataRoot), slot, committeeIndex}, }), parseReq: ({query}) => ({ attestationDataRoot: fromHexString(query.attestation_data_root), slot: query.slot, - index: query.slot, + committeeIndex: query.slot, }), schema: { - query: {attestation_data_root: Schema.StringRequired, slot: Schema.UintRequired, index: Schema.UintRequired}, + query: { + attestation_data_root: Schema.StringRequired, + slot: Schema.UintRequired, + committeeIndex: Schema.UintRequired, + }, }, }, resp: { diff --git a/packages/api/test/unit/beacon/testData/events.ts b/packages/api/test/unit/beacon/testData/events.ts index 8a7610a26836..7966d0ea3b8a 100644 --- a/packages/api/test/unit/beacon/testData/events.ts +++ b/packages/api/test/unit/beacon/testData/events.ts @@ -31,18 +31,21 @@ export const eventTestData: EventData = { block: "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf", executionOptimistic: false, }, - [EventType.attestation]: ssz.phase0.Attestation.fromJson({ - aggregation_bits: "0x01", - signature: - "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", - data: { - slot: "1", - index: "1", - beacon_block_root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", - source: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, - target: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, - }, - }), + [EventType.attestation]: { + version: ForkName.altair, + data: ssz.phase0.Attestation.fromJson({ + aggregation_bits: "0x01", + signature: + "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", + data: { + slot: "1", + index: "1", + beacon_block_root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", + source: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, + target: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, + }, + }), + }, [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit.fromJson({ message: {epoch: "1", validator_index: "1"}, signature: @@ -72,44 +75,47 @@ export const eventTestData: EventData = { "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", }, }), - [EventType.attesterSlashing]: ssz.phase0.AttesterSlashing.fromJson({ - attestation_1: { - attesting_indices: ["0", "1"], - data: { - slot: "0", - index: "0", - beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", - source: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", - }, - target: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", + [EventType.attesterSlashing]: { + version: ForkName.altair, + data: ssz.phase0.AttesterSlashing.fromJson({ + attestation_1: { + attesting_indices: ["0", "1"], + data: { + slot: "0", + index: "0", + beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", + source: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + target: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, }, + signature: + "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", }, - signature: - "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - }, - attestation_2: { - attesting_indices: ["0", "1"], - data: { - slot: "0", - index: "0", - beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", - source: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", - }, - target: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", + attestation_2: { + attesting_indices: ["0", "1"], + data: { + slot: "0", + index: "0", + beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", + source: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + target: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, }, + signature: + "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", }, - signature: - "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - }, - }), + }), + }, [EventType.blsToExecutionChange]: ssz.capella.SignedBLSToExecutionChange.fromJson({ message: { validator_index: "1", diff --git a/packages/api/test/unit/beacon/testData/validator.ts b/packages/api/test/unit/beacon/testData/validator.ts index 11fd7dd26425..5f82d4a818b0 100644 --- a/packages/api/test/unit/beacon/testData/validator.ts +++ b/packages/api/test/unit/beacon/testData/validator.ts @@ -99,8 +99,8 @@ export const testData: GenericServerTestCases = { res: {data: ssz.altair.SyncCommitteeContribution.defaultValue()}, }, getAggregatedAttestation: { - args: {attestationDataRoot: ZERO_HASH, slot: 32000}, - res: {data: ssz.phase0.Attestation.defaultValue()}, + args: {attestationDataRoot: ZERO_HASH, slot: 32000, index: 2}, + res: {data: ssz.phase0.Attestation.defaultValue(), meta: {version: ForkName.phase0}}, }, publishAggregateAndProofs: { args: {signedAggregateAndProofs: [ssz.phase0.SignedAggregateAndProof.defaultValue()]}, diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 77f6b24f28af..3f594fa4c3ae 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -27,12 +27,13 @@ export function getBeaconPoolApi({ async getPoolAttestations({slot, committeeIndex}) { // Already filtered by slot let attestations = chain.aggregatedAttestationPool.getAll(slot); + const fork = chain.config.getForkName(slot ?? attestations[0].data.slot) ?? ForkName.phase0; if (committeeIndex !== undefined) { attestations = attestations.filter((attestation) => committeeIndex === attestation.data.index); } - return {data: attestations}; + return {data: attestations, meta: {version: fork}}; }, async getPoolAttesterSlashings() { diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 63f860e7a193..856b301d7a21 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1072,23 +1072,26 @@ export function getValidatorApi( }; }, - async getAggregatedAttestation({attestationDataRoot, slot}) { + async getAggregatedAttestation({attestationDataRoot, slot, committeeIndex}) { notWhileSyncing(); await waitForSlot(slot); // Must never request for a future slot > currentSlot const dataRootHex = toRootHex(attestationDataRoot); - const aggregate = chain.attestationPool.getAggregate(slot, dataRootHex); + const aggregate = chain.attestationPool.getAggregate(slot, committeeIndex, dataRootHex); if (!aggregate) { - throw new ApiError(404, `No aggregated attestation for slot=${slot}, dataRoot=${dataRootHex}`); + throw new ApiError( + 404, + `No aggregated attestation for slot=${slot} committeeIndex=${committeeIndex}, dataRoot=${dataRootHex}` + ); } metrics?.production.producedAggregateParticipants.observe(aggregate.aggregationBits.getTrueBitIndexes().length); return { data: aggregate, - version: config.getForkName(slot), + meta: {version: config.getForkName(slot)}, }; }, diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index d5368bcfe4fe..31d461766947 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -30,7 +30,7 @@ import { getBlockRootAtSlot, } from "@lodestar/state-transition"; import {IForkChoice, EpochDifference} from "@lodestar/fork-choice"; -import {MapDef, toRootHex} from "@lodestar/utils"; +import {MapDef, toRootHex, assert} from "@lodestar/utils"; import {intersectUint8Arrays, IntersectResult} from "../../util/bitArray.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; import {InsertOutcome} from "./types.js"; @@ -141,10 +141,8 @@ export class AggregatedAttestationPool { ? // this attestation is added to pool after validation attestation.committeeBits.getSingleTrueBit() : attestation.data.index; - if (committeeIndex === null) { - // this should not happen because attestation should be validated before reaching this - throw Error(`Invalid attestation slot=${slot} committeeIndex=${committeeIndex}`); - } + // this should not happen because attestation should be validated before reaching this + assert.notNull(committeeIndex, "Committee index should not be null in aggregated attestation pool"); let attestationGroup = attestationGroupByIndex.get(committeeIndex); if (!attestationGroup) { attestationGroup = new MatchingDataAttestationGroup(committee, attestation.data); diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index 9db73cf4a013..a9286ac4949a 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -1,7 +1,7 @@ import {BitArray} from "@chainsafe/ssz"; import {Signature, aggregateSignatures} from "@chainsafe/blst"; -import {Slot, RootHex, allForks} from "@lodestar/types"; -import {MapDef} from "@lodestar/utils"; +import {Slot, RootHex, allForks, isElectraAttestation} from "@lodestar/types"; +import {MapDef, assert} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; @@ -35,6 +35,8 @@ type AggregateFast = AggregateFastPhase0 | AggregateFastElectra; /** Hex string of DataRoot `TODO` */ type DataRootHex = string; +type CommitteeIndex = number; + /** * A pool of `Attestation` that is specially designed to store "unaggregated" attestations from * the native aggregation scheme. @@ -59,8 +61,8 @@ type DataRootHex = string; * receives and it can be triggered manually. */ export class AttestationPool { - private readonly attestationByRootBySlot = new MapDef>( - () => new Map() + private readonly attestationByRootBySlot = new MapDef>>( + () => new Map>() ); private lowestPermissibleSlot = 0; @@ -116,14 +118,26 @@ export class AttestationPool { throw new OpPoolError({code: OpPoolErrorCode.REACHED_MAX_PER_SLOT}); } + const committeeIndex = isElectraAttestation(attestation) + ? // this attestation is added to pool after validation + attestation.committeeBits.getSingleTrueBit() + : attestation.data.index; + // this should not happen because attestation should be validated before reaching this + assert.notNull(committeeIndex, "Committee index should not be null in attestation pool"); + // Pre-aggregate the contribution with existing items - const aggregate = aggregateByRoot.get(attDataRootHex); + let aggregateByIndex = aggregateByRoot.get(attDataRootHex); + if (aggregateByIndex === undefined) { + aggregateByIndex = new Map(); + aggregateByRoot.set(attDataRootHex, aggregateByIndex); + } + const aggregate = aggregateByIndex.get(committeeIndex); if (aggregate) { // Aggregate mutating return aggregateAttestationInto(aggregate, attestation); } else { // Create new aggregate - aggregateByRoot.set(attDataRootHex, attestationToAggregate(attestation)); + aggregateByIndex.set(committeeIndex, attestationToAggregate(attestation)); return InsertOutcome.NewData; } } @@ -131,8 +145,8 @@ export class AttestationPool { /** * For validator API to get an aggregate */ - getAggregate(slot: Slot, dataRootHex: RootHex): allForks.Attestation | null { - const aggregate = this.attestationByRootBySlot.get(slot)?.get(dataRootHex); + getAggregate(slot: Slot, committeeIndex: CommitteeIndex, dataRootHex: RootHex): allForks.Attestation | null { + const aggregate = this.attestationByRootBySlot.get(slot)?.get(dataRootHex)?.get(committeeIndex); if (!aggregate) { // TODO: Add metric for missing aggregates return null; @@ -165,8 +179,10 @@ export class AttestationPool { for (const aggregateByRoot of aggregateByRoots) { if (aggregateByRoot) { - for (const aggFast of aggregateByRoot.values()) { - attestations.push(fastToAttestation(aggFast)); + for (const aggFastByIndex of aggregateByRoot.values()) { + for (const aggFast of aggFastByIndex.values()) { + attestations.push(fastToAttestation(aggFast)); + } } } } @@ -179,35 +195,13 @@ export class AttestationPool { // - Insert attestations coming from gossip and API /** - * Aggregate a new contribution into `aggregate` mutating it + * Aggregate a new attestation into `aggregate` mutating it */ function aggregateAttestationInto(aggregate: AggregateFast, attestation: allForks.Attestation): InsertOutcome { const bitIndex = attestation.aggregationBits.getSingleTrueBit(); // Should never happen, attestations are verified against this exact condition before - if (bitIndex === null) { - throw Error("Invalid attestation not exactly one bit set"); - } - - if ("committeeBits" in attestation && !("committeeBits" in aggregate)) { - throw Error("Attempt to aggregate electra attestation into phase0 attestation"); - } - - if (!("committeeBits" in attestation) && "committeeBits" in aggregate) { - throw Error("Attempt to aggregate phase0 attestation into electra attestation"); - } - - if ("committeeBits" in attestation) { - // We assume attestation.committeeBits should already be validated in api and gossip handler and should be non-null - const attestationCommitteeIndex = attestation.committeeBits.getSingleTrueBit(); - const aggregateCommitteeIndex = (aggregate as AggregateFastElectra).committeeBits.getSingleTrueBit(); - - if (attestationCommitteeIndex !== aggregateCommitteeIndex) { - throw Error( - `Committee index mismatched: attestation ${attestationCommitteeIndex} aggregate ${aggregateCommitteeIndex}` - ); - } - } + assert.notNull(bitIndex, "Invalid attestation in pool, not exactly one bit set"); if (aggregate.aggregationBits.get(bitIndex) === true) { return InsertOutcome.AlreadyKnown; @@ -222,7 +216,7 @@ function aggregateAttestationInto(aggregate: AggregateFast, attestation: allFork * Format `contribution` into an efficient `aggregate` to add more contributions in with aggregateContributionInto() */ function attestationToAggregate(attestation: allForks.Attestation): AggregateFast { - if ("committeeBits" in attestation) { + if (isElectraAttestation(attestation)) { return { data: attestation.data, // clone because it will be mutated @@ -243,18 +237,5 @@ function attestationToAggregate(attestation: allForks.Attestation): AggregateFas * Unwrap AggregateFast to phase0.Attestation */ function fastToAttestation(aggFast: AggregateFast): allForks.Attestation { - if ("committeeBits" in aggFast) { - return { - data: aggFast.data, - aggregationBits: aggFast.aggregationBits, - committeeBits: aggFast.committeeBits, - signature: aggFast.signature.toBytes(), - }; - } else { - return { - data: aggFast.data, - aggregationBits: aggFast.aggregationBits, - signature: aggFast.signature.toBytes(), - }; - } + return {...aggFast, signature: aggFast.signature.toBytes()}; } diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index d55b4e300232..c2ecb95907e9 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -82,7 +82,7 @@ async function validateAggregateAndProof( throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_COMMITTEE_BIT_SET}); } // [REJECT] aggregate.data.index == 0 - if (attData.index === 0) { + if (attData.index !== 0) { throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NON_ZERO_ATTESTATION_DATA_INDEX}); } } else { diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 813f9a25fc2d..046e1180309c 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -44,7 +44,7 @@ export type AttestationValidationResult = { export type AttestationOrBytes = ApiAttestation | GossipAttestation; /** attestation from api */ -export type ApiAttestation = {attestation: phase0.Attestation; serializedData: null}; // TODO Electra: add new attestation type +export type ApiAttestation = {attestation: phase0.Attestation; serializedData: null}; /** attestation from gossip */ export type GossipAttestation = { @@ -298,7 +298,7 @@ async function validateGossipAttestationNoSignatureCheck( } // [REJECT] aggregate.data.index == 0 - if (attData.index === 0) { + if (attData.index !== 0) { throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NON_ZERO_ATTESTATION_DATA_INDEX}); } } else { diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 273353d8632b..6c43151cc137 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -120,7 +120,7 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - altair processRegistryUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processRegistryUpdates(state, cache.value), + fn: (state) => processRegistryUpdates(ForkSeq.altair, state, cache.value), }); // TODO: Needs a better state to test with, current does not include enough actions: 39.985 us/op @@ -141,7 +141,7 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - altair processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(state, cache.value), + fn: (state) => processEffectiveBalanceUpdates(ForkSeq.altair, state, cache.value), }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index eeaf8bfc5400..5b8300df3f18 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -99,7 +99,7 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - capella processRegistryUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processRegistryUpdates(state, cache.value), + fn: (state) => processRegistryUpdates(ForkSeq.capella, state, cache.value), }); // TODO: Needs a better state to test with, current does not include enough actions: 39.985 us/op @@ -120,7 +120,7 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - capella processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(state, cache.value), + fn: (state) => processEffectiveBalanceUpdates(ForkSeq.capella, state, cache.value), }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index 4e43634b1669..411577878102 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -102,7 +102,7 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - phase0 processRegistryUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processRegistryUpdates(state, cache.value), + fn: (state) => processRegistryUpdates(ForkSeq.phase0, state, cache.value), }); // TODO: Needs a better state to test with, current does not include enough actions: 39.985 us/op @@ -123,7 +123,7 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - phase0 processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(state, cache.value), + fn: (state) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache.value), }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts index 0fb1d448142f..d94daac9e59b 100644 --- a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts @@ -1,6 +1,7 @@ import {itBench} from "@dapplion/benchmark"; import {ssz} from "@lodestar/types"; import {config} from "@lodestar/config/default"; +import {ForkSeq} from "@lodestar/params"; import {beforeProcessEpoch, CachedBeaconStateAllForks, EpochTransitionCache} from "../../../src/index.js"; import {processEffectiveBalanceUpdates} from "../../../src/epoch/processEffectiveBalanceUpdates.js"; import {numValidators} from "../util.js"; @@ -35,7 +36,7 @@ describe("phase0 processEffectiveBalanceUpdates", () => { minRuns: 5, // Worst case is very slow before: () => getEffectiveBalanceTestData(vc, changeRatio), beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => processEffectiveBalanceUpdates(state, cache), + fn: ({state, cache}) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache), }); } }); diff --git a/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts b/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts index ccfd2405a665..2d57de44f8ee 100644 --- a/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processRegistryUpdates.test.ts @@ -1,4 +1,5 @@ import {itBench} from "@dapplion/benchmark"; +import {ForkSeq} from "@lodestar/params"; import {beforeProcessEpoch, CachedBeaconStateAllForks, EpochTransitionCache} from "../../../src/index.js"; import {processRegistryUpdates} from "../../../src/epoch/processRegistryUpdates.js"; import {generatePerfTestCachedStatePhase0, numValidators} from "../util.js"; @@ -62,7 +63,7 @@ describe("phase0 processRegistryUpdates", () => { noThreshold: notTrack, before: () => getRegistryUpdatesTestData(vc, lengths), beforeEach: async ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => processRegistryUpdates(state, cache), + fn: ({state, cache}) => processRegistryUpdates(ForkSeq.phase0, state, cache), }); } }); diff --git a/packages/utils/src/assert.ts b/packages/utils/src/assert.ts index aa86161cca44..91612b0e6407 100644 --- a/packages/utils/src/assert.ts +++ b/packages/utils/src/assert.ts @@ -21,6 +21,18 @@ export const assert = { } }, + /** + * Assert not null + * ``` + * actual !== null + * ``` + */ + notNull(actual: T | null, message?: string): asserts actual is T { + if (!(actual !== null)) { + throw new AssertionError(`${message || "Expected value to be not null"}`); + } + }, + /** * Assert less than or equal * ```js diff --git a/packages/utils/test/unit/assert.test.ts b/packages/utils/test/unit/assert.test.ts index 0555bcbd01a0..3b413efa11be 100644 --- a/packages/utils/test/unit/assert.test.ts +++ b/packages/utils/test/unit/assert.test.ts @@ -20,8 +20,18 @@ describe("assert", () => { }); }); + describe("notNull with custom message", () => { + it("Should not throw error with not null value", () => { + expect(() => assert.notNull(0)).not.toThrow(); + expect(() => assert.notNull("")).not.toThrow(); + }); + it("Should throw with null value", () => { + expect(() => assert.notNull(null, "something must not be null")).toThrow("something must not be null"); + }); + }); + const cases: { - op: keyof Omit; + op: keyof Omit; args: [number, number]; ok: boolean; }[] = [ diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 31eb9e662e8c..a0cf8e8768e7 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -139,7 +139,7 @@ export class AttestationService { // Then download, sign and publish a `SignedAggregateAndProof` for each // validator that is elected to aggregate for this `slot` and `committeeIndex`. - await this.produceAndPublishAggregates(attestation, dutiesSameCommittee); + await this.produceAndPublishAggregates(attestation, index, dutiesSameCommittee); } private async runAttestationTasksGrouped( @@ -165,7 +165,7 @@ export class AttestationService { await Promise.all( Array.from(dutiesByCommitteeIndex.entries()).map(([index, dutiesSameCommittee]) => { const attestationData: phase0.AttestationData = {...attestationNoCommittee, index}; - return this.produceAndPublishAggregates(attestationData, dutiesSameCommittee); + return this.produceAndPublishAggregates(attestationData, index, dutiesSameCommittee); }) ); } @@ -256,9 +256,10 @@ export class AttestationService { */ private async produceAndPublishAggregates( attestation: phase0.AttestationData, + committeeIndex: number, duties: AttDutyAndProof[] ): Promise { - const logCtx = {slot: attestation.slot, index: attestation.index}; + const logCtx = {slot: attestation.slot, index: committeeIndex}; // No validator is aggregator, skip if (duties.every(({selectionProof}) => selectionProof === null)) { @@ -269,6 +270,7 @@ export class AttestationService { const res = await this.api.validator.getAggregatedAttestation({ attestationDataRoot: ssz.phase0.AttestationData.hashTreeRoot(attestation), slot: attestation.slot, + committeeIndex, }); const aggregate = res.value(); this.metrics?.numParticipantsInAggregate.observe(aggregate.aggregationBits.getTrueBitIndexes().length); diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 4678f6188382..a807c27569eb 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -20,6 +20,7 @@ import { DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_APPLICATION_BUILDER, ForkSeq, + MAX_COMMITTEES_PER_SLOT, } from "@lodestar/params"; import { altair, @@ -531,7 +532,7 @@ export class ValidatorStore { return { aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), data: attestationData, - committeeBits: BitArray.fromSingleBit(duty.committeesAtSlot, duty.committeeIndex), + committeeBits: BitArray.fromSingleBit(MAX_COMMITTEES_PER_SLOT, duty.committeeIndex), signature: await this.getSignature(duty.pubkey, signingRoot, signingSlot, signableMessage), } as electra.Attestation; } else { From 08932765a785e060f207eba7b764f3c8fcd9590b Mon Sep 17 00:00:00 2001 From: g11tech Date: Wed, 8 May 2024 17:11:08 +0530 Subject: [PATCH 034/145] feat: update engineapi endpoints to v4 (#6747) --- packages/beacon-node/src/execution/engine/http.ts | 6 +++--- packages/beacon-node/src/execution/engine/mock.ts | 4 ++-- packages/beacon-node/src/execution/engine/types.ts | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index ac3402852c35..10a805e26d66 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -199,7 +199,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { ): Promise { const method = ForkSeq[fork] >= ForkSeq.electra - ? "engine_newPayloadV6110" + ? "engine_newPayloadV4" : ForkSeq[fork] >= ForkSeq.deneb ? "engine_newPayloadV3" : ForkSeq[fork] >= ForkSeq.capella @@ -220,7 +220,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { const serializedVersionedHashes = serializeVersionedHashes(versionedHashes); const parentBeaconBlockRoot = serializeBeaconBlockRoot(parentBlockRoot); - const method = ForkSeq[fork] >= ForkSeq.electra ? "engine_newPayloadV6110" : "engine_newPayloadV3"; + const method = ForkSeq[fork] >= ForkSeq.electra ? "engine_newPayloadV4" : "engine_newPayloadV3"; engineRequest = { method, params: [serializedExecutionPayload, serializedVersionedHashes, parentBeaconBlockRoot], @@ -395,7 +395,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { }> { const method = ForkSeq[fork] >= ForkSeq.electra - ? "engine_getPayloadV6110" + ? "engine_getPayloadV4" : ForkSeq[fork] >= ForkSeq.deneb ? "engine_getPayloadV3" : ForkSeq[fork] >= ForkSeq.capella diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index 5a50f1697de9..4cff2a8d00f4 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -89,14 +89,14 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_newPayloadV1: this.notifyNewPayload.bind(this), engine_newPayloadV2: this.notifyNewPayload.bind(this), engine_newPayloadV3: this.notifyNewPayload.bind(this), - engine_newPayloadV6110: this.notifyNewPayload.bind(this), + engine_newPayloadV4: this.notifyNewPayload.bind(this), engine_forkchoiceUpdatedV1: this.notifyForkchoiceUpdate.bind(this), engine_forkchoiceUpdatedV2: this.notifyForkchoiceUpdate.bind(this), engine_forkchoiceUpdatedV3: this.notifyForkchoiceUpdate.bind(this), engine_getPayloadV1: this.getPayload.bind(this), engine_getPayloadV2: this.getPayload.bind(this), engine_getPayloadV3: this.getPayload.bind(this), - engine_getPayloadV6110: this.getPayload.bind(this), + engine_getPayloadV4: this.getPayload.bind(this), engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this), engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this), engine_getClientVersionV1: this.getClientVersionV1.bind(this), diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index f0701d48c5fc..49b34ea97086 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -28,7 +28,7 @@ export type EngineApiRpcParamTypes = { engine_newPayloadV1: [ExecutionPayloadRpc]; engine_newPayloadV2: [ExecutionPayloadRpc]; engine_newPayloadV3: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; - engine_newPayloadV6110: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; + engine_newPayloadV4: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; /** * 1. Object - Payload validity status with respect to the consensus rules: * - blockHash: DATA, 32 Bytes - block hash value of the payload @@ -52,7 +52,7 @@ export type EngineApiRpcParamTypes = { engine_getPayloadV1: [QUANTITY]; engine_getPayloadV2: [QUANTITY]; engine_getPayloadV3: [QUANTITY]; - engine_getPayloadV6110: [QUANTITY]; + engine_getPayloadV4: [QUANTITY]; /** * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure @@ -85,7 +85,7 @@ export type EngineApiRpcReturnTypes = { engine_newPayloadV1: PayloadStatus; engine_newPayloadV2: PayloadStatus; engine_newPayloadV3: PayloadStatus; - engine_newPayloadV6110: PayloadStatus; + engine_newPayloadV4: PayloadStatus; engine_forkchoiceUpdatedV1: { payloadStatus: PayloadStatus; payloadId: QUANTITY | null; @@ -104,7 +104,7 @@ export type EngineApiRpcReturnTypes = { engine_getPayloadV1: ExecutionPayloadRpc; engine_getPayloadV2: ExecutionPayloadResponse; engine_getPayloadV3: ExecutionPayloadResponse; - engine_getPayloadV6110: ExecutionPayloadResponse; + engine_getPayloadV4: ExecutionPayloadResponse; engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[]; From 2b0ec1255b2a9a560ef07f346460942f7450ea9a Mon Sep 17 00:00:00 2001 From: NC Date: Wed, 8 May 2024 18:12:36 +0300 Subject: [PATCH 035/145] feat: rename deposit receipt to deposit request for Pectra (#6748) * Rename receipt to request * Remove stats.html --- packages/beacon-node/src/chain/chain.ts | 2 +- .../src/eth1/eth1DepositDataTracker.ts | 4 +- .../src/execution/engine/interface.ts | 4 +- .../src/execution/engine/payloadIdCache.ts | 2 +- .../beacon-node/src/execution/engine/types.ts | 40 +++++++++---------- .../test/sim/electra-interop.test.ts | 28 ++++++------- .../test/spec/presets/operations.test.ts | 6 +-- .../test/unit/executionEngine/http.test.ts | 8 ++-- packages/beacon-node/test/utils/state.ts | 2 +- packages/light-client/src/spec/utils.ts | 8 ++-- .../src/block/processDeposit.ts | 6 +-- ...sitReceipt.ts => processDepositRequest.ts} | 10 ++--- .../src/block/processOperations.ts | 8 ++-- .../src/slot/upgradeStateToElectra.ts | 6 +-- packages/state-transition/src/util/deposit.ts | 4 +- .../state-transition/src/util/execution.ts | 4 +- packages/state-transition/src/util/genesis.ts | 2 +- .../test/unit/util/deposit.test.ts | 4 +- packages/types/src/electra/sszTypes.ts | 12 +++--- packages/types/src/electra/types.ts | 4 +- 20 files changed, 82 insertions(+), 82 deletions(-) rename packages/state-transition/src/block/{processDepositReceipt.ts => processDepositRequest.ts} (55%) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 9f67857fe530..9f30936305f0 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1147,7 +1147,7 @@ export class BeaconChain implements IBeaconChain { // Will resolve this later // if (cpEpoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity)) { // // finalizedState can be safely casted to Electra state since cp is already post-Electra - // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositReceiptsStartIndex) { + // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositRequestsStartIndex) { // // Signal eth1 to stop polling eth1Data // this.eth1.stopPollingEth1Data(); // } diff --git a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts index c0b3ab35a73a..d0578718f29f 100644 --- a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts +++ b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts @@ -129,9 +129,9 @@ export class Eth1DepositDataTracker { async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { if ( state.epochCtx.isAfterElectra() && - state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositReceiptsStartIndex + state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositRequestsStartIndex ) { - // No need to poll eth1Data since Electra deprecates the mechanism after depositReceiptsStartIndex is reached + // No need to poll eth1Data since Electra deprecates the mechanism after depositRequestsStartIndex is reached return {eth1Data: state.eth1Data, deposits: []}; } const eth1Data = this.forcedEth1DataVote ?? (await this.getEth1Data(state)); diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index 91a8ce1cffe3..c2e44901afbb 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -3,10 +3,10 @@ import {KZGCommitment, Blob, KZGProof} from "@lodestar/types/deneb"; import {Root, RootHex, capella, Wei, ExecutionPayload} from "@lodestar/types"; import {DATA} from "../../eth1/provider/utils.js"; -import {PayloadIdCache, PayloadId, WithdrawalV1, DepositReceiptV1} from "./payloadIdCache.js"; +import {PayloadIdCache, PayloadId, WithdrawalV1, DepositRequestV1} from "./payloadIdCache.js"; import {ExecutionPayloadBody} from "./types.js"; -export {PayloadIdCache, type PayloadId, type WithdrawalV1, type DepositReceiptV1}; +export {PayloadIdCache, type PayloadId, type WithdrawalV1, type DepositRequestV1}; export enum ExecutionPayloadStatus { /** given payload is valid */ diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index b5fe3d33e267..f79c28582459 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -18,7 +18,7 @@ export type WithdrawalV1 = { amount: QUANTITY; }; -export type DepositReceiptV1 = { +export type DepositRequestV1 = { pubkey: DATA; withdrawalCredentials: DATA; amount: QUANTITY; diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 49b34ea97086..3c1da820a610 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1, DepositReceiptV1, ExecutionLayerWithdrawalRequestV1} from "./payloadIdCache.js"; +import {WithdrawalV1, DepositRequestV1, ExecutionLayerWithdrawalRequestV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -127,14 +127,14 @@ export type ExecutionPayloadBodyRpc = { withdrawals: WithdrawalV1[] | null | undefined; // currently there is a discepancy between EL and CL field name references for deposit requests // its likely CL receipt will be renamed to requests - depositRequests: DepositReceiptV1[] | null | undefined; + depositRequests: DepositRequestV1[] | null | undefined; withdrawalRequests: ExecutionLayerWithdrawalRequestV1[] | null | undefined; }; export type ExecutionPayloadBody = { transactions: bellatrix.Transaction[]; withdrawals: capella.Withdrawals | null; - depositReceipts: electra.DepositReceipts | null; + depositRequests: electra.DepositRequests | null; withdrawalRequests: electra.ExecutionLayerWithdrawalRequests | null; }; @@ -157,7 +157,7 @@ export type ExecutionPayloadRpc = { blobGasUsed?: QUANTITY; // DENEB excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB - depositRequests?: DepositReceiptRpc[]; // ELECTRA + depositRequests?: DepositRequestRpc[]; // ELECTRA withdrawalRequests?: ExecutionLayerWithdrawalRequestRpc[]; // ELECTRA }; @@ -168,7 +168,7 @@ export type WithdrawalRpc = { amount: QUANTITY; }; -export type DepositReceiptRpc = DepositReceiptV1; +export type DepositRequestRpc = DepositRequestV1; export type ExecutionLayerWithdrawalRequestRpc = ExecutionLayerWithdrawalRequestV1; export type VersionedHashesRpc = DATA[]; @@ -233,10 +233,10 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload payload.excessBlobGas = numToQuantity(excessBlobGas); } - // ELECTRA adds depositReceipts/depositRequests to the ExecutionPayload + // ELECTRA adds depositRequests/depositRequests to the ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositReceipts, withdrawalRequests} = data as electra.ExecutionPayload; - payload.depositRequests = depositReceipts.map(serializeDepositReceipt); + const {depositRequests, withdrawalRequests} = data as electra.ExecutionPayload; + payload.depositRequests = depositRequests.map(serializeDepositRequest); payload.withdrawalRequests = withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest); } @@ -326,7 +326,7 @@ export function parseExecutionPayload( } if (ForkSeq[fork] >= ForkSeq.electra) { - // electra adds depositRequests/depositReceipts + // electra adds depositRequests/depositRequests const {depositRequests, withdrawalRequests} = data; // Geth can also reply with null if (depositRequests == null) { @@ -334,7 +334,7 @@ export function parseExecutionPayload( `depositRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` ); } - (executionPayload as electra.ExecutionPayload).depositReceipts = depositRequests.map(deserializeDepositReceipt); + (executionPayload as electra.ExecutionPayload).depositRequests = depositRequests.map(deserializeDepositRequest); if (withdrawalRequests == null) { throw Error( @@ -412,24 +412,24 @@ export function deserializeWithdrawal(serialized: WithdrawalRpc): capella.Withdr } as capella.Withdrawal; } -export function serializeDepositReceipt(depositReceipt: electra.DepositReceipt): DepositReceiptRpc { +export function serializeDepositRequest(depositRequest: electra.DepositRequest): DepositRequestRpc { return { - pubkey: bytesToData(depositReceipt.pubkey), - withdrawalCredentials: bytesToData(depositReceipt.withdrawalCredentials), - amount: numToQuantity(depositReceipt.amount), - signature: bytesToData(depositReceipt.signature), - index: numToQuantity(depositReceipt.index), + pubkey: bytesToData(depositRequest.pubkey), + withdrawalCredentials: bytesToData(depositRequest.withdrawalCredentials), + amount: numToQuantity(depositRequest.amount), + signature: bytesToData(depositRequest.signature), + index: numToQuantity(depositRequest.index), }; } -export function deserializeDepositReceipt(serialized: DepositReceiptRpc): electra.DepositReceipt { +export function deserializeDepositRequest(serialized: DepositRequestRpc): electra.DepositRequest { return { pubkey: dataToBytes(serialized.pubkey, 48), withdrawalCredentials: dataToBytes(serialized.withdrawalCredentials, 32), amount: quantityToNum(serialized.amount), signature: dataToBytes(serialized.signature, 96), index: quantityToNum(serialized.index), - } as electra.DepositReceipt; + } as electra.DepositRequest; } export function serializeExecutionLayerWithdrawalRequest( @@ -457,7 +457,7 @@ export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | ? { transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, - depositReceipts: data.depositRequests ? data.depositRequests.map(deserializeDepositReceipt) : null, + depositRequests: data.depositRequests ? data.depositRequests.map(deserializeDepositRequest) : null, withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(deserializeExecutionLayerWithdrawalRequest) : null, @@ -470,7 +470,7 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) ? { transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, - depositRequests: data.depositReceipts ? data.depositReceipts.map(serializeDepositReceipt) : null, + depositRequests: data.depositRequests ? data.depositRequests.map(serializeDepositRequest) : null, withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest) : null, diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 29483b249c85..c503052a74f4 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -72,7 +72,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { } }); - it("Send and get payloads with depositReceipts to/from EL", async () => { + it("Send and get payloads with depositRequests to/from EL", async () => { const {elClient, tearDownCallBack} = await runEL( {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, {...elRunOptions, ttd: BigInt(0)}, @@ -111,7 +111,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { // 2. Send raw deposit transaction A and B. tx A is to be imported via newPayload, tx B is to be included in payload via getPayload const depositTransactionA = "0x02f9021c8217de808459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120749715de5d1226545c6b3790f515d551a5cc5bf1d49c87a696860554d2fc4f14000000000000000000000000000000000000000000000000000000000000003096a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9c080a09f597089338d7f44f5c59f8230bb38f243849228a8d4e9d2e2956e6050f5b2c7a076486996c7e62802b8f95eee114783e4b403fd11093ba96286ff42c595f24452"; - const depositReceiptA = { + const depositRequestA = { amount: 32000000000, index: 0, pubkey: dataToBytes( @@ -127,7 +127,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { const depositTransactionB = "0x02f9021c8217de018459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120a18b4c7cab0afa273ea9504904521ea8421a4e32740b7611bd3d5095ca99f0cb0000000000000000000000000000000000000000000000000000000000000030a5c85a60ba2905c215f6a12872e62b1ee037051364244043a5f639aa81b04a204c55e7cc851f29c7c183be253ea1510b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020001db70c485b6264692f26b8aeaab5b0c384180df8e2184a21a808a3ec8e86ca00000000000000000000000000000000000000000000000000000000000000609561731785b48cf1886412234531e4940064584463e96ac63a1a154320227e333fb51addc4a89b7e0d3f862d7c1fd4ea03bd8eb3d8806f1e7daf591cbbbb92b0beb74d13c01617f22c5026b4f9f9f294a8a7c32db895de3b01bee0132c9209e1c001a0644e0a763a34b4bfb9f56a677857b57fcf15e3db57e2f57060e92084f75f3d82a018ba8eaacbd8e6f6917675b1d0362b12ca82850ca8ef9c010430760c2b2e0cb5"; - const depositReceiptB = { + const depositRequestB = { amount: 32000000000, index: 1, pubkey: dataToBytes( @@ -168,7 +168,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { excessBlobGas: 0n, transactions: [dataToBytes(depositTransactionA, null)], withdrawals: [], - depositReceipts: [depositReceiptA], + depositRequests: [depositRequestA], blockNumber: 1, blockHash: dataToBytes(newPayloadBlockHash, 32), receiptsRoot: dataToBytes("0x79ee3424eb720a3ad4b1c5a372bb8160580cbe4d893778660f34213c685627a9", 32), @@ -205,7 +205,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { ); if (!payloadId2) throw Error("InvalidPayloadId"); - // 5. Get the payload. Check depositReceipts field contains deposit + // 5. Get the payload. Check depositRequests field contains deposit // Wait a bit first for besu to pick up tx from the tx pool. await sleep(1000); const payloadAndBlockValue = await executionEngine.getPayload(ForkName.electra, payloadId2); @@ -221,16 +221,16 @@ describe("executionEngine / ExecutionEngineHttp", function () { } } - if (payload.depositReceipts.length !== 1) { - throw Error(`Number of depositReceipts mismatched. Expected: 1, actual: ${payload.depositReceipts.length}`); + if (payload.depositRequests.length !== 1) { + throw Error(`Number of depositRequests mismatched. Expected: 1, actual: ${payload.depositRequests.length}`); } - const actualDepositReceipt = payload.depositReceipts[0]; + const actualDepositRequest = payload.depositRequests[0]; assert.deepStrictEqual( - actualDepositReceipt, - depositReceiptB, - `Deposit receipts mismatched. Expected: ${JSON.stringify(depositReceiptB)}, actual: ${JSON.stringify( - actualDepositReceipt + actualDepositRequest, + depositRequestB, + `Deposit receipts mismatched. Expected: ${JSON.stringify(depositRequestB)}, actual: ${JSON.stringify( + actualDepositRequest )}` ); }); @@ -432,8 +432,8 @@ describe("executionEngine / ExecutionEngineHttp", function () { throw Error("Historical validator length for epoch 1 or 2 is not dropped properly"); } - if (headState.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { - throw Error("state.depositReceiptsStartIndex is not set upon processing new deposit receipt"); + if (headState.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + throw Error("state.depositRequestsStartIndex is not set upon processing new deposit receipt"); } // wait for 1 slot to print current epoch stats diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 9b8214a88c0a..c674c1dd18f9 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -57,9 +57,9 @@ const operationFns: Record> = blockFns.processDeposit(fork, state, testCase.deposit); }, - deposit_receipt: (state, testCase: {deposit_receipt: electra.DepositReceipt}) => { + deposit_receipt: (state, testCase: {deposit_receipt: electra.DepositRequest}) => { const fork = state.config.getForkSeq(state.slot); - blockFns.processDepositReceipt(fork, state as CachedBeaconStateElectra, testCase.deposit_receipt); + blockFns.processDepositRequest(fork, state as CachedBeaconStateElectra, testCase.deposit_receipt); }, proposer_slashing: (state, testCase: {proposer_slashing: phase0.ProposerSlashing}) => { @@ -127,7 +127,7 @@ const operations: TestRunnerFn = (fork, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, - deposit_receipt: ssz.electra.DepositReceipt, + deposit_receipt: ssz.electra.DepositRequest, proposer_slashing: ssz.phase0.ProposerSlashing, voluntary_exit: ssz.phase0.SignedVoluntaryExit, // Altair diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index 842f39a5ff90..2cc08c8c78d2 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -193,7 +193,7 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], - depositReceipts: null, // depositReceipts is null pre-electra + depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, }, null, // null returned for missing blocks @@ -203,7 +203,7 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella - depositReceipts: null, // depositReceipts is null pre-electra + depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, }, ], @@ -252,7 +252,7 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], - depositReceipts: null, // depositReceipts is null pre-electra + depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, }, null, // null returned for missing blocks @@ -262,7 +262,7 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella - depositReceipts: null, // depositReceipts is null pre-electra + depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, }, ], diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index ad4dd4b88dac..bb55adca1eb0 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -97,7 +97,7 @@ export function generateState( if (forkSeq >= ForkSeq.electra) { const stateElectra = state as electra.BeaconState; - stateElectra.depositReceiptsStartIndex = 2023n; + stateElectra.depositRequestsStartIndex = 2023n; stateElectra.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue(); } diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 1b460dd39cf2..679371a54d6c 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -115,9 +115,9 @@ export function upgradeLightClientHeader( // eslint-disable-next-line no-fallthrough case ForkName.electra: - (upgradedHeader as LightClientHeader).execution.depositReceiptsRoot = - ssz.electra.LightClientHeader.fields.execution.fields.depositReceiptsRoot.defaultValue(); - (upgradedHeader as electra.LightClientHeader).execution.withdrawalRequestsRoot = + (upgradedHeader as LightClientHeader).execution.depositRequestsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.depositRequestsRoot.defaultValue(); + (upgradedHeader as LightClientHeader).execution.withdrawalRequestsRoot = ssz.electra.LightClientHeader.fields.execution.fields.withdrawalRequestsRoot.defaultValue(); // Break if no further upgrades is required else fall through @@ -157,7 +157,7 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC if (epoch < config.ELECTRA_FORK_EPOCH) { if ( - (header as LightClientHeader).execution.depositReceiptsRoot !== undefined || + (header as LightClientHeader).execution.depositRequestsRoot !== undefined || (header as LightClientHeader).execution.withdrawalRequestsRoot !== undefined ) { return false; diff --git a/packages/state-transition/src/block/processDeposit.ts b/packages/state-transition/src/block/processDeposit.ts index 65b3cbfb10fd..bc140b864ad9 100644 --- a/packages/state-transition/src/block/processDeposit.ts +++ b/packages/state-transition/src/block/processDeposit.ts @@ -12,7 +12,7 @@ import { } from "@lodestar/params"; import {DepositData} from "@lodestar/types/lib/phase0/types.js"; -import {DepositReceipt} from "@lodestar/types/lib/electra/types.js"; +import {DepositRequest} from "@lodestar/types/lib/electra/types.js"; import {BeaconConfig} from "@lodestar/config"; import {ZERO_HASH} from "../constants/index.js"; import { @@ -53,13 +53,13 @@ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, /** * Adds a new validator into the registry. Or increase balance if already exist. - * Follows applyDeposit() in consensus spec. Will be used by processDeposit() and processDepositReceipt() + * Follows applyDeposit() in consensus spec. Will be used by processDeposit() and processDepositRequest() * */ export function applyDeposit( fork: ForkSeq, state: CachedBeaconStateAllForks, - deposit: DepositData | DepositReceipt + deposit: DepositData | DepositRequest ): void { const {config, validators, epochCtx} = state; const {pubkey, withdrawalCredentials, amount} = deposit; diff --git a/packages/state-transition/src/block/processDepositReceipt.ts b/packages/state-transition/src/block/processDepositRequest.ts similarity index 55% rename from packages/state-transition/src/block/processDepositReceipt.ts rename to packages/state-transition/src/block/processDepositRequest.ts index 140e38d634fd..ca6fe4206188 100644 --- a/packages/state-transition/src/block/processDepositReceipt.ts +++ b/packages/state-transition/src/block/processDepositRequest.ts @@ -4,14 +4,14 @@ import {ForkSeq, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; import {CachedBeaconStateElectra} from "../types.js"; import {applyDeposit} from "./processDeposit.js"; -export function processDepositReceipt( +export function processDepositRequest( fork: ForkSeq, state: CachedBeaconStateElectra, - depositReceipt: electra.DepositReceipt + depositRequest: electra.DepositRequest ): void { - if (state.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { - state.depositReceiptsStartIndex = BigInt(depositReceipt.index); + if (state.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + state.depositRequestsStartIndex = BigInt(depositRequest.index); } - applyDeposit(fork, state, depositReceipt); + applyDeposit(fork, state, depositRequest); } diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 8d65762931e7..3ddc3668e487 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -10,7 +10,7 @@ import {processDeposit} from "./processDeposit.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; import {processExecutionLayerWithdrawalRequest} from "./processExecutionLayerWithdrawalRequest.js"; -import {processDepositReceipt} from "./processDepositReceipt.js"; +import {processDepositRequest} from "./processDepositRequest.js"; import {ProcessBlockOpts} from "./types.js"; import {processConsolidation} from "./processConsolidation.js"; @@ -22,7 +22,7 @@ export { processVoluntaryExit, processExecutionLayerWithdrawalRequest, processBlsToExecutionChange, - processDepositReceipt, + processDepositRequest, }; export function processOperations( @@ -70,8 +70,8 @@ export function processOperations( processExecutionLayerWithdrawalRequest(fork, state as CachedBeaconStateElectra, elWithdrawalRequest); } - for (const depositReceipt of bodyElectra.executionPayload.depositReceipts) { - processDepositReceipt(fork, stateElectra, depositReceipt); + for (const depositRequest of bodyElectra.executionPayload.depositRequests) { + processDepositRequest(fork, stateElectra, depositRequest); } for (const consolidation of bodyElectra.consolidations) { diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 760595c3a86b..a6bc6c331e11 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -25,9 +25,9 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache epoch: stateDeneb.epochCtx.epoch, }); - // latestExecutionPayloadHeader's depositReceiptsRoot and withdrawalRequestsRoot set to zeros by default - // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX - stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default + // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; const validatorsArr = stateElectra.validators.getAllReadonly(); diff --git a/packages/state-transition/src/util/deposit.ts b/packages/state-transition/src/util/deposit.ts index 493099fdd982..e8ef93c515d2 100644 --- a/packages/state-transition/src/util/deposit.ts +++ b/packages/state-transition/src/util/deposit.ts @@ -9,9 +9,9 @@ export function getEth1DepositCount(state: CachedBeaconStateAllForks, eth1Data?: // eth1DataIndexLimit = min(UintNum64, UintBn64) can be safely casted as UintNum64 // since the result lies within upper and lower bound of UintNum64 const eth1DataIndexLimit: UintNum64 = - eth1DataToUse.depositCount < electraState.depositReceiptsStartIndex + eth1DataToUse.depositCount < electraState.depositRequestsStartIndex ? eth1DataToUse.depositCount - : Number(electraState.depositReceiptsStartIndex); + : Number(electraState.depositRequestsStartIndex); if (state.eth1DepositIndex < eth1DataIndexLimit) { return Math.min(MAX_DEPOSITS, eth1DataIndexLimit - state.eth1DepositIndex); diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index c7f0ec2f395c..2c55dc61a502 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -172,8 +172,8 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio } if (fork >= ForkSeq.electra) { - (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositReceiptsRoot = - ssz.electra.DepositReceipts.hashTreeRoot((payload as electra.ExecutionPayload).depositReceipts); + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositRequestsRoot = + ssz.electra.DepositRequests.hashTreeRoot((payload as electra.ExecutionPayload).depositRequests); (bellatrixPayloadFields as electra.ExecutionPayloadHeader).withdrawalRequestsRoot = ssz.electra.ExecutionLayerWithdrawalRequests.hashTreeRoot( (payload as electra.ExecutionPayload).withdrawalRequests diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 537edc7f976f..d5f4c577711f 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -293,7 +293,7 @@ export function initializeBeaconStateFromEth1( stateElectra.latestExecutionPayloadHeader = (executionPayloadHeader as CompositeViewDU) ?? ssz.electra.ExecutionPayloadHeader.defaultViewDU(); - stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; } state.commit(); diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts index e8f7f7a86af8..677a15724ece 100644 --- a/packages/state-transition/test/unit/util/deposit.test.ts +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -43,7 +43,7 @@ describe("getEth1DepositCount", () => { throw Error("Not a post-Electra state"); } - postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.depositRequestsStartIndex = 1000n; postElectraState.eth1Data.depositCount = 995; // 1. Should get less than MAX_DEPOSIT @@ -77,7 +77,7 @@ describe("getEth1DepositCount", () => { throw Error("Not a post-Electra state"); } - postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.depositRequestsStartIndex = 1000n; postElectraState.eth1Data.depositCount = 1005; // Before eth1DepositIndex reaching the start index diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 26539dd61365..246c731d8382 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -110,7 +110,7 @@ export const SignedAggregateAndProof = new ContainerType( {typeName: "SignedAggregateAndProof", jsonCase: "eth2"} ); -export const DepositReceipt = new ContainerType( +export const DepositRequest = new ContainerType( { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, @@ -118,10 +118,10 @@ export const DepositReceipt = new ContainerType( signature: BLSSignature, index: DepositIndex, }, - {typeName: "DepositReceipt", jsonCase: "eth2"} + {typeName: "DepositRequest", jsonCase: "eth2"} ); -export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); +export const DepositRequests = new ListCompositeType(DepositRequest, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); export const ExecutionLayerWithdrawalRequest = new ContainerType( { @@ -139,7 +139,7 @@ export const ExecutionLayerWithdrawalRequests = new ListCompositeType( export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, - depositReceipts: DepositReceipts, // New in ELECTRA + depositRequests: DepositRequests, // New in ELECTRA withdrawalRequests: ExecutionLayerWithdrawalRequests, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} @@ -148,7 +148,7 @@ export const ExecutionPayload = new ContainerType( export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, - depositReceiptsRoot: Root, // New in ELECTRA + depositRequestsRoot: Root, // New in ELECTRA withdrawalRequestsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} @@ -335,7 +335,7 @@ export const BeaconState = new ContainerType( nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, // Deep history valid from Capella onwards historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, - depositReceiptsStartIndex: UintBn64, // New in ELECTRA + depositRequestsStartIndex: UintBn64, // New in ELECTRA depositBalanceToConsume: Gwei, // [New in Electra] exitBalanceToConsume: Gwei, // [New in Electra] earliestExitEpoch: Epoch, // [New in Electra] diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 4de209b48960..53b95e95525c 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -9,8 +9,8 @@ export type AttesterSlashing = ValueOf; export type AggregateAndProof = ValueOf; export type SignedAggregateAndProof = ValueOf; -export type DepositReceipt = ValueOf; -export type DepositReceipts = ValueOf; +export type DepositRequest = ValueOf; +export type DepositRequests = ValueOf; export type ExecutionLayerWithdrawalRequest = ValueOf; export type ExecutionLayerWithdrawalRequests = ValueOf; From 3cf76cb5e446890d4adee8dca158ec0d93de1699 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 8 May 2024 17:14:44 +0200 Subject: [PATCH 036/145] test: enable spec tests related to eip-7549 (#6741) * initial commit * Update gossip validation * Update attestation gossip validation * aggregateAndProof validation * Extend spec runner to be more flexible * Add missing state attributes for electra * Fix ss data types for electra spec * Make the spec runner more flexible * Fix the bug in process attestation * Update the sepc test version * clean up * Misc * Fix the build erros * feat: get attestations for electra block (#6732) * feat: getAttestationsForBlock() for electra * chore: fix lint * fix: MAX_ATTESTATIONS_PER_GROUP_ELECTRA and address PR comments * chore: unit test aggregateConsolidation * Fix rebase mistake * Address my own comment :) --------- Co-authored-by: Navie Chan * Fix check-types * Address comments * Fix the build erros * Extend spec runner to be more flexible * Add missing state attributes for electra * Fix ss data types for electra spec * Make the spec runner more flexible * Fix the bug in process attestation * Update the sepc test version * Fix rebase issue * Update committee index count check --------- Co-authored-by: NC Co-authored-by: Navie Chan Co-authored-by: tuyennhv --- .../test/spec/presets/operations.test.ts | 4 +-- .../test/spec/specTestVersioning.ts | 2 +- .../test/spec/utils/specTestIterator.ts | 27 +++++++++++++------ .../src/block/processAttestationPhase0.ts | 19 +++++++------ packages/types/src/electra/sszTypes.ts | 20 +++++++------- 5 files changed, 43 insertions(+), 29 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index c674c1dd18f9..ca215f1e197c 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -122,8 +122,8 @@ const operations: TestRunnerFn = (fork, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, - attestation: ssz.phase0.Attestation, - attester_slashing: ssz.phase0.AttesterSlashing, + attestation: fork === ForkName.electra ? ssz.electra.Attestation : ssz.phase0.Attestation, + attester_slashing: fork === ForkName.electra ? ssz.electra.AttesterSlashing : ssz.phase0.AttesterSlashing, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index 06b02ab5304e..9195532379a0 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.5.0-alpha.1", + specVersion: "v1.5.0-alpha.2", // Target directory is the host package root: 'packages/*/spec-tests' outputDir: path.join(__dirname, "../../spec-tests"), specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests", diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 88d7cbdea9e6..b99bc281cf50 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -14,7 +14,8 @@ const ARTIFACT_FILENAMES = new Set([ ]); export interface SkipOpts { - skippedPrefixes?: string[]; + skippedTestSuites?: RegExp[]; + skippedTests?: RegExp[]; skippedForks?: string[]; skippedRunners?: string[]; skippedHandlers?: string[]; @@ -57,14 +58,17 @@ const coveredTestRunners = [ // ], // ``` export const defaultSkipOpts: SkipOpts = { - skippedForks: ["electra", "eip7594"], + skippedForks: ["eip7594"], // TODO: capella // BeaconBlockBody proof in lightclient is the new addition in v1.3.0-rc.2-hotfix // Skip them for now to enable subsequently - skippedPrefixes: [ - "capella/light_client/single_merkle_proof/BeaconBlockBody", - "deneb/light_client/single_merkle_proof/BeaconBlockBody", + skippedTestSuites: [ + /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, + /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, + // /^electra\/(?!operations\/attestations)(?!operations\/attester_slashing)/, + /^electra\/(?!operations\/attestation)/, ], + skippedTests: [], skippedRunners: ["merkle_proof", "networking"], }; @@ -100,7 +104,10 @@ export function specTestIterator( opts: SkipOpts = defaultSkipOpts ): void { for (const forkStr of readdirSyncSpec(configDirpath)) { - if (opts?.skippedForks?.includes(forkStr)) { + if ( + opts?.skippedForks?.includes(forkStr) || + (process.env.SPEC_FILTER_FORK && forkStr !== process.env.SPEC_FILTER_FORK) + ) { continue; } const fork = forkStr as ForkName; @@ -134,7 +141,7 @@ export function specTestIterator( for (const testSuite of readdirSyncSpec(testHandlerDirpath)) { const testId = `${fork}/${testRunnerName}/${testHandler}/${testSuite}`; - if (opts?.skippedPrefixes?.some((skippedPrefix) => testId.startsWith(skippedPrefix))) { + if (opts?.skippedTestSuites?.some((skippedMatch) => testId.match(skippedMatch))) { displaySkipTest(testId); } else if (fork === undefined) { displayFailTest(testId, `Unknown fork ${forkStr}`); @@ -150,7 +157,11 @@ export function specTestIterator( // Generic testRunner else { const {testFunction, options} = testRunner.fn(fork, testHandler, testSuite); - + if (opts.skippedTests && options.shouldSkip === undefined) { + options.shouldSkip = (_testCase: any, name: string, _index: number): boolean => { + return opts?.skippedTests?.some((skippedMatch) => name.match(skippedMatch)) ?? false; + }; + } describeDirectorySpecTest(testId, testSuiteDirpath, testFunction, options); } } diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index c7c6b3b531a8..3bb5b71fa267 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -91,17 +91,20 @@ export function validateAttestation( if (fork >= ForkSeq.electra) { assert.equal(data.index, 0, `AttestationData.index must be zero: index=${data.index}`); const attestationElectra = attestation as electra.Attestation; - const committeeBitsLength = attestationElectra.committeeBits.bitLen; - - if (committeeBitsLength > committeeCount) { - throw new Error( - `Attestation committee bits length are longer than number of committees: committeeBitsLength=${committeeBitsLength} numCommittees=${committeeCount}` - ); - } - // TODO Electra: this should be obsolete soon when the spec switches to committeeIndices const committeeIndices = attestationElectra.committeeBits.getTrueBitIndexes(); + if (committeeIndices.length === 0) { + throw Error("Attestation should have at least one committee bit set"); + } else { + const lastCommitteeIndex = committeeIndices[committeeIndices.length - 1]; + if (lastCommitteeIndex >= committeeCount) { + throw new Error( + `Attestation committee index exceeds committee count: lastCommitteeIndex=${lastCommitteeIndex} numCommittees=${committeeCount}` + ); + } + } + // Get total number of attestation participant of every committee specified const participantCount = committeeIndices .map((committeeIndex) => epochCtx.getBeaconCommittee(data.slot, committeeIndex).length) diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 246c731d8382..ee8d2702c65d 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -19,8 +19,8 @@ import { MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, MAX_CONSOLIDATIONS, PENDING_BALANCE_DEPOSITS_LIMIT, - PENDING_CONSOLIDATIONS_LIMIT, PENDING_PARTIAL_WITHDRAWALS_LIMIT, + PENDING_CONSOLIDATIONS_LIMIT, } from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; @@ -335,15 +335,15 @@ export const BeaconState = new ContainerType( nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, // Deep history valid from Capella onwards historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, - depositRequestsStartIndex: UintBn64, // New in ELECTRA - depositBalanceToConsume: Gwei, // [New in Electra] - exitBalanceToConsume: Gwei, // [New in Electra] - earliestExitEpoch: Epoch, // [New in Electra] - consolidationBalanceToConsume: Gwei, // [New in Electra] - earliestConsolidationEpoch: Epoch, // [New in Electra] - pendingBalanceDeposits: new ListCompositeType(PendingBalanceDeposit, PENDING_BALANCE_DEPOSITS_LIMIT), // [New in Electra] - pendingPartialWithdrawals: new ListCompositeType(PartialWithdrawal, PENDING_PARTIAL_WITHDRAWALS_LIMIT), // [New in Electra] - pendingConsolidations: new ListCompositeType(PendingConsolidation, PENDING_CONSOLIDATIONS_LIMIT), // [New in Electra] + depositRequestsStartIndex: UintBn64, // New in ELECTRA:EIP6110 + depositBalanceToConsume: Gwei, // New in Electra:EIP7251 + exitBalanceToConsume: Gwei, // New in Electra:EIP7251 + earliestExitEpoch: Epoch, // New in Electra:EIP7251 + consolidationBalanceToConsume: Gwei, // New in Electra:EIP7251 + earliestConsolidationEpoch: Epoch, // New in Electra:EIP7251 + pendingBalanceDeposits: new ListCompositeType(PendingBalanceDeposit, Number(PENDING_BALANCE_DEPOSITS_LIMIT)), // new in electra:eip7251 + pendingPartialWithdrawals: new ListCompositeType(PartialWithdrawal, Number(PENDING_PARTIAL_WITHDRAWALS_LIMIT)), // New in Electra:EIP7251 + pendingConsolidations: new ListCompositeType(PendingConsolidation, Number(PENDING_CONSOLIDATIONS_LIMIT)), // new in electra:eip7251 }, {typeName: "BeaconState", jsonCase: "eth2"} ); From a0fff8f4ebd0e0abed455ddf93192e95f6310ab5 Mon Sep 17 00:00:00 2001 From: NC Date: Wed, 8 May 2024 18:37:14 +0300 Subject: [PATCH 037/145] fix: fix e2e test in electra-fork (#6751) Update spec version --- packages/params/test/e2e/ensure-config-is-synced.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/params/test/e2e/ensure-config-is-synced.test.ts b/packages/params/test/e2e/ensure-config-is-synced.test.ts index 06fb4bae000c..515d8f44a710 100644 --- a/packages/params/test/e2e/ensure-config-is-synced.test.ts +++ b/packages/params/test/e2e/ensure-config-is-synced.test.ts @@ -8,7 +8,7 @@ import {loadConfigYaml} from "../yaml.js"; // Not e2e, but slow. Run with e2e tests /** https://github.com/ethereum/consensus-specs/releases */ -const specConfigCommit = "v1.4.0-beta.5"; +const specConfigCommit = "v1.5.0-alpha.2"; describe("Ensure config is synced", function () { vi.setConfig({testTimeout: 60 * 1000}); From 15b7cdf8f9cd6ebd79a8e88abbce89f74c38075b Mon Sep 17 00:00:00 2001 From: g11tech Date: Thu, 9 May 2024 00:55:18 +0530 Subject: [PATCH 038/145] feat: get the basic integration working with the ethereumjs electra build (#6752) --- .../el-interop/ethereumjsdocker/electra.tmpl | 94 +++++++++++++++++++ .../test/sim/electra-interop.test.ts | 65 +++++++------ 2 files changed, 126 insertions(+), 33 deletions(-) create mode 100644 packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl diff --git a/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl b/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl new file mode 100644 index 000000000000..d557d505c649 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl @@ -0,0 +1,94 @@ +{ +"config": { +"chainId":1, +"homesteadBlock":0, +"eip150Block":0, +"eip155Block":0, +"eip158Block":0, +"byzantiumBlock":0, +"constantinopleBlock":0, +"petersburgBlock":0, +"istanbulBlock":0, +"muirGlacierBlock":0, +"berlinBlock":0, +"londonBlock":0, +"shanghaiTime":0, +"cancunTime": 0, +"pragueTime": 0, +"clique": { +"blockperiodseconds": 5, +"epochlength": 30000 +}, +"terminalTotalDifficulty":${TTD} +}, +"nonce":"0x42", +"timestamp":"0x0", +"extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", +"gasLimit":"0x1C9C380", +"difficulty":"0x400000000", +"mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", +"coinbase":"0x0000000000000000000000000000000000000000", +"alloc":{ + "0x610adc49ecd66cbf176a8247ebd59096c031bd9f": { + "balance": "0x6d6172697573766477000000" + }, + "0xa4664C40AACeBD82A2Db79f0ea36C06Bc6A19Adb": { + "balance": "1000000000000000000000000000" + }, + "0x00000000219ab540356cBB839Cbe05303d7705Fa": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a2646970667358221220dceca8706b29e917dacf25fceef95acac8d90d765ac926663ce4096195952b6164736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + }, + "0x25a219378dad9b3503c8268c9ca836a52427a4fb": { + "balance": "0", + "nonce": "1", + "code": "0x60203611603157600143035f35116029575f356120000143116029576120005f3506545f5260205ff35b5f5f5260205ff35b5f5ffd00" + }, + "0x00A3ca265EBcb825B45F985A16CEFB49958cE017": { + "balance": "0", + "nonce": "1", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe146090573615156028575f545f5260205ff35b366038141561012e5760115f54600182026001905f5b5f82111560595781019083028483029004916001019190603e565b90939004341061012e57600154600101600155600354806003026004013381556001015f3581556001016020359055600101600355005b6003546002548082038060101160a4575060105b5f5b81811460dd5780604c02838201600302600401805490600101805490600101549160601b83528260140152906034015260010160a6565b910180921460ed579060025560f8565b90505f6002555f6003555b5f548061049d141561010757505f5b60015460028282011161011c5750505f610122565b01600290035b5f555f600155604c025ff35b5f5ffd", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": + "0x000000000000000000000000000000000000000000000000000000000000049d" + } + } +}, +"number":"0x0", +"gasUsed":"0x0", +"parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", +"baseFeePerGas":"0x7" +} \ No newline at end of file diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index c503052a74f4..78010ff364ad 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -28,9 +28,7 @@ import {logFilesDir} from "./params.js"; import {shell} from "./shell.js"; // NOTE: How to run -// DEV_RUN=true EL_BINARY_DIR=naviechan/besu:v6110 EL_SCRIPT_DIR=besudocker yarn vitest --run test/sim/electra-interop.test.ts -// or -// DEV_RUN=true EL_BINARY_DIR=/Volumes/fast_boi/navie/Documents/workspace/besu/build/install/besu/bin EL_SCRIPT_DIR=besu yarn vitest --run test/sim/electra-interop.test.ts +// DEV_RUN=true EL_BINARY_DIR=ethpandaops/ethereumjs:master-0e06ddf EL_SCRIPT_DIR=ethereumjsdocker yarn vitest --run test/sim/electra-interop.test.ts // ``` /* eslint-disable no-console, @typescript-eslint/naming-convention */ @@ -110,35 +108,35 @@ describe("executionEngine / ExecutionEngineHttp", function () { // 2. Send raw deposit transaction A and B. tx A is to be imported via newPayload, tx B is to be included in payload via getPayload const depositTransactionA = - "0x02f9021c8217de808459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120749715de5d1226545c6b3790f515d551a5cc5bf1d49c87a696860554d2fc4f14000000000000000000000000000000000000000000000000000000000000003096a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9c080a09f597089338d7f44f5c59f8230bb38f243849228a8d4e9d2e2956e6050f5b2c7a076486996c7e62802b8f95eee114783e4b403fd11093ba96286ff42c595f24452"; + "0x02f90213018080648401c9c3809400000000219ab540356cbb839cbe05303d7705fa8901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000001208cd4e5a69709cf8ee5b1b73d6efbf3f33bcac92fb7e4ce62b2467542fb50a72d0000000000000000000000000000000000000000000000000000000000000030ac842878bb70009552a4cfcad801d6e659c50bd50d7d03306790cb455ce7363c5b6972f0159d170f625a99b2064dbefc000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020010000000000000000000000818ccb1c4eda80270b04d6df822b1e72dd83c3030000000000000000000000000000000000000000000000000000000000000060a747f75c72d0cf0d2b52504c7385b516f0523e2f0842416399f42b4aee5c6384a5674f6426b1cc3d0827886fa9b909e616f5c9f61f986013ed2b9bf37071cbae951136265b549f44e3c8e26233c0433e9124b7fd0dc86e82f9fedfc0a179d769c080a067c9857d27a42f8fde4d5cf2d6c324af94469ac93ec867eacdd9002e1297835fa07927224866e03d51fb1ae94390e7aec453cad8df9e048892e98f945178eab254"; const depositRequestA = { amount: 32000000000, index: 0, pubkey: dataToBytes( - "0x96a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9", + "0xac842878bb70009552a4cfcad801d6e659c50bd50d7d03306790cb455ce7363c5b6972f0159d170f625a99b2064dbefc", 48 ), signature: dataToBytes( - "0xb1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9", + "0xa747f75c72d0cf0d2b52504c7385b516f0523e2f0842416399f42b4aee5c6384a5674f6426b1cc3d0827886fa9b909e616f5c9f61f986013ed2b9bf37071cbae951136265b549f44e3c8e26233c0433e9124b7fd0dc86e82f9fedfc0a179d769", 96 ), - withdrawalCredentials: dataToBytes("0x003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef2", 32), + withdrawalCredentials: dataToBytes("0x010000000000000000000000818ccb1c4eda80270b04d6df822b1e72dd83c303", 32), }; const depositTransactionB = - "0x02f9021c8217de018459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120a18b4c7cab0afa273ea9504904521ea8421a4e32740b7611bd3d5095ca99f0cb0000000000000000000000000000000000000000000000000000000000000030a5c85a60ba2905c215f6a12872e62b1ee037051364244043a5f639aa81b04a204c55e7cc851f29c7c183be253ea1510b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020001db70c485b6264692f26b8aeaab5b0c384180df8e2184a21a808a3ec8e86ca00000000000000000000000000000000000000000000000000000000000000609561731785b48cf1886412234531e4940064584463e96ac63a1a154320227e333fb51addc4a89b7e0d3f862d7c1fd4ea03bd8eb3d8806f1e7daf591cbbbb92b0beb74d13c01617f22c5026b4f9f9f294a8a7c32db895de3b01bee0132c9209e1c001a0644e0a763a34b4bfb9f56a677857b57fcf15e3db57e2f57060e92084f75f3d82a018ba8eaacbd8e6f6917675b1d0362b12ca82850ca8ef9c010430760c2b2e0cb5"; + "0x02f90213010180648401c9c3809400000000219ab540356cbb839cbe05303d7705fa8901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120a7ec6a3459bf9389265f62abbdffcd0ef20924bd03e4856d3b964edf565bd8e80000000000000000000000000000000000000000000000000000000000000030a5290ddb9abd6a7fb8bac3414c6c7ff093a18ff297c1eada20464de388b14aafa505bfc98847ca7e6f7ca3aa9d4ca769000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020010000000000000000000000da628fed218cbe3a9e684a9f51c49dd63a229a1d000000000000000000000000000000000000000000000000000000000000006080e12262f94795ce3453f17eea2dd44843ff7977d303b192c1d2a4ce0dbebc8856c398d6445cbf244ba9e99307ead1e30b2544a5e9693cdd5196a33c46e2dd8a8b83afc8278c1ea79cd5c13cac2b96a62257b3636787d0f1e0f881c50a4667ddc080a0b653aad27e504d4fcd19b7c317ffbd2a26a81d6ac14ecea6a891a63dcf7816dfa02953273b4cddc93b2a9ba21aaeb0db988cb1086319dd0b91f79bc101adfe32e4"; const depositRequestB = { amount: 32000000000, index: 1, pubkey: dataToBytes( - "0xa5c85a60ba2905c215f6a12872e62b1ee037051364244043a5f639aa81b04a204c55e7cc851f29c7c183be253ea1510b", + "0xa5290ddb9abd6a7fb8bac3414c6c7ff093a18ff297c1eada20464de388b14aafa505bfc98847ca7e6f7ca3aa9d4ca769", 48 ), signature: dataToBytes( - "0x9561731785b48cf1886412234531e4940064584463e96ac63a1a154320227e333fb51addc4a89b7e0d3f862d7c1fd4ea03bd8eb3d8806f1e7daf591cbbbb92b0beb74d13c01617f22c5026b4f9f9f294a8a7c32db895de3b01bee0132c9209e1", + "0x80e12262f94795ce3453f17eea2dd44843ff7977d303b192c1d2a4ce0dbebc8856c398d6445cbf244ba9e99307ead1e30b2544a5e9693cdd5196a33c46e2dd8a8b83afc8278c1ea79cd5c13cac2b96a62257b3636787d0f1e0f881c50a4667dd", 96 ), - withdrawalCredentials: dataToBytes("0x001db70c485b6264692f26b8aeaab5b0c384180df8e2184a21a808a3ec8e86ca", 32), + withdrawalCredentials: dataToBytes("0x010000000000000000000000da628fed218cbe3a9e684a9f51c49dd63a229a1d", 32), }; sendRawTransactionBig(ethRpcUrl, depositTransactionA, `${dataPath}/deposit.json`).catch((e: Error) => { @@ -150,18 +148,18 @@ describe("executionEngine / ExecutionEngineHttp", function () { }); // 3. Import new payload with tx A and deposit receipt A - const newPayloadBlockHash = "0xfd1189e6ea0814b7d40d4e50b31ae5feabbb2acff39399457bbdda7cb5ccd490"; + const newPayloadBlockHash = "0x4cec1852552239cf78e8bd2db35ff9396acb6b40c3ce486e6e3028bc75c9faec"; const newPayload = { - parentHash: dataToBytes("0x26118cf71453320edcebbc4ebb34af5b578087a32385b80108bf691fa23efc42", 32), + parentHash: dataToBytes("0xeb86e5aca89ea5477a6e169a389efbbe7e5a3d5f5c5296bcde3a4b032ea9bae8", 32), feeRecipient: dataToBytes("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", 20), - stateRoot: dataToBytes("0x14208ac0e218167936e220b72d5d5887a963cb858ea2f2d268518f014a3da3fa", 32), + stateRoot: dataToBytes("0x686ce0478cabce79b298712fefee4aefd2fac1ab4a4813936d2c1ccca788bbc3", 32), logsBloom: dataToBytes( - "0x10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000", + "0x00000000000000000000400000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000020000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000", 256 ), prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), gasLimit: 30000000, - gasUsed: 84846, + gasUsed: 84714, timestamp: 16, extraData: dataToBytes("0x", 0), baseFeePerGas: 7n, @@ -171,7 +169,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { depositRequests: [depositRequestA], blockNumber: 1, blockHash: dataToBytes(newPayloadBlockHash, 32), - receiptsRoot: dataToBytes("0x79ee3424eb720a3ad4b1c5a372bb8160580cbe4d893778660f34213c685627a9", 32), + receiptsRoot: dataToBytes("0x0b67bea29f17eeb290685e01e9a2e4cd77a83471d9985a8ce27997a7ed3ee3f8", 32), blobGasUsed: 0n, withdrawalRequests: [], }; @@ -235,21 +233,22 @@ describe("executionEngine / ExecutionEngineHttp", function () { ); }); - it("Post-merge, run for a few blocks", async function () { - console.log("\n\nPost-merge, run for a few blocks\n\n"); - const {elClient, tearDownCallBack} = await runEL( - {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, - {...elRunOptions, ttd: BigInt(0)}, - controller.signal - ); - afterEachCallbacks.push(() => tearDownCallBack()); - - await runNodeWithEL({ - elClient, - electraEpoch: 0, - testName: "post-merge", - }); - }); + // TODO: get this post merge run working + // it("Post-merge, run for a few blocks", async function () { + // console.log("\n\nPost-merge, run for a few blocks\n\n"); + // const {elClient, tearDownCallBack} = await runEL( + // {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, + // {...elRunOptions, ttd: BigInt(0)}, + // controller.signal + // ); + // afterEachCallbacks.push(() => tearDownCallBack()); + + // await runNodeWithEL({ + // elClient, + // electraEpoch: 0, + // testName: "post-merge", + // }); + // }); /** * Want to test two things: @@ -364,7 +363,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { // send raw tx at slot 1 const depositTransaction = - "0x02f9021e8217de8085012a05f20085019254d380830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120ef950826b191ebea0bbafa92a2c6bffa8239c6f456d92891ce2852b8360f0d30000000000000000000000000000000000000000000000000000000000000003095e4f91aea91a9e00387fad9d60997cff6cbf68d42d1b6629a7b248cdef255f94a2a2381e5d4125273fe42da5f7aa0e1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b6c06e65228046268aa918baf78e072c25e65aa0bcf258cefcac3371c47df81bc4d43ca942f5fc28f9a563e925fd9c5010bc8c300add3faf3af0d61fabaaf03694020feaafb03e47c1bc4fcf082684c7ed3f7d5839d1722214b24f95ad2b226cc080a0be1161617492e4ca2fcb89edcadf5e71e8cac0d6447d18cfde9b55e5a8412417a07ec8c47dd484036c745049bb2e2980d44e38d4dacac50dc4a14a2f23c52f2e5f"; + "0x02f90213018080648401c9c3809400000000219ab540356cbb839cbe05303d7705fa8901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000001208cd4e5a69709cf8ee5b1b73d6efbf3f33bcac92fb7e4ce62b2467542fb50a72d0000000000000000000000000000000000000000000000000000000000000030ac842878bb70009552a4cfcad801d6e659c50bd50d7d03306790cb455ce7363c5b6972f0159d170f625a99b2064dbefc000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020010000000000000000000000818ccb1c4eda80270b04d6df822b1e72dd83c3030000000000000000000000000000000000000000000000000000000000000060a747f75c72d0cf0d2b52504c7385b516f0523e2f0842416399f42b4aee5c6384a5674f6426b1cc3d0827886fa9b909e616f5c9f61f986013ed2b9bf37071cbae951136265b549f44e3c8e26233c0433e9124b7fd0dc86e82f9fedfc0a179d769c080a067c9857d27a42f8fde4d5cf2d6c324af94469ac93ec867eacdd9002e1297835fa07927224866e03d51fb1ae94390e7aec453cad8df9e048892e98f945178eab254"; sendRawTransactionBig(ethRpcUrl, depositTransaction, `${dataPath}/deposit.json`).catch((e: Error) => { loggerNodeA.error("Fail to send raw deposit transaction", undefined, e); }); From 9f599f38a9b42cc44e5dec8ffc4401466654e840 Mon Sep 17 00:00:00 2001 From: g11tech Date: Thu, 9 May 2024 22:29:39 +0530 Subject: [PATCH 039/145] feat: apply some fixes and hacks to get the single node devnet working with fork transition (#6754) --- .../chain/produceBlock/produceBlockBody.ts | 5 ++ .../src/slot/upgradeStateToElectra.ts | 74 +++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index ba560d5a7ff0..53f59850b647 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -17,6 +17,7 @@ import { BlindedBeaconBlockBody, BlindedBeaconBlock, sszTypesFor, + electra, } from "@lodestar/types"; import { CachedBeaconStateAllForks, @@ -342,6 +343,10 @@ export async function produceBlockBody( } } + if (ForkSeq[fork] >= ForkSeq.electra) { + (blockBody as electra.BeaconBlockBody).consolidations = []; + } + Object.assign(logMeta, {executionPayloadValue}); this.logger.verbose("Produced beacon block body", logMeta); diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index a6bc6c331e11..121425265b50 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -14,6 +14,80 @@ import { export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): CachedBeaconStateElectra { const {config} = stateDeneb; + ssz.deneb.BeaconState.commitViewDU(stateDeneb); + const stateElectraCloned = stateDeneb; + + const stateElectraView = ssz.electra.BeaconState.defaultViewDU(); + stateElectraView.genesisTime = stateElectraCloned.genesisTime; + stateElectraView.genesisValidatorsRoot = stateElectraCloned.genesisValidatorsRoot; + stateElectraView.slot = stateElectraCloned.slot; + stateElectraView.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: stateDeneb.fork.currentVersion, + currentVersion: config.ELECTRA_FORK_VERSION, + epoch: stateDeneb.epochCtx.epoch, + }); + stateElectraView.latestBlockHeader = stateElectraCloned.latestBlockHeader; + stateElectraView.blockRoots = stateElectraCloned.blockRoots; + stateElectraView.stateRoots = stateElectraCloned.stateRoots; + stateElectraView.historicalRoots = stateElectraCloned.historicalRoots; + stateElectraView.eth1Data = stateElectraCloned.eth1Data; + stateElectraView.eth1DataVotes = stateElectraCloned.eth1DataVotes; + stateElectraView.eth1DepositIndex = stateElectraCloned.eth1DepositIndex; + stateElectraView.validators = stateElectraCloned.validators; + stateElectraView.balances = stateElectraCloned.balances; + stateElectraView.randaoMixes = stateElectraCloned.randaoMixes; + stateElectraView.slashings = stateElectraCloned.slashings; + stateElectraView.previousEpochParticipation = stateElectraCloned.previousEpochParticipation; + stateElectraView.currentEpochParticipation = stateElectraCloned.currentEpochParticipation; + stateElectraView.justificationBits = stateElectraCloned.justificationBits; + stateElectraView.previousJustifiedCheckpoint = stateElectraCloned.previousJustifiedCheckpoint; + stateElectraView.currentJustifiedCheckpoint = stateElectraCloned.currentJustifiedCheckpoint; + stateElectraView.finalizedCheckpoint = stateElectraCloned.finalizedCheckpoint; + stateElectraView.inactivityScores = stateElectraCloned.inactivityScores; + stateElectraView.currentSyncCommittee = stateElectraCloned.currentSyncCommittee; + stateElectraView.nextSyncCommittee = stateElectraCloned.nextSyncCommittee; + stateElectraView.latestExecutionPayloadHeader = ssz.electra.BeaconState.fields.latestExecutionPayloadHeader.toViewDU({ + ...stateElectraCloned.latestExecutionPayloadHeader.toValue(), + depositRequestsRoot: ssz.Root.defaultValue(), + withdrawalRequestsRoot: ssz.Root.defaultValue(), + }); + stateElectraView.nextWithdrawalIndex = stateDeneb.nextWithdrawalIndex; + stateElectraView.nextWithdrawalValidatorIndex = stateDeneb.nextWithdrawalValidatorIndex; + stateElectraView.historicalSummaries = stateElectraCloned.historicalSummaries; + + // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default + // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectraView.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + + const validatorsArr = stateElectraView.validators.getAllReadonly(); + + for (let i = 0; i < validatorsArr.length; i++) { + const validator = validatorsArr[i]; + + // [EIP-7251]: add validators that are not yet active to pending balance deposits + if (validator.activationEligibilityEpoch === FAR_FUTURE_EPOCH) { + queueEntireBalanceAndResetValidator(stateElectraView as CachedBeaconStateElectra, i); + } + + // [EIP-7251]: Ensure early adopters of compounding credentials go through the activation churn + const withdrawalCredential = validator.withdrawalCredentials; + if (hasCompoundingWithdrawalCredential(withdrawalCredential)) { + queueExcessActiveBalance(stateElectraView as CachedBeaconStateElectra, i); + } + } + + const stateElectra = getCachedBeaconState(stateElectraView, stateDeneb); + // Commit new added fields ViewDU to the root node + stateElectra.commit(); + // Clear cache to ensure the cache of deneb fields is not used by new ELECTRA fields + stateElectra["clearCache"](); + + return stateElectra; +} + +export function upgradeStateToElectraOriginal(stateDeneb: CachedBeaconStateDeneb): CachedBeaconStateElectra { + const {config} = stateDeneb; + const stateElectraNode = ssz.deneb.BeaconState.commitViewDU(stateDeneb); const stateElectraView = ssz.electra.BeaconState.getViewDU(stateElectraNode); From c34eac700460d036aed302f8ac5d40cb6fdadfb7 Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 10 May 2024 17:52:08 +0700 Subject: [PATCH 040/145] fix: get aggregate and proofs signature sets (#6757) fix: get signature for SignedAggregateAndProof based on fork --- .../validation/signatureSets/aggregateAndProof.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts index 59787341cfb9..411f33f68e66 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -1,7 +1,7 @@ -import {PublicKey} from "@chainsafe/blst"; -import {DOMAIN_AGGREGATE_AND_PROOF} from "@lodestar/params"; -import {ssz} from "@lodestar/types"; -import {Epoch, phase0} from "@lodestar/types"; +import type {PublicKey} from "@chainsafe/bls/types"; +import {DOMAIN_AGGREGATE_AND_PROOF, ForkSeq} from "@lodestar/params"; +import {allForks, ssz} from "@lodestar/types"; +import {Epoch} from "@lodestar/types"; import { computeSigningRoot, computeStartSlotAtEpoch, @@ -13,7 +13,7 @@ import {BeaconConfig} from "@lodestar/config"; export function getAggregateAndProofSigningRoot( config: BeaconConfig, epoch: Epoch, - aggregateAndProof: phase0.SignedAggregateAndProof + aggregateAndProof: allForks.SignedAggregateAndProof ): Uint8Array { // previously, we call `const aggregatorDomain = state.config.getDomain(state.slot, DOMAIN_AGGREGATE_AND_PROOF, slot);` // at fork boundary, it's required to dial to target epoch https://github.com/ChainSafe/lodestar/blob/v1.11.3/packages/beacon-node/src/chain/validation/attestation.ts#L573 @@ -21,14 +21,15 @@ export function getAggregateAndProofSigningRoot( const slot = computeStartSlotAtEpoch(epoch); const fork = config.getForkName(slot); const aggregatorDomain = config.getDomainAtFork(fork, DOMAIN_AGGREGATE_AND_PROOF); - return computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof.message, aggregatorDomain); + const sszType = ForkSeq[fork] >= ForkSeq.electra ? ssz.electra.AggregateAndProof : ssz.phase0.AggregateAndProof; + return computeSigningRoot(sszType, aggregateAndProof.message, aggregatorDomain); } export function getAggregateAndProofSignatureSet( config: BeaconConfig, epoch: Epoch, aggregator: PublicKey, - aggregateAndProof: phase0.SignedAggregateAndProof + aggregateAndProof: allForks.SignedAggregateAndProof ): ISignatureSet { return createSingleSignatureSetFromComponents( aggregator, From 81e76820c6957279ae96e01e3fabfb19d2e06cec Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Sat, 11 May 2024 00:01:42 +0200 Subject: [PATCH 041/145] test(spec): fix attestors slashing specs for electra fork (#6758) * Fix attester slashing specs for electra * Remove unused import * Add code comment * Update the expression * Update the fork check --- packages/beacon-node/test/spec/utils/specTestIterator.ts | 3 +-- .../src/block/isValidIndexedAttestation.ts | 8 ++++++-- packages/state-transition/src/util/epoch.ts | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index b99bc281cf50..393983bb8a56 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -65,8 +65,7 @@ export const defaultSkipOpts: SkipOpts = { skippedTestSuites: [ /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, - // /^electra\/(?!operations\/attestations)(?!operations\/attester_slashing)/, - /^electra\/(?!operations\/attestation)/, + /^electra\/(?!operations\/attestations)(?!operations\/attester_slashing)/, ], skippedTests: [], skippedRunners: ["merkle_proof", "networking"], diff --git a/packages/state-transition/src/block/isValidIndexedAttestation.ts b/packages/state-transition/src/block/isValidIndexedAttestation.ts index e3965b97ee73..33d92a208260 100644 --- a/packages/state-transition/src/block/isValidIndexedAttestation.ts +++ b/packages/state-transition/src/block/isValidIndexedAttestation.ts @@ -1,4 +1,4 @@ -import {MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; +import {ForkSeq, MAX_COMMITTEES_PER_SLOT, MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; import {phase0} from "@lodestar/types"; import {CachedBeaconStateAllForks} from "../types.js"; import {verifySignatureSet} from "../util/index.js"; @@ -44,7 +44,11 @@ export function isValidIndexedAttestationBigint( */ export function isValidIndexedAttestationIndices(state: CachedBeaconStateAllForks, indices: number[]): boolean { // verify max number of indices - if (!(indices.length > 0 && indices.length <= MAX_VALIDATORS_PER_COMMITTEE)) { + const maxIndices = + state.config.getForkSeq(state.slot) >= ForkSeq.electra + ? MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT + : MAX_VALIDATORS_PER_COMMITTEE; + if (!(indices.length > 0 && indices.length <= maxIndices)) { return false; } diff --git a/packages/state-transition/src/util/epoch.ts b/packages/state-transition/src/util/epoch.ts index 3ac77556e3cb..67837780dca9 100644 --- a/packages/state-transition/src/util/epoch.ts +++ b/packages/state-transition/src/util/epoch.ts @@ -52,7 +52,7 @@ export function computeExitEpochAndUpdateChurn(state: CachedBeaconStateElectra, // Exit doesn't fit in the current earliest epoch. if (exitBalance > exitBalanceToConsume) { const balanceToProcess = Number(exitBalance) - exitBalanceToConsume; - const additionalEpochs = Math.floor((balanceToProcess - 1) / (perEpochChurn + 1)); + const additionalEpochs = Math.floor((balanceToProcess - 1) / perEpochChurn) + 1; earliestExitEpoch += additionalEpochs; exitBalanceToConsume += additionalEpochs * perEpochChurn; } From f2be31798f96dd104d9bfaa9d9e0b420e6714cfe Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 13 May 2024 14:02:31 +0530 Subject: [PATCH 042/145] chore: fix types and lint (#6750) * chore: fix types and lint * fx * type and lint fix --- .../src/api/impl/config/constants.ts | 10 +++++++ .../test/sim/electra-interop.test.ts | 30 +++++++++---------- .../spec/presets/epoch_processing.test.ts | 10 +++++-- .../test/spec/presets/operations.test.ts | 3 +- packages/types/src/electra/index.ts | 5 ++-- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/packages/beacon-node/src/api/impl/config/constants.ts b/packages/beacon-node/src/api/impl/config/constants.ts index 87ffce91b4d9..b5f810157581 100644 --- a/packages/beacon-node/src/api/impl/config/constants.ts +++ b/packages/beacon-node/src/api/impl/config/constants.ts @@ -36,6 +36,10 @@ import { SYNC_COMMITTEE_SUBNET_COUNT, BLOB_TX_TYPE, VERSIONED_HASH_VERSION_KZG, + COMPOUNDING_WITHDRAWAL_PREFIX, + DOMAIN_CONSOLIDATION, + UNSET_DEPOSIT_RECEIPTS_START_INDEX, + FULL_EXIT_REQUEST_AMOUNT, } from "@lodestar/params"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -57,6 +61,7 @@ export const specConstants = { // ## Withdrawal prefixes BLS_WITHDRAWAL_PREFIX, ETH1_ADDRESS_WITHDRAWAL_PREFIX, + COMPOUNDING_WITHDRAWAL_PREFIX, // ## Domain types DOMAIN_BEACON_PROPOSER, DOMAIN_BEACON_ATTESTER, @@ -66,6 +71,7 @@ export const specConstants = { DOMAIN_SELECTION_PROOF, DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_APPLICATION_BUILDER, + DOMAIN_CONSOLIDATION, // phase0/validator.md TARGET_AGGREGATORS_PER_COMMITTEE, @@ -100,4 +106,8 @@ export const specConstants = { // Deneb types BLOB_TX_TYPE, VERSIONED_HASH_VERSION_KZG, + + // electra + UNSET_DEPOSIT_RECEIPTS_START_INDEX, + FULL_EXIT_REQUEST_AMOUNT, }; diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 78010ff364ad..2dfa5ee0f77c 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -234,21 +234,21 @@ describe("executionEngine / ExecutionEngineHttp", function () { }); // TODO: get this post merge run working - // it("Post-merge, run for a few blocks", async function () { - // console.log("\n\nPost-merge, run for a few blocks\n\n"); - // const {elClient, tearDownCallBack} = await runEL( - // {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, - // {...elRunOptions, ttd: BigInt(0)}, - // controller.signal - // ); - // afterEachCallbacks.push(() => tearDownCallBack()); - - // await runNodeWithEL({ - // elClient, - // electraEpoch: 0, - // testName: "post-merge", - // }); - // }); + it.skip("Post-merge, run for a few blocks", async function () { + console.log("\n\nPost-merge, run for a few blocks\n\n"); + const {elClient, tearDownCallBack} = await runEL( + {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, + {...elRunOptions, ttd: BigInt(0)}, + controller.signal + ); + afterEachCallbacks.push(() => tearDownCallBack()); + + await runNodeWithEL({ + elClient, + electraEpoch: 0, + testName: "post-merge", + }); + }); /** * Want to test two things: diff --git a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts index a244762143f3..b089863fdad3 100644 --- a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts +++ b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts @@ -22,7 +22,10 @@ export type EpochTransitionFn = (state: CachedBeaconStateAllForks, epochTransiti /* eslint-disable @typescript-eslint/naming-convention */ const epochTransitionFns: Record = { - effective_balance_updates: epochFns.processEffectiveBalanceUpdates, + effective_balance_updates: (state, epochTransitionCache) => { + const fork = state.config.getForkSeq(state.slot); + epochFns.processEffectiveBalanceUpdates(fork, state, epochTransitionCache); + }, eth1_data_reset: epochFns.processEth1DataReset, historical_roots_update: epochFns.processHistoricalRootsUpdate, inactivity_updates: epochFns.processInactivityUpdates as EpochTransitionFn, @@ -30,7 +33,10 @@ const epochTransitionFns: Record = { participation_flag_updates: epochFns.processParticipationFlagUpdates as EpochTransitionFn, participation_record_updates: epochFns.processParticipationRecordUpdates as EpochTransitionFn, randao_mixes_reset: epochFns.processRandaoMixesReset, - registry_updates: epochFns.processRegistryUpdates, + registry_updates: (state, epochTransitionCache) => { + const fork = state.config.getForkSeq(state.slot); + epochFns.processRegistryUpdates(fork, state, epochTransitionCache); + }, rewards_and_penalties: epochFns.processRewardsAndPenalties, slashings: epochFns.processSlashings, slashings_reset: epochFns.processSlashingsReset, diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index ca215f1e197c..99d0e578d244 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -71,7 +71,8 @@ const operationFns: Record> = sync_aggregate_random: sync_aggregate, voluntary_exit: (state, testCase: {voluntary_exit: phase0.SignedVoluntaryExit}) => { - blockFns.processVoluntaryExit(state, testCase.voluntary_exit); + const fork = state.config.getForkSeq(state.slot); + blockFns.processVoluntaryExit(fork, state, testCase.voluntary_exit); }, execution_payload: (state, testCase: {body: bellatrix.BeaconBlockBody; execution: {execution_valid: boolean}}) => { diff --git a/packages/types/src/electra/index.ts b/packages/types/src/electra/index.ts index 7856cd729620..981b2015e02a 100644 --- a/packages/types/src/electra/index.ts +++ b/packages/types/src/electra/index.ts @@ -1,3 +1,4 @@ export * from "./types.js"; -export * as ts from "./types.js"; -export * as ssz from "./sszTypes.js"; +import * as ts from "./types.js"; +import * as ssz from "./sszTypes.js"; +export {ts, ssz}; From 6393d0a67360b79171a0c334f7b5fb1bac78a58b Mon Sep 17 00:00:00 2001 From: NC Date: Tue, 14 May 2024 05:52:02 +0300 Subject: [PATCH 043/145] fix: fix electra genesis spec test (#6764) * process pending deposit from eth1 * Fix the genesis params * fix * Fix * clean up --------- Co-authored-by: Nazar Hussain --- .../beacon-node/src/chain/genesis/genesis.ts | 8 ++++- packages/state-transition/src/util/genesis.ts | 29 ++++++++++++++----- packages/types/src/electra/sszTypes.ts | 7 ++++- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/packages/beacon-node/src/chain/genesis/genesis.ts b/packages/beacon-node/src/chain/genesis/genesis.ts index 979476c69530..5efd352357eb 100644 --- a/packages/beacon-node/src/chain/genesis/genesis.ts +++ b/packages/beacon-node/src/chain/genesis/genesis.ts @@ -174,7 +174,13 @@ export class GenesisBuilder implements IGenesisBuilder { }; }); - const {activatedValidatorCount} = applyDeposits(this.config, this.state, newDeposits, this.depositTree); + const {activatedValidatorCount} = applyDeposits( + this.config.getForkSeq(this.state.slot), + this.config, + this.state, + newDeposits, + this.depositTree + ); this.activatedValidatorCount += activatedValidatorCount; // TODO: If necessary persist deposits here to this.db.depositData, this.db.depositDataRoot diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index d5f4c577711f..ca894fd5b4e5 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -4,6 +4,7 @@ import { EFFECTIVE_BALANCE_INCREMENT, EPOCHS_PER_HISTORICAL_VECTOR, ForkName, + ForkSeq, GENESIS_EPOCH, GENESIS_SLOT, MAX_EFFECTIVE_BALANCE, @@ -11,10 +12,11 @@ import { } from "@lodestar/params"; import {Bytes32, phase0, Root, ssz, TimeSeconds} from "@lodestar/types"; -import {CachedBeaconStateAllForks, BeaconStateAllForks} from "../types.js"; +import {CachedBeaconStateAllForks, BeaconStateAllForks, CachedBeaconStateElectra} from "../types.js"; import {createCachedBeaconState} from "../cache/stateCache.js"; import {EpochCacheImmutableData} from "../cache/epochCache.js"; import {processDeposit} from "../block/processDeposit.js"; +import {increaseBalance} from "../index.js"; import {computeEpochAtSlot} from "./epoch.js"; import {getActiveValidatorIndices} from "./validator.js"; import {getTemporaryBlockHeader} from "./blockRoot.js"; @@ -127,6 +129,7 @@ export function applyTimestamp(config: ChainForkConfig, state: CachedBeaconState * @returns active validator indices */ export function applyDeposits( + fork: ForkSeq, config: ChainForkConfig, state: CachedBeaconStateAllForks, newDeposits: phase0.Deposit[], @@ -164,6 +167,16 @@ export function applyDeposits( processDeposit(fork, state, deposit); } + // Process deposit balance updates + if (fork >= ForkSeq.electra) { + const stateElectra = state as CachedBeaconStateElectra; + stateElectra.commit(); + for (const {index: validatorIndex, amount} of stateElectra.pendingBalanceDeposits.getAllReadonly()) { + increaseBalance(state, validatorIndex, Number(amount)); + } + stateElectra.pendingBalanceDeposits = ssz.electra.PendingBalanceDeposits.defaultViewDU(); + } + // Process activations const {epochCtx} = state; const balancesArr = state.balances.getAll(); @@ -226,6 +239,8 @@ export function initializeBeaconStateFromEth1( getTemporaryBlockHeader(config, config.getForkTypes(GENESIS_SLOT).BeaconBlock.defaultValue()) ); + const fork = config.getForkSeq(GENESIS_SLOT); + // We need a CachedBeaconState to run processDeposit() which uses various caches. // However at this point the state's syncCommittees are not known. // This function can be called by: @@ -240,13 +255,13 @@ export function initializeBeaconStateFromEth1( applyEth1BlockHash(state, eth1BlockHash); // Process deposits - applyDeposits(config, state, deposits, fullDepositDataRootList); + applyDeposits(fork, config, state, deposits, fullDepositDataRootList); // Commit before reading all validators in `getActiveValidatorIndices()` state.commit(); const activeValidatorIndices = getActiveValidatorIndices(state, computeEpochAtSlot(GENESIS_SLOT)); - if (GENESIS_SLOT >= config.ALTAIR_FORK_EPOCH) { + if (fork >= ForkSeq.altair) { const {syncCommittee} = getNextSyncCommittee( state, activeValidatorIndices, @@ -259,7 +274,7 @@ export function initializeBeaconStateFromEth1( stateAltair.nextSyncCommittee = ssz.altair.SyncCommittee.toViewDU(syncCommittee); } - if (GENESIS_SLOT >= config.BELLATRIX_FORK_EPOCH) { + if (fork >= ForkSeq.bellatrix) { const stateBellatrix = state as CompositeViewDU; stateBellatrix.fork.previousVersion = config.BELLATRIX_FORK_VERSION; stateBellatrix.fork.currentVersion = config.BELLATRIX_FORK_VERSION; @@ -268,7 +283,7 @@ export function initializeBeaconStateFromEth1( ssz.bellatrix.ExecutionPayloadHeader.defaultViewDU(); } - if (GENESIS_SLOT >= config.CAPELLA_FORK_EPOCH) { + if (fork >= ForkSeq.capella) { const stateCapella = state as CompositeViewDU; stateCapella.fork.previousVersion = config.CAPELLA_FORK_VERSION; stateCapella.fork.currentVersion = config.CAPELLA_FORK_VERSION; @@ -277,7 +292,7 @@ export function initializeBeaconStateFromEth1( ssz.capella.ExecutionPayloadHeader.defaultViewDU(); } - if (GENESIS_SLOT >= config.DENEB_FORK_EPOCH) { + if (fork >= ForkSeq.deneb) { const stateDeneb = state as CompositeViewDU; stateDeneb.fork.previousVersion = config.DENEB_FORK_VERSION; stateDeneb.fork.currentVersion = config.DENEB_FORK_VERSION; @@ -286,7 +301,7 @@ export function initializeBeaconStateFromEth1( ssz.deneb.ExecutionPayloadHeader.defaultViewDU(); } - if (GENESIS_SLOT >= config.ELECTRA_FORK_EPOCH) { + if (fork >= ForkSeq.electra) { const stateElectra = state as CompositeViewDU; stateElectra.fork.previousVersion = config.ELECTRA_FORK_VERSION; stateElectra.fork.currentVersion = config.ELECTRA_FORK_VERSION; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index ee8d2702c65d..d3d9843e63f1 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -275,6 +275,11 @@ export const PendingBalanceDeposit = new ContainerType( {typeName: "PendingBalanceDeposit", jsonCase: "eth2"} ); +export const PendingBalanceDeposits = new ListCompositeType( + PendingBalanceDeposit, + Number(PENDING_BALANCE_DEPOSITS_LIMIT) +); + export const PartialWithdrawal = new ContainerType( { index: ValidatorIndex, @@ -341,7 +346,7 @@ export const BeaconState = new ContainerType( earliestExitEpoch: Epoch, // New in Electra:EIP7251 consolidationBalanceToConsume: Gwei, // New in Electra:EIP7251 earliestConsolidationEpoch: Epoch, // New in Electra:EIP7251 - pendingBalanceDeposits: new ListCompositeType(PendingBalanceDeposit, Number(PENDING_BALANCE_DEPOSITS_LIMIT)), // new in electra:eip7251 + pendingBalanceDeposits: PendingBalanceDeposits, // new in electra:eip7251 pendingPartialWithdrawals: new ListCompositeType(PartialWithdrawal, Number(PENDING_PARTIAL_WITHDRAWALS_LIMIT)), // New in Electra:EIP7251 pendingConsolidations: new ListCompositeType(PendingConsolidation, Number(PENDING_CONSOLIDATIONS_LIMIT)), // new in electra:eip7251 }, From 55567db67ea185c46cb7c4b20df6d3846f14403b Mon Sep 17 00:00:00 2001 From: NC Date: Tue, 14 May 2024 06:39:55 +0300 Subject: [PATCH 044/145] feat: support missing electra spec test (#6765) Add spec test --- .../test/spec/presets/epoch_processing.test.ts | 2 ++ .../test/spec/presets/operations.test.ts | 17 +++++++++++++++++ .../src/block/processOperations.ts | 1 + 3 files changed, 20 insertions(+) diff --git a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts index b089863fdad3..f159612e416c 100644 --- a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts +++ b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts @@ -42,6 +42,8 @@ const epochTransitionFns: Record = { slashings_reset: epochFns.processSlashingsReset, sync_committee_updates: epochFns.processSyncCommitteeUpdates as EpochTransitionFn, historical_summaries_update: epochFns.processHistoricalSummariesUpdate as EpochTransitionFn, + pending_balance_deposits: epochFns.processPendingBalanceDeposits as EpochTransitionFn, + pending_consolidations: epochFns.processPendingConsolidations as EpochTransitionFn, }; /** diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 99d0e578d244..fe70c091aa5a 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -91,6 +91,21 @@ const operationFns: Record> = withdrawals: (state, testCase: {execution_payload: capella.ExecutionPayload}) => { blockFns.processWithdrawals(ForkSeq.capella, state as CachedBeaconStateCapella, testCase.execution_payload); }, + + consolidation: (state, testCase: {consolidation: electra.SignedConsolidation}) => { + blockFns.processConsolidation(state as CachedBeaconStateElectra, testCase.consolidation); + }, + + execution_layer_withdrawal_request: ( + state, + testCase: {execution_layer_withdrawal_request: electra.ExecutionLayerWithdrawalRequest} + ) => { + blockFns.processExecutionLayerWithdrawalRequest( + ForkSeq.electra, + state as CachedBeaconStateElectra, + testCase.execution_layer_withdrawal_request + ); + }, }; export type BlockProcessFn = (state: T, testCase: any) => void; @@ -140,6 +155,8 @@ const operations: TestRunnerFn = (fork, : ssz.bellatrix.ExecutionPayload, // Capella address_change: ssz.capella.SignedBLSToExecutionChange, + consolidation: ssz.electra.SignedConsolidation, + execution_layer_withdrawal_request: ssz.electra.ExecutionLayerWithdrawalRequest, }, shouldError: (testCase) => testCase.post === undefined, getExpected: (testCase) => testCase.post, diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 3ddc3668e487..a2bd691337ad 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -23,6 +23,7 @@ export { processExecutionLayerWithdrawalRequest, processBlsToExecutionChange, processDepositRequest, + processConsolidation, }; export function processOperations( From 81080166624cb68314c60d8e32f1e4e3185ffe07 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 14 May 2024 10:37:56 +0300 Subject: [PATCH 045/145] test: fix ssz types in fork_choice spec tests (#6767) --- packages/beacon-node/test/spec/presets/fork_choice.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index 01adcc77500b..9f8d5c727879 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -343,8 +343,8 @@ const forkChoiceTest = [BLOCK_FILE_NAME]: ssz[fork].SignedBeaconBlock, [BLOBS_FILE_NAME]: ssz.deneb.Blobs, [POW_BLOCK_FILE_NAME]: ssz.bellatrix.PowBlock, - [ATTESTATION_FILE_NAME]: ssz.phase0.Attestation, - [ATTESTER_SLASHING_FILE_NAME]: ssz.phase0.AttesterSlashing, + [ATTESTATION_FILE_NAME]: ssz.allForks[fork].Attestation, + [ATTESTER_SLASHING_FILE_NAME]: ssz.allForks[fork].AttesterSlashing, }, mapToTestCase: (t: Record) => { // t has input file name as key From 52be86eeaef54165ba38ad0c6c46bd04dd0f4fb2 Mon Sep 17 00:00:00 2001 From: Julien Date: Tue, 14 May 2024 00:50:29 -0700 Subject: [PATCH 046/145] chore: update EffectiveBalanceIncrements type (#6763) * chore: update EffectiveBalanceIncrements type * chore: remove now irrelevant tests --- .../src/cache/effectiveBalanceIncrements.ts | 13 ++-- .../state-transition/src/cache/epochCache.ts | 8 +-- packages/state-transition/src/util/balance.ts | 4 +- .../test/memory/effectiveBalanceIncrements.ts | 62 ------------------- .../perf/dataStructures/arrayish.memory.ts | 54 ---------------- .../test/perf/dataStructures/arrayish.test.ts | 43 ------------- .../effectiveBalanceIncrements.test.ts | 32 ---------- 7 files changed, 12 insertions(+), 204 deletions(-) delete mode 100644 packages/state-transition/test/memory/effectiveBalanceIncrements.ts delete mode 100644 packages/state-transition/test/perf/dataStructures/effectiveBalanceIncrements.test.ts diff --git a/packages/state-transition/src/cache/effectiveBalanceIncrements.ts b/packages/state-transition/src/cache/effectiveBalanceIncrements.ts index a82eb0300432..bd72b333a03c 100644 --- a/packages/state-transition/src/cache/effectiveBalanceIncrements.ts +++ b/packages/state-transition/src/cache/effectiveBalanceIncrements.ts @@ -3,18 +3,17 @@ import {BeaconStateAllForks} from "../types.js"; /** * Alias to allow easier refactoring. - * TODO: Estimate the risk of future proof of MAX_EFFECTIVE_BALANCE_INCREMENT < 255 */ -export type EffectiveBalanceIncrements = Uint8Array; +export type EffectiveBalanceIncrements = Uint16Array; -/** Helper to prevent re-writting tests downstream if we change Uint8Array to number[] */ +/** Helper to prevent re-writting tests downstream if we change Uint16Array to number[] */ export function getEffectiveBalanceIncrementsZeroed(len: number): EffectiveBalanceIncrements { - return new Uint8Array(len); + return new Uint16Array(len); } /** * effectiveBalanceIncrements length will always be equal or greater than validatorCount. The - * getEffectiveBalanceIncrementsByteLen() modulo is used to reduce the frequency at which its Uint8Array is recreated. + * getEffectiveBalanceIncrementsByteLen() modulo is used to reduce the frequency at which its Uint16Array is recreated. * if effectiveBalanceIncrements has length greater than validatorCount it's not a problem since those values would * never be accessed. */ @@ -22,7 +21,7 @@ export function getEffectiveBalanceIncrementsWithLen(validatorCount: number): Ef // TODO: Research what's the best number to minimize both memory cost and copy costs const byteLen = 1024 * Math.ceil(validatorCount / 1024); - return new Uint8Array(byteLen); + return new Uint16Array(byteLen); } /** @@ -32,7 +31,7 @@ export function getEffectiveBalanceIncrementsWithLen(validatorCount: number): Ef */ export function getEffectiveBalanceIncrements(state: BeaconStateAllForks): EffectiveBalanceIncrements { const validatorsArr = state.validators.getAllReadonlyValues(); - const effectiveBalanceIncrements = new Uint8Array(validatorsArr.length); + const effectiveBalanceIncrements = new Uint16Array(validatorsArr.length); for (let i = 0; i < validatorsArr.length; i++) { effectiveBalanceIncrements[i] = Math.floor(validatorsArr[i].effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 71c3b3d15f1a..dbcacec0062d 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -653,8 +653,8 @@ export class EpochCache { beforeEpochTransition(): void { // Clone (copy) before being mutated in processEffectiveBalanceUpdates - // NOTE: Force to use Uint8Array.slice (copy) instead of Buffer.call (not copy) - this.effectiveBalanceIncrements = Uint8Array.prototype.slice.call(this.effectiveBalanceIncrements, 0); + // NOTE: Force to use Uint16Array.slice (copy) instead of Buffer.call (not copy) + this.effectiveBalanceIncrements = Uint16Array.prototype.slice.call(this.effectiveBalanceIncrements, 0); } /** @@ -1048,13 +1048,13 @@ export class EpochCache { const newLength = index >= this.effectiveBalanceIncrements.length ? index + 1 : this.effectiveBalanceIncrements.length; const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - this.effectiveBalanceIncrements = new Uint8Array(newLength); + this.effectiveBalanceIncrements = new Uint16Array(newLength); this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); } else { if (index >= this.effectiveBalanceIncrements.length) { // Clone and extend effectiveBalanceIncrements const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements = new Uint16Array(getEffectiveBalanceIncrementsByteLen(index + 1)); this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); } } diff --git a/packages/state-transition/src/util/balance.ts b/packages/state-transition/src/util/balance.ts index e305c745ab72..e9b7a06e4130 100644 --- a/packages/state-transition/src/util/balance.ts +++ b/packages/state-transition/src/util/balance.ts @@ -56,8 +56,8 @@ export function getEffectiveBalanceIncrementsZeroInactive( const validatorCount = justifiedState.validators.length; const {effectiveBalanceIncrements} = justifiedState.epochCtx; // Slice up to `validatorCount` since it won't be mutated, nor accessed beyond `validatorCount` - // NOTE: Force to use Uint8Array.slice (copy) instead of Buffer.call (not copy) - const effectiveBalanceIncrementsZeroInactive = Uint8Array.prototype.slice.call( + // NOTE: Force to use Uint16Array.slice (copy) instead of Buffer.call (not copy) + const effectiveBalanceIncrementsZeroInactive = Uint16Array.prototype.slice.call( effectiveBalanceIncrements, 0, validatorCount diff --git a/packages/state-transition/test/memory/effectiveBalanceIncrements.ts b/packages/state-transition/test/memory/effectiveBalanceIncrements.ts deleted file mode 100644 index f1c603b85657..000000000000 --- a/packages/state-transition/test/memory/effectiveBalanceIncrements.ts +++ /dev/null @@ -1,62 +0,0 @@ -import {MutableVector} from "@chainsafe/persistent-ts"; -import {testRunnerMemory} from "@lodestar/beacon-node/test/memory/testRunnerMemory"; -import {newZeroedArray} from "../../src/index.js"; - -// Results in Linux Feb 2022 -// -// EffectiveBalanceIncrements Uint8Array 300000 - 299873.5 bytes / instance -// EffectiveBalanceIncrements array 300000 - 2400093.1 bytes / instance -// EffectiveBalanceIncrements MutableVector 300000 - 4380557.0 bytes / instance -// EffectiveBalanceIncrements MutableVector 300000 cloned 10 - 4399575.0 bytes / instance -// -// With MutableVector, break even at 14 instances of Uint8Array -// 4380557 / 299873 = 14 - -const vc = 300_000; -const cloneTimes = 10; - -testRunnerMemoryBpi([ - { - id: `EffectiveBalanceIncrements Uint8Array ${vc}`, - getInstance: () => new Uint8Array(vc), - }, - { - id: `EffectiveBalanceIncrements array ${vc}`, - getInstance: () => newZeroedArray(vc), - }, - { - id: `EffectiveBalanceIncrements MutableVector ${vc}`, - getInstance: () => MutableVector.from(newZeroedArray(vc)), - }, - { - id: `EffectiveBalanceIncrements MutableVector ${vc} cloned ${cloneTimes}`, - getInstance: () => { - const mv = MutableVector.from(newZeroedArray(vc)); - const mvs = [mv]; - for (let i = 0; i < cloneTimes; i++) { - const mvc = mv.clone(); - mvc.push(0); - mvs.push(mvc); - } - return mvs; - }, - }, -]); - -/** - * Test bytes per instance in different representations of raw binary data - */ -function testRunnerMemoryBpi(testCases: {getInstance: (bytes: number) => unknown; id: string}[]): void { - const longestId = Math.max(...testCases.map(({id}) => id.length)); - - for (const {id, getInstance} of testCases) { - const bpi = testRunnerMemory({ - getInstance, - convergeFactor: 1 / 100, - sampleEvery: 5, - }); - - // eslint-disable-next-line no-console - console.log(`${id.padEnd(longestId)} - ${bpi.toFixed(1)} bytes / instance`); - } -} diff --git a/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts b/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts index 1f4141912627..7e10f447181f 100644 --- a/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts +++ b/packages/state-transition/test/perf/dataStructures/arrayish.memory.ts @@ -1,5 +1,3 @@ -import {MutableVector} from "@chainsafe/persistent-ts"; - const refs: any[] = []; const xs: number[] = []; const arrayBuffersArr: number[] = []; @@ -23,7 +21,6 @@ const size = 100; const testType = TestType.Set; let arrayNumGlobal: number[] | null = null; -let mutableVectorGlobal: MutableVector | null = null; for (let i = 0; i < 1e8; i++) { switch (testType as TestType) { @@ -65,49 +62,6 @@ for (let i = 0; i < 1e8; i++) { break; } - // size | 100 | 1000 | 10000 | - // ---- | ------ | ------ | ------ | - // rssM | 1817.4 | 15518. | 154335 | - case TestType.MutableVector: { - const items = createArray(size); - const mutableVector = MutableVector.from(items); - refs.push(mutableVector); - break; - } - - // size | 100 | 1000 | - // ---- | ------ | ------ | - // rssM | 58.68 | 55.89 | - case TestType.MutableVectorClone: { - if (!mutableVectorGlobal) { - const items = createArray(size); - mutableVectorGlobal = MutableVector.from(items); - } - refs.push(mutableVectorGlobal.clone()); - break; - } - - // Grid of size / changes, all values = rssM in bytes - // | 100 | 1000 | 10000 | - // ----- | ------ | ------ | ------ | - // 1 | 793.45 | 801.53 | 1137.9 | - // 10 | 803.98 | 802.36 | 1144.9 | - // 100 | 1573.2 | 1826.4 | 2172.0 | - // 1000 | - | 11250. | 11886. | - // 10000 | - | - | 111365 | - case TestType.MutableVectorCloneAndMutate: { - if (!mutableVectorGlobal) { - const items = createArray(size); - mutableVectorGlobal = MutableVector.from(items); - } - const newArr = mutableVectorGlobal.clone(); - for (let j = 0; j < 10000; j++) { - newArr.set(j, i); - } - refs.push(newArr); - break; - } - // size | 100 | 1000 | // ---- | ------ | ------ | // rssM | 2646.8 | 20855. | @@ -161,14 +115,6 @@ for (let i = 0; i < 1e8; i++) { } } -function createArray(n: number): number[] { - const items: number[] = []; - for (let i = 0; i < n; i++) { - items.push(i); - } - return items; -} - /** * From https://github.com/simple-statistics/simple-statistics/blob/d0d177baf74976a2421638bce98ab028c5afb537/src/linear_regression.js * diff --git a/packages/state-transition/test/perf/dataStructures/arrayish.test.ts b/packages/state-transition/test/perf/dataStructures/arrayish.test.ts index 59162b6eecca..5b6af0d989b6 100644 --- a/packages/state-transition/test/perf/dataStructures/arrayish.test.ts +++ b/packages/state-transition/test/perf/dataStructures/arrayish.test.ts @@ -1,6 +1,5 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {LeafNode, toGindex, Tree, zeroNode} from "@chainsafe/persistent-merkle-tree"; -import {MutableVector} from "@chainsafe/persistent-ts"; // Understand the cost of each array-ish data structure to: // - Get one element @@ -99,48 +98,6 @@ describe("Tree (persistent-merkle-tree)", () => { } }); -describe("MutableVector", () => { - // Don't track regressions in CI - setBenchOpts({noThreshold: true}); - - let items: number[]; - let mutableVector: MutableVector; - - before(function () { - items = createArray(n); - mutableVector = MutableVector.from(items); - }); - - itBench(`MutableVector ${n} create`, () => { - MutableVector.from(items); - }); - - itBench({id: `MutableVector ${n} get(${ih})`, runsFactor}, () => { - for (let i = 0; i < runsFactor; i++) mutableVector.get(ih - i); - }); - - itBench({id: `MutableVector ${n} set(${ih})`, runsFactor}, () => { - for (let i = 0; i < runsFactor; i++) mutableVector.set(ih - i, 10000000); - }); - - itBench(`MutableVector ${n} toArray()`, () => { - mutableVector.toArray(); - }); - - itBench(`MutableVector ${n} iterate all - toArray() + loop`, () => { - const mvArr = mutableVector.toArray(); - for (let i = 0; i < n; i++) { - mvArr[i]; - } - }); - - itBench(`MutableVector ${n} iterate all - get(i)`, () => { - for (let i = 0; i < n; i++) { - mutableVector.get(i); - } - }); -}); - describe("Array", () => { // Don't track regressions in CI setBenchOpts({noThreshold: true}); diff --git a/packages/state-transition/test/perf/dataStructures/effectiveBalanceIncrements.test.ts b/packages/state-transition/test/perf/dataStructures/effectiveBalanceIncrements.test.ts deleted file mode 100644 index 13c2d982e86b..000000000000 --- a/packages/state-transition/test/perf/dataStructures/effectiveBalanceIncrements.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import {itBench, setBenchOpts} from "@dapplion/benchmark"; -import {MutableVector} from "@chainsafe/persistent-ts"; -import {newZeroedArray} from "../../../src/index.js"; - -describe("effectiveBalanceIncrements", () => { - setBenchOpts({noThreshold: true}); - - const vc = 300_000; - const uint8Array = new Uint8Array(vc); - const mv = MutableVector.from(newZeroedArray(vc)); - - itBench(`effectiveBalanceIncrements clone Uint8Array ${vc}`, () => { - uint8Array.slice(0); - }); - - itBench(`effectiveBalanceIncrements clone MutableVector ${vc}`, () => { - mv.clone(); - }); - - itBench(`effectiveBalanceIncrements rw all Uint8Array ${vc}`, () => { - for (let i = 0; i < vc; i++) { - uint8Array[i]++; - } - }); - - itBench(`effectiveBalanceIncrements rw all MutableVector ${vc}`, () => { - for (let i = 0; i < vc; i++) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - mv.set(i, mv.get(i)! + 1); - } - }); -}); From 9bd982cbfad3744c1c040c5d154a8898df52bece Mon Sep 17 00:00:00 2001 From: Navie Chan Date: Tue, 14 May 2024 11:37:56 +0300 Subject: [PATCH 047/145] Fix ssz_static --- packages/beacon-node/src/chain/chain.ts | 2 +- .../src/eth1/eth1DepositDataTracker.ts | 4 +-- .../beacon-node/src/execution/engine/types.ts | 6 ++-- .../test/sim/electra-interop.test.ts | 4 +-- .../test/spec/presets/ssz_static.test.ts | 1 + packages/beacon-node/test/utils/state.ts | 2 +- packages/light-client/src/spec/utils.ts | 6 ++-- packages/params/src/index.ts | 4 +++ .../src/block/processDepositRequest.ts | 4 +-- .../processExecutionLayerWithdrawalRequest.ts | 2 +- .../src/block/processOperations.ts | 2 +- .../src/slot/upgradeStateToElectra.ts | 14 +++++----- packages/state-transition/src/util/deposit.ts | 4 +-- .../state-transition/src/util/execution.ts | 4 +-- packages/state-transition/src/util/genesis.ts | 2 +- .../test/unit/util/deposit.test.ts | 4 +-- packages/types/src/electra/sszTypes.ts | 28 ++++++++++--------- packages/types/src/electra/types.ts | 6 ++-- 18 files changed, 53 insertions(+), 46 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 9f30936305f0..9f67857fe530 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1147,7 +1147,7 @@ export class BeaconChain implements IBeaconChain { // Will resolve this later // if (cpEpoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity)) { // // finalizedState can be safely casted to Electra state since cp is already post-Electra - // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositRequestsStartIndex) { + // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositReceiptsStartIndex) { // // Signal eth1 to stop polling eth1Data // this.eth1.stopPollingEth1Data(); // } diff --git a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts index d0578718f29f..c0b3ab35a73a 100644 --- a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts +++ b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts @@ -129,9 +129,9 @@ export class Eth1DepositDataTracker { async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { if ( state.epochCtx.isAfterElectra() && - state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositRequestsStartIndex + state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositReceiptsStartIndex ) { - // No need to poll eth1Data since Electra deprecates the mechanism after depositRequestsStartIndex is reached + // No need to poll eth1Data since Electra deprecates the mechanism after depositReceiptsStartIndex is reached return {eth1Data: state.eth1Data, deposits: []}; } const eth1Data = this.forcedEth1DataVote ?? (await this.getEth1Data(state)); diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 3c1da820a610..de744d56d6cd 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -235,8 +235,8 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload // ELECTRA adds depositRequests/depositRequests to the ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositRequests, withdrawalRequests} = data as electra.ExecutionPayload; - payload.depositRequests = depositRequests.map(serializeDepositRequest); + const {depositReceipts, withdrawalRequests} = data as electra.ExecutionPayload; + payload.depositRequests = depositReceipts.map(serializeDepositRequest); payload.withdrawalRequests = withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest); } @@ -334,7 +334,7 @@ export function parseExecutionPayload( `depositRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` ); } - (executionPayload as electra.ExecutionPayload).depositRequests = depositRequests.map(deserializeDepositRequest); + (executionPayload as electra.ExecutionPayload).depositReceipts = depositRequests.map(deserializeDepositRequest); if (withdrawalRequests == null) { throw Error( diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 2dfa5ee0f77c..37bed1f7b843 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -431,8 +431,8 @@ describe("executionEngine / ExecutionEngineHttp", function () { throw Error("Historical validator length for epoch 1 or 2 is not dropped properly"); } - if (headState.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { - throw Error("state.depositRequestsStartIndex is not set upon processing new deposit receipt"); + if (headState.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + throw Error("state.depositReceiptsStartIndex is not set upon processing new deposit receipt"); } // wait for 1 slot to print current epoch stats diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index d81b47ecf587..cca6bb57c308 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -45,6 +45,7 @@ const sszStatic = /* eslint-disable @typescript-eslint/strict-boolean-expressions */ const sszType = (sszTypesFor(fork) as Types)[typeName] || + (ssz.electra as Types)[typeName] || (ssz.deneb as Types)[typeName] || (ssz.capella as Types)[typeName] || (ssz.bellatrix as Types)[typeName] || diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index bb55adca1eb0..ad4dd4b88dac 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -97,7 +97,7 @@ export function generateState( if (forkSeq >= ForkSeq.electra) { const stateElectra = state as electra.BeaconState; - stateElectra.depositRequestsStartIndex = 2023n; + stateElectra.depositReceiptsStartIndex = 2023n; stateElectra.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue(); } diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 679371a54d6c..b1145860833f 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -115,8 +115,8 @@ export function upgradeLightClientHeader( // eslint-disable-next-line no-fallthrough case ForkName.electra: - (upgradedHeader as LightClientHeader).execution.depositRequestsRoot = - ssz.electra.LightClientHeader.fields.execution.fields.depositRequestsRoot.defaultValue(); + (upgradedHeader as LightClientHeader).execution.depositReceiptsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.depositReceiptsRoot.defaultValue(); (upgradedHeader as LightClientHeader).execution.withdrawalRequestsRoot = ssz.electra.LightClientHeader.fields.execution.fields.withdrawalRequestsRoot.defaultValue(); @@ -157,7 +157,7 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC if (epoch < config.ELECTRA_FORK_EPOCH) { if ( - (header as LightClientHeader).execution.depositRequestsRoot !== undefined || + (header as LightClientHeader).execution.depositReceiptsRoot !== undefined || (header as LightClientHeader).execution.withdrawalRequestsRoot !== undefined ) { return false; diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index c1d3f1bc1981..20439568156e 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -264,3 +264,7 @@ export const BLOBSIDECAR_FIXED_SIZE = ACTIVE_PRESET === PresetName.minimal ? 131 // Electra Misc export const UNSET_DEPOSIT_RECEIPTS_START_INDEX = 2n ** 64n - 1n; export const FULL_EXIT_REQUEST_AMOUNT = 0; +export const NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA = 87; +export const NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA = 6; +export const FINALIZED_ROOT_DEPTH_ELECTRA = 7; +export const FINALIZED_ROOT_INDEX_ELECTRA = 169; \ No newline at end of file diff --git a/packages/state-transition/src/block/processDepositRequest.ts b/packages/state-transition/src/block/processDepositRequest.ts index ca6fe4206188..d5525f3cd544 100644 --- a/packages/state-transition/src/block/processDepositRequest.ts +++ b/packages/state-transition/src/block/processDepositRequest.ts @@ -9,8 +9,8 @@ export function processDepositRequest( state: CachedBeaconStateElectra, depositRequest: electra.DepositRequest ): void { - if (state.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { - state.depositRequestsStartIndex = BigInt(depositRequest.index); + if (state.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + state.depositReceiptsStartIndex = BigInt(depositRequest.index); } applyDeposit(fork, state, depositRequest); diff --git a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts index e16fad105aaa..1b795789cb77 100644 --- a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts @@ -70,7 +70,7 @@ export function processExecutionLayerWithdrawalRequest( const exitQueueEpoch = computeExitEpochAndUpdateChurn(state, amountToWithdraw); const withdrawableEpoch = exitQueueEpoch + config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; - const pendingPartialWithdrawal = ssz.electra.PartialWithdrawal.toViewDU({ + const pendingPartialWithdrawal = ssz.electra.PendingPartialWithdrawal.toViewDU({ index: validatorIndex, amount: amountToWithdraw, withdrawableEpoch, diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index a2bd691337ad..86672a6572e9 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -71,7 +71,7 @@ export function processOperations( processExecutionLayerWithdrawalRequest(fork, state as CachedBeaconStateElectra, elWithdrawalRequest); } - for (const depositRequest of bodyElectra.executionPayload.depositRequests) { + for (const depositRequest of bodyElectra.executionPayload.depositReceipts) { processDepositRequest(fork, stateElectra, depositRequest); } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 121425265b50..417c3cd717c1 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -48,16 +48,16 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache stateElectraView.nextSyncCommittee = stateElectraCloned.nextSyncCommittee; stateElectraView.latestExecutionPayloadHeader = ssz.electra.BeaconState.fields.latestExecutionPayloadHeader.toViewDU({ ...stateElectraCloned.latestExecutionPayloadHeader.toValue(), - depositRequestsRoot: ssz.Root.defaultValue(), + depositReceiptsRoot: ssz.Root.defaultValue(), withdrawalRequestsRoot: ssz.Root.defaultValue(), }); stateElectraView.nextWithdrawalIndex = stateDeneb.nextWithdrawalIndex; stateElectraView.nextWithdrawalValidatorIndex = stateDeneb.nextWithdrawalValidatorIndex; stateElectraView.historicalSummaries = stateElectraCloned.historicalSummaries; - // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default - // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX - stateElectraView.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + // latestExecutionPayloadHeader's depositReceiptsRoot and withdrawalRequestsRoot set to zeros by default + // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectraView.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; const validatorsArr = stateElectraView.validators.getAllReadonly(); @@ -99,9 +99,9 @@ export function upgradeStateToElectraOriginal(stateDeneb: CachedBeaconStateDeneb epoch: stateDeneb.epochCtx.epoch, }); - // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default - // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX - stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + // latestExecutionPayloadHeader's depositReceiptsRoot and withdrawalRequestsRoot set to zeros by default + // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; const validatorsArr = stateElectra.validators.getAllReadonly(); diff --git a/packages/state-transition/src/util/deposit.ts b/packages/state-transition/src/util/deposit.ts index e8ef93c515d2..493099fdd982 100644 --- a/packages/state-transition/src/util/deposit.ts +++ b/packages/state-transition/src/util/deposit.ts @@ -9,9 +9,9 @@ export function getEth1DepositCount(state: CachedBeaconStateAllForks, eth1Data?: // eth1DataIndexLimit = min(UintNum64, UintBn64) can be safely casted as UintNum64 // since the result lies within upper and lower bound of UintNum64 const eth1DataIndexLimit: UintNum64 = - eth1DataToUse.depositCount < electraState.depositRequestsStartIndex + eth1DataToUse.depositCount < electraState.depositReceiptsStartIndex ? eth1DataToUse.depositCount - : Number(electraState.depositRequestsStartIndex); + : Number(electraState.depositReceiptsStartIndex); if (state.eth1DepositIndex < eth1DataIndexLimit) { return Math.min(MAX_DEPOSITS, eth1DataIndexLimit - state.eth1DepositIndex); diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 2c55dc61a502..c7f0ec2f395c 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -172,8 +172,8 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio } if (fork >= ForkSeq.electra) { - (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositRequestsRoot = - ssz.electra.DepositRequests.hashTreeRoot((payload as electra.ExecutionPayload).depositRequests); + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositReceiptsRoot = + ssz.electra.DepositReceipts.hashTreeRoot((payload as electra.ExecutionPayload).depositReceipts); (bellatrixPayloadFields as electra.ExecutionPayloadHeader).withdrawalRequestsRoot = ssz.electra.ExecutionLayerWithdrawalRequests.hashTreeRoot( (payload as electra.ExecutionPayload).withdrawalRequests diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index ca894fd5b4e5..e1003dca1946 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -308,7 +308,7 @@ export function initializeBeaconStateFromEth1( stateElectra.latestExecutionPayloadHeader = (executionPayloadHeader as CompositeViewDU) ?? ssz.electra.ExecutionPayloadHeader.defaultViewDU(); - stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; } state.commit(); diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts index 677a15724ece..e8f7f7a86af8 100644 --- a/packages/state-transition/test/unit/util/deposit.test.ts +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -43,7 +43,7 @@ describe("getEth1DepositCount", () => { throw Error("Not a post-Electra state"); } - postElectraState.depositRequestsStartIndex = 1000n; + postElectraState.depositReceiptsStartIndex = 1000n; postElectraState.eth1Data.depositCount = 995; // 1. Should get less than MAX_DEPOSIT @@ -77,7 +77,7 @@ describe("getEth1DepositCount", () => { throw Error("Not a post-Electra state"); } - postElectraState.depositRequestsStartIndex = 1000n; + postElectraState.depositReceiptsStartIndex = 1000n; postElectraState.eth1Data.depositCount = 1005; // Before eth1DepositIndex reaching the start index diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index d3d9843e63f1..b2191215eb2c 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -21,6 +21,8 @@ import { PENDING_BALANCE_DEPOSITS_LIMIT, PENDING_PARTIAL_WITHDRAWALS_LIMIT, PENDING_CONSOLIDATIONS_LIMIT, + FINALIZED_ROOT_DEPTH_ELECTRA, + NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA, } from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; @@ -110,7 +112,7 @@ export const SignedAggregateAndProof = new ContainerType( {typeName: "SignedAggregateAndProof", jsonCase: "eth2"} ); -export const DepositRequest = new ContainerType( +export const DepositReceipt = new ContainerType( { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, @@ -118,10 +120,10 @@ export const DepositRequest = new ContainerType( signature: BLSSignature, index: DepositIndex, }, - {typeName: "DepositRequest", jsonCase: "eth2"} + {typeName: "DepositReceipt", jsonCase: "eth2"} ); -export const DepositRequests = new ListCompositeType(DepositRequest, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); +export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); export const ExecutionLayerWithdrawalRequest = new ContainerType( { @@ -139,7 +141,7 @@ export const ExecutionLayerWithdrawalRequests = new ListCompositeType( export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, - depositRequests: DepositRequests, // New in ELECTRA + depositReceipts: DepositReceipts, // New in ELECTRA withdrawalRequests: ExecutionLayerWithdrawalRequests, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} @@ -148,7 +150,7 @@ export const ExecutionPayload = new ContainerType( export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, - depositRequestsRoot: Root, // New in ELECTRA + depositReceiptsRoot: Root, // New in ELECTRA withdrawalRequestsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} @@ -280,13 +282,13 @@ export const PendingBalanceDeposits = new ListCompositeType( Number(PENDING_BALANCE_DEPOSITS_LIMIT) ); -export const PartialWithdrawal = new ContainerType( +export const PendingPartialWithdrawal = new ContainerType( { index: ValidatorIndex, amount: Gwei, withdrawableEpoch: Epoch, }, - {typeName: "PartialWithdrawal", jsonCase: "eth2"} + {typeName: "PendingPartialWithdrawal", jsonCase: "eth2"} ); export const PendingConsolidation = new ContainerType( @@ -340,14 +342,14 @@ export const BeaconState = new ContainerType( nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, // Deep history valid from Capella onwards historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, - depositRequestsStartIndex: UintBn64, // New in ELECTRA:EIP6110 + depositReceiptsStartIndex: UintBn64, // New in ELECTRA:EIP6110 depositBalanceToConsume: Gwei, // New in Electra:EIP7251 exitBalanceToConsume: Gwei, // New in Electra:EIP7251 earliestExitEpoch: Epoch, // New in Electra:EIP7251 consolidationBalanceToConsume: Gwei, // New in Electra:EIP7251 earliestConsolidationEpoch: Epoch, // New in Electra:EIP7251 pendingBalanceDeposits: PendingBalanceDeposits, // new in electra:eip7251 - pendingPartialWithdrawals: new ListCompositeType(PartialWithdrawal, Number(PENDING_PARTIAL_WITHDRAWALS_LIMIT)), // New in Electra:EIP7251 + pendingPartialWithdrawals: new ListCompositeType(PendingPartialWithdrawal, Number(PENDING_PARTIAL_WITHDRAWALS_LIMIT)), // New in Electra:EIP7251 pendingConsolidations: new ListCompositeType(PendingConsolidation, Number(PENDING_CONSOLIDATIONS_LIMIT)), // new in electra:eip7251 }, {typeName: "BeaconState", jsonCase: "eth2"} @@ -366,7 +368,7 @@ export const LightClientBootstrap = new ContainerType( { header: LightClientHeader, currentSyncCommittee: altairSsz.SyncCommittee, - currentSyncCommitteeBranch: altairSsz.LightClientBootstrap.fields.currentSyncCommitteeBranch, + currentSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), }, {typeName: "LightClientBootstrap", jsonCase: "eth2"} ); @@ -375,9 +377,9 @@ export const LightClientUpdate = new ContainerType( { attestedHeader: LightClientHeader, nextSyncCommittee: altairSsz.SyncCommittee, - nextSyncCommitteeBranch: altairSsz.LightClientUpdate.fields.nextSyncCommitteeBranch, + nextSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), finalizedHeader: LightClientHeader, - finalityBranch: altairSsz.LightClientUpdate.fields.finalityBranch, + finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, }, @@ -388,7 +390,7 @@ export const LightClientFinalityUpdate = new ContainerType( { attestedHeader: LightClientHeader, finalizedHeader: LightClientHeader, - finalityBranch: altairSsz.LightClientFinalityUpdate.fields.finalityBranch, + finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, }, diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 53b95e95525c..fc95ecde562d 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -9,8 +9,8 @@ export type AttesterSlashing = ValueOf; export type AggregateAndProof = ValueOf; export type SignedAggregateAndProof = ValueOf; -export type DepositRequest = ValueOf; -export type DepositRequests = ValueOf; +export type DepositRequest = ValueOf; +export type DepositRequests = ValueOf; export type ExecutionLayerWithdrawalRequest = ValueOf; export type ExecutionLayerWithdrawalRequests = ValueOf; @@ -47,5 +47,5 @@ export type Consolidation = ValueOf; export type SignedConsolidation = ValueOf; export type PendingBalanceDeposit = ValueOf; -export type PartialWithdrawal = ValueOf; +export type PendingPartialWithdrawal = ValueOf; export type PendingConsolidation = ValueOf; From dc2a197827bec78fd6110e7be49fbebcb7a3345f Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Tue, 14 May 2024 04:42:06 -0400 Subject: [PATCH 048/145] fix: inline sourcemaps to help with debugging (#6768) --- vite.base.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/vite.base.config.ts b/vite.base.config.ts index 65e1bad01500..f9030d9edb7f 100644 --- a/vite.base.config.ts +++ b/vite.base.config.ts @@ -40,6 +40,7 @@ export function getBaseViteConfig( esbuild: { banner, legalComments: "none", + sourcemap: "inline", }, build: { // "modules" refer to ['es2020', 'edge88', 'firefox78', 'chrome87', 'safari14'] From 3e6990ae2044f467930e0436673198e80bb661b6 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Tue, 14 May 2024 11:46:01 +0200 Subject: [PATCH 049/145] fix: additional epoch calculation logic for consolidation churn (#6770) Fix the chunk limit logic --- packages/state-transition/src/util/epoch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/state-transition/src/util/epoch.ts b/packages/state-transition/src/util/epoch.ts index 67837780dca9..b9b911137fd9 100644 --- a/packages/state-transition/src/util/epoch.ts +++ b/packages/state-transition/src/util/epoch.ts @@ -83,7 +83,7 @@ export function computeConsolidationEpochAndUpdateChurn( // Consolidation doesn't fit in the current earliest epoch. if (consolidationBalance > consolidationBalanceToConsume) { const balanceToProcess = Number(consolidationBalance) - consolidationBalanceToConsume; - const additionalEpochs = Math.floor((balanceToProcess - 1) / (perEpochConsolidationChurn + 1)); + const additionalEpochs = Math.floor((balanceToProcess - 1) / perEpochConsolidationChurn) + 1; earliestConsolidationEpoch += additionalEpochs; consolidationBalanceToConsume += additionalEpochs * perEpochConsolidationChurn; } From bd1bb6cb2f4124ca181ec5706c6505b9903ed9ec Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 14 May 2024 12:56:01 +0300 Subject: [PATCH 050/145] fix: electra fork transition spec tests (#6769) * fix: electra fork transition * fix: merge issue * chore: remove unwanted change --- .../src/slot/upgradeStateToElectra.ts | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 417c3cd717c1..72255ccddee3 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -1,4 +1,4 @@ -import {ssz} from "@lodestar/types"; +import {Epoch, ValidatorIndex, ssz} from "@lodestar/types"; import {FAR_FUTURE_EPOCH, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; import {CachedBeaconStateDeneb} from "../types.js"; import {CachedBeaconStateElectra, getCachedBeaconState} from "../cache/stateCache.js"; @@ -7,6 +7,8 @@ import { queueEntireBalanceAndResetValidator, queueExcessActiveBalance, } from "../util/electra.js"; +import {computeActivationExitEpoch} from "../util/epoch.js"; +import {getActivationExitChurnLimit, getConsolidationChurnLimit} from "../util/validator.js"; /** * Upgrade a state from Capella to Deneb. @@ -58,17 +60,52 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache // latestExecutionPayloadHeader's depositReceiptsRoot and withdrawalRequestsRoot set to zeros by default // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX stateElectraView.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + stateElectraView.depositBalanceToConsume = BigInt(0); + stateElectraView.exitBalanceToConsume = BigInt(0); const validatorsArr = stateElectraView.validators.getAllReadonly(); + const exitEpochs: Epoch[] = []; + + // [EIP-7251]: add validators that are not yet active to pending balance deposits + const preActivation: ValidatorIndex[] = []; + for (let validatorIndex = 0; validatorIndex < validatorsArr.length; validatorIndex++) { + const {activationEpoch, exitEpoch} = validatorsArr[validatorIndex]; + if (activationEpoch === FAR_FUTURE_EPOCH) { + preActivation.push(validatorIndex); + } + if (exitEpoch !== FAR_FUTURE_EPOCH) { + exitEpochs.push(exitEpoch); + } + } + + const currentEpochPre = stateDeneb.epochCtx.epoch; + + if (exitEpochs.length === 0) { + exitEpochs.push(currentEpochPre); + } + stateElectraView.earliestExitEpoch = Math.max(...exitEpochs) + 1; + stateElectraView.consolidationBalanceToConsume = BigInt(0); + stateElectraView.earliestConsolidationEpoch = computeActivationExitEpoch(currentEpochPre); + // stateElectraView.pendingBalanceDeposits = ssz.electra.PendingBalanceDeposits.defaultViewDU(); + // pendingBalanceDeposits, pendingPartialWithdrawals, pendingConsolidations are default values + // TODO-electra: can we improve this? + stateElectraView.commit(); + const tmpElectraState = getCachedBeaconState(stateElectraView, stateDeneb); + stateElectraView.exitBalanceToConsume = BigInt(getActivationExitChurnLimit(tmpElectraState)); + stateElectraView.consolidationBalanceToConsume = BigInt(getConsolidationChurnLimit(tmpElectraState)); + + preActivation.sort((i0, i1) => { + const res = validatorsArr[i0].activationEligibilityEpoch - validatorsArr[i1].activationEligibilityEpoch; + return res !== 0 ? res : i0 - i1; + }); + + for (const validatorIndex of preActivation) { + queueEntireBalanceAndResetValidator(stateElectraView as CachedBeaconStateElectra, validatorIndex); + } for (let i = 0; i < validatorsArr.length; i++) { const validator = validatorsArr[i]; - // [EIP-7251]: add validators that are not yet active to pending balance deposits - if (validator.activationEligibilityEpoch === FAR_FUTURE_EPOCH) { - queueEntireBalanceAndResetValidator(stateElectraView as CachedBeaconStateElectra, i); - } - // [EIP-7251]: Ensure early adopters of compounding credentials go through the activation churn const withdrawalCredential = validator.withdrawalCredentials; if (hasCompoundingWithdrawalCredential(withdrawalCredential)) { From f0eab386df4daad9fbd562d6f8c5cbf021647403 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 14 May 2024 14:50:01 +0300 Subject: [PATCH 051/145] test: fix ssz_static spec tests for all forks (#6771) --- .../beacon-node/test/spec/presets/operations.test.ts | 7 ++++--- .../beacon-node/test/spec/presets/ssz_static.test.ts | 5 +++++ packages/params/src/index.ts | 2 +- packages/types/src/electra/sszTypes.ts | 11 ++++------- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index fe70c091aa5a..de8a32a6056a 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -138,12 +138,12 @@ const operations: TestRunnerFn = (fork, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, - attestation: fork === ForkName.electra ? ssz.electra.Attestation : ssz.phase0.Attestation, - attester_slashing: fork === ForkName.electra ? ssz.electra.AttesterSlashing : ssz.phase0.AttesterSlashing, + attestation: ssz.allForks[fork].Attestation, + attester_slashing: ssz.allForks[fork].AttesterSlashing, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, - deposit_receipt: ssz.electra.DepositRequest, + deposit_receipt: ssz.electra.DepositReceipt, proposer_slashing: ssz.phase0.ProposerSlashing, voluntary_exit: ssz.phase0.SignedVoluntaryExit, // Altair @@ -155,6 +155,7 @@ const operations: TestRunnerFn = (fork, : ssz.bellatrix.ExecutionPayload, // Capella address_change: ssz.capella.SignedBLSToExecutionChange, + // Electra consolidation: ssz.electra.SignedConsolidation, execution_layer_withdrawal_request: ssz.electra.ExecutionLayerWithdrawalRequest, }, diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index cca6bb57c308..ad828d5b5bb9 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -2,8 +2,13 @@ import fs from "node:fs"; import path from "node:path"; import {it, vi} from "vitest"; import {Type} from "@chainsafe/ssz"; +<<<<<<< HEAD import {ssz, sszTypesFor} from "@lodestar/types"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; +======= +import {ssz} from "@lodestar/types"; +import {ACTIVE_PRESET, ForkName, ForkLightClient, ForkExecution} from "@lodestar/params"; +>>>>>>> 82d0692d63d (test: fix ssz_static spec tests for all forks (#6771)) import {replaceUintTypeWithUintBigintType} from "../utils/replaceUintTypeWithUintBigintType.js"; import {parseSszStaticTestcase} from "../utils/sszTestCaseParser.js"; import {runValidSszTest} from "../utils/runValidSszTest.js"; diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 20439568156e..0c2e3b34794b 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -267,4 +267,4 @@ export const FULL_EXIT_REQUEST_AMOUNT = 0; export const NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA = 87; export const NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA = 6; export const FINALIZED_ROOT_DEPTH_ELECTRA = 7; -export const FINALIZED_ROOT_INDEX_ELECTRA = 169; \ No newline at end of file +export const FINALIZED_ROOT_INDEX_ELECTRA = 169; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index b2191215eb2c..0aa2872c3a73 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -277,10 +277,7 @@ export const PendingBalanceDeposit = new ContainerType( {typeName: "PendingBalanceDeposit", jsonCase: "eth2"} ); -export const PendingBalanceDeposits = new ListCompositeType( - PendingBalanceDeposit, - Number(PENDING_BALANCE_DEPOSITS_LIMIT) -); +export const PendingBalanceDeposits = new ListCompositeType(PendingBalanceDeposit, PENDING_BALANCE_DEPOSITS_LIMIT); export const PendingPartialWithdrawal = new ContainerType( { @@ -348,9 +345,9 @@ export const BeaconState = new ContainerType( earliestExitEpoch: Epoch, // New in Electra:EIP7251 consolidationBalanceToConsume: Gwei, // New in Electra:EIP7251 earliestConsolidationEpoch: Epoch, // New in Electra:EIP7251 - pendingBalanceDeposits: PendingBalanceDeposits, // new in electra:eip7251 - pendingPartialWithdrawals: new ListCompositeType(PendingPartialWithdrawal, Number(PENDING_PARTIAL_WITHDRAWALS_LIMIT)), // New in Electra:EIP7251 - pendingConsolidations: new ListCompositeType(PendingConsolidation, Number(PENDING_CONSOLIDATIONS_LIMIT)), // new in electra:eip7251 + pendingBalanceDeposits: PendingBalanceDeposits, // New in Electra:EIP7251 + pendingPartialWithdrawals: new ListCompositeType(PendingPartialWithdrawal, PENDING_PARTIAL_WITHDRAWALS_LIMIT), // New in Electra:EIP7251 + pendingConsolidations: new ListCompositeType(PendingConsolidation, PENDING_CONSOLIDATIONS_LIMIT), // New in Electra:EIP7251 }, {typeName: "BeaconState", jsonCase: "eth2"} ); From 65b6827a8c68ed9401694015b4ca4994fc2b5a5a Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Tue, 14 May 2024 14:11:00 +0200 Subject: [PATCH 052/145] chore(spec): remove the skip specs for electra (#6772) Remove the skip spec for electra --- packages/beacon-node/test/spec/utils/specTestIterator.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 393983bb8a56..63fe0cc10442 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -65,7 +65,6 @@ export const defaultSkipOpts: SkipOpts = { skippedTestSuites: [ /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, - /^electra\/(?!operations\/attestations)(?!operations\/attester_slashing)/, ], skippedTests: [], skippedRunners: ["merkle_proof", "networking"], From 7f2a77a5349177f799bd5b71e9889d8bbc59775a Mon Sep 17 00:00:00 2001 From: NC Date: Tue, 14 May 2024 15:26:36 +0300 Subject: [PATCH 053/145] fix: use mutable validator object (#6774) Use mutable validator object --- .../src/block/processExecutionLayerWithdrawalRequest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts index 1b795789cb77..e02129a09d05 100644 --- a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts @@ -37,7 +37,7 @@ export function processExecutionLayerWithdrawalRequest( return; } - const validator = validators.getReadonly(validatorIndex); + const validator = validators.get(validatorIndex); if (!isValidatorEligibleForWithdrawOrExit(validator, executionLayerWithdrawalRequest.sourceAddress, state)) { return; } From 486a7667ea40d841bc5ce3a5863df3a2a2884dff Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Tue, 14 May 2024 11:42:28 -0400 Subject: [PATCH 054/145] test: fix balance spec tests (#6777) * fix: remove epochCache.balances and invalid MAX_EFFECTIVE_BALANCE check * fix: update rewardsAndPenalties balance updates * docs: add comment to check epochTransitionCache --- .../src/epoch/processEffectiveBalanceUpdates.ts | 7 +++++-- .../src/epoch/processRewardsAndPenalties.ts | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index e83d2545ef50..f4fae6e41d4f 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -41,7 +41,10 @@ export function processEffectiveBalanceUpdates( // epochTransitionCache.balances is set in processRewardsAndPenalties(), so it's recycled here for performance. // It defaults to `state.balances.getAll()` to make Typescript happy and for spec tests - const balances = cache.balances ?? state.balances.getAll(); + const balances = state.balances.getAll(); + + // TODO: (@matthewkeil) This was causing additional failures but should not. Check the EpochTransitionCache for why + // const balances = cache.balances ?? state.balances.getAll(); for (let i = 0, len = balances.length; i < len; i++) { const balance = balances[i]; @@ -55,7 +58,7 @@ export function processEffectiveBalanceUpdates( // Too big effectiveBalance > balance + DOWNWARD_THRESHOLD || // Too small. Check effectiveBalance < MAX_EFFECTIVE_BALANCE to prevent unnecessary updates - (effectiveBalance < MAX_EFFECTIVE_BALANCE && effectiveBalance < balance - UPWARD_THRESHOLD) + effectiveBalance + UPWARD_THRESHOLD < balance ) { // Update the state tree // Should happen rarely, so it's fine to update the tree diff --git a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts index 61680b81002a..041655d75825 100644 --- a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts +++ b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts @@ -28,7 +28,8 @@ export function processRewardsAndPenalties( const balances = state.balances.getAll(); for (let i = 0, len = rewards.length; i < len; i++) { - balances[i] += rewards[i] - penalties[i] - (slashingPenalties[i] ?? 0); + const result = balances[i] + rewards[i] - penalties[i] - (slashingPenalties[i] ?? 0) + balances[i] = Math.max(result, 0); } // important: do not change state one balance at a time. Set them all at once, constructing the tree in one go From f82b355b55cdae81e1066cad2045c0b1b5a02f26 Mon Sep 17 00:00:00 2001 From: NC Date: Tue, 14 May 2024 22:15:17 +0300 Subject: [PATCH 055/145] fix: effective balance cache is not in sync with validator effective balance (#6780) Update eb cache at fork transition --- packages/state-transition/src/util/electra.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/state-transition/src/util/electra.ts b/packages/state-transition/src/util/electra.ts index 00c5bb183eb4..53f53aaf8d61 100644 --- a/packages/state-transition/src/util/electra.ts +++ b/packages/state-transition/src/util/electra.ts @@ -93,6 +93,7 @@ export function queueEntireBalanceAndResetValidator(state: CachedBeaconStateElec const validator = state.validators.get(index); validator.effectiveBalance = 0; + state.epochCtx.effectiveBalanceIncrementsSet(index, 0); validator.activationEligibilityEpoch = FAR_FUTURE_EPOCH; const pendingBalanceDeposit = ssz.electra.PendingBalanceDeposit.toViewDU({ From 1d57ac79ac7d6c6c14968101497791d80d4fb595 Mon Sep 17 00:00:00 2001 From: NC Date: Thu, 16 May 2024 11:05:01 +0300 Subject: [PATCH 056/145] fix: make electra-fork passes lint and check-types (#6785) fix lint and check-types --- packages/beacon-node/test/sim/electra-interop.test.ts | 6 +++--- packages/fork-choice/test/perf/forkChoice/util.ts | 2 +- .../fork-choice/test/unit/forkChoice/forkChoice.test.ts | 6 +++--- .../test/unit/forkChoice/getProposerHead.test.ts | 6 +++--- .../fork-choice/test/unit/protoArray/computeDeltas.test.ts | 2 +- .../src/epoch/processRewardsAndPenalties.ts | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 37bed1f7b843..962e0aa5dea9 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -219,11 +219,11 @@ describe("executionEngine / ExecutionEngineHttp", function () { } } - if (payload.depositRequests.length !== 1) { - throw Error(`Number of depositRequests mismatched. Expected: 1, actual: ${payload.depositRequests.length}`); + if (payload.depositReceipts.length !== 1) { + throw Error(`Number of depositRequests mismatched. Expected: 1, actual: ${payload.depositReceipts.length}`); } - const actualDepositRequest = payload.depositRequests[0]; + const actualDepositRequest = payload.depositReceipts[0]; assert.deepStrictEqual( actualDepositRequest, depositRequestB, diff --git a/packages/fork-choice/test/perf/forkChoice/util.ts b/packages/fork-choice/test/perf/forkChoice/util.ts index dbd049e257c1..6c04ac817fb2 100644 --- a/packages/fork-choice/test/perf/forkChoice/util.ts +++ b/packages/fork-choice/test/perf/forkChoice/util.ts @@ -41,7 +41,7 @@ export function initializeForkChoice(opts: Opts): ForkChoice { genesisSlot ); - const balances = new Uint8Array(Array.from({length: opts.initialValidatorCount}, () => 32)); + const balances = new Uint16Array(Array.from({length: opts.initialValidatorCount}, () => 32)); const fcStore: IForkChoiceStore = { currentSlot: genesisSlot, diff --git a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts index fcb7376cffa8..40988a0e4a71 100644 --- a/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/forkChoice.test.ts @@ -52,16 +52,16 @@ describe("Forkchoice", function () { currentSlot: genesisSlot + 1, justified: { checkpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, - balances: new Uint8Array([32]), + balances: new Uint16Array([32]), totalBalance: 32, }, unrealizedJustified: { checkpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, - balances: new Uint8Array([32]), + balances: new Uint16Array([32]), }, finalizedCheckpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, unrealizedFinalizedCheckpoint: {epoch: genesisEpoch, root: fromHexString(finalizedRoot), rootHex: finalizedRoot}, - justifiedBalancesGetter: () => new Uint8Array([32]), + justifiedBalancesGetter: () => new Uint16Array([32]), equivocatingIndices: new Set(), }; diff --git a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts index cc14b5b57b92..f603a4069b83 100644 --- a/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts +++ b/packages/fork-choice/test/unit/forkChoice/getProposerHead.test.ts @@ -102,12 +102,12 @@ describe("Forkchoice / GetProposerHead", function () { currentSlot: genesisSlot + 1, justified: { checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot}, - balances: new Uint8Array(Array(32).fill(150)), + balances: new Uint16Array(Array(32).fill(150)), totalBalance: 32 * 150, }, unrealizedJustified: { checkpoint: {epoch: genesisEpoch, root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot}, - balances: new Uint8Array(Array(32).fill(150)), + balances: new Uint16Array(Array(32).fill(150)), }, finalizedCheckpoint: { epoch: genesisEpoch, @@ -119,7 +119,7 @@ describe("Forkchoice / GetProposerHead", function () { root: fromHexString(genesisBlock.blockRoot), rootHex: genesisBlock.blockRoot, }, - justifiedBalancesGetter: () => new Uint8Array(Array(32).fill(150)), + justifiedBalancesGetter: () => new Uint16Array(Array(32).fill(150)), equivocatingIndices: new Set(), }; diff --git a/packages/fork-choice/test/unit/protoArray/computeDeltas.test.ts b/packages/fork-choice/test/unit/protoArray/computeDeltas.test.ts index fde551d43cda..4428807bd13d 100644 --- a/packages/fork-choice/test/unit/protoArray/computeDeltas.test.ts +++ b/packages/fork-choice/test/unit/protoArray/computeDeltas.test.ts @@ -253,7 +253,7 @@ describe("computeDeltas", () => { nextEpoch: 0, })); - const balances = new Uint8Array([firstBalance, secondBalance]); + const balances = new Uint16Array([firstBalance, secondBalance]); // 1st validator is part of an attester slashing const equivocatingIndices = new Set([0]); let deltas = computeDeltas(indices.size, votes, balances, balances, equivocatingIndices); diff --git a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts index 041655d75825..6c5d5aa3cb5a 100644 --- a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts +++ b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts @@ -28,7 +28,7 @@ export function processRewardsAndPenalties( const balances = state.balances.getAll(); for (let i = 0, len = rewards.length; i < len; i++) { - const result = balances[i] + rewards[i] - penalties[i] - (slashingPenalties[i] ?? 0) + const result = balances[i] + rewards[i] - penalties[i] - (slashingPenalties[i] ?? 0); balances[i] = Math.max(result, 0); } From edc396ee334acc6d4c0e5b9f624944c085d9b000 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 16 May 2024 11:54:51 +0300 Subject: [PATCH 057/145] fix: update data format of WithdrawalRequestV1 (#6789) --- packages/beacon-node/src/execution/engine/payloadIdCache.ts | 2 +- packages/beacon-node/src/execution/engine/types.ts | 4 ++-- .../src/block/processExecutionLayerWithdrawalRequest.ts | 2 +- packages/types/src/electra/sszTypes.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index f79c28582459..35a50e97cfb3 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -28,7 +28,7 @@ export type DepositRequestV1 = { export type ExecutionLayerWithdrawalRequestV1 = { sourceAddress: DATA; - validatorPubkey: DATA; + validatorPublicKey: DATA; amount: QUANTITY; }; diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index de744d56d6cd..c7b0aefd07ad 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -437,7 +437,7 @@ export function serializeExecutionLayerWithdrawalRequest( ): ExecutionLayerWithdrawalRequestRpc { return { sourceAddress: bytesToData(withdrawalRequest.sourceAddress), - validatorPubkey: bytesToData(withdrawalRequest.validatorPubkey), + validatorPublicKey: bytesToData(withdrawalRequest.validatorPublicKey), amount: numToQuantity(withdrawalRequest.amount), }; } @@ -447,7 +447,7 @@ export function deserializeExecutionLayerWithdrawalRequest( ): electra.ExecutionLayerWithdrawalRequest { return { sourceAddress: dataToBytes(withdrawalRequest.sourceAddress, 20), - validatorPubkey: dataToBytes(withdrawalRequest.validatorPubkey, 48), + validatorPublicKey: dataToBytes(withdrawalRequest.validatorPublicKey, 48), amount: quantityToNum(withdrawalRequest.amount), }; } diff --git a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts index e02129a09d05..935acd522da1 100644 --- a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts @@ -32,7 +32,7 @@ export function processExecutionLayerWithdrawalRequest( // bail out if validator is not in beacon state // note that we don't need to check for 6110 unfinalized vals as they won't be eligible for withdraw/exit anyway - const validatorIndex = pubkey2index.get(executionLayerWithdrawalRequest.validatorPubkey); + const validatorIndex = pubkey2index.get(executionLayerWithdrawalRequest.validatorPublicKey); if (validatorIndex === undefined) { return; } diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 0aa2872c3a73..3596bdbce5bf 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -128,7 +128,7 @@ export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT export const ExecutionLayerWithdrawalRequest = new ContainerType( { sourceAddress: ExecutionAddress, - validatorPubkey: BLSPubkey, + validatorPublicKey: BLSPubkey, amount: UintNum64, }, {typeName: "ExecutionLayerWithdrawalRequest", jsonCase: "eth2"} From 943c91393c475e38b86373ec753d9a1f8aa2397e Mon Sep 17 00:00:00 2001 From: NC Date: Thu, 16 May 2024 12:53:29 +0300 Subject: [PATCH 058/145] fix: publish attestations with non-zero committee index (#6790) Fix publishing att with non-zero comm index --- packages/validator/src/services/attestation.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index a0cf8e8768e7..f4aeeab15260 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -159,12 +159,13 @@ export class AttestationService { this.metrics?.attesterStepCallProduceAggregate.observe(this.clock.secFromSlot(slot + 2 / 3)); const dutiesByCommitteeIndex = groupAttDutiesByCommitteeIndex(dutiesAll); + const isAfterElectra = computeEpochAtSlot(slot) >= this.config.ELECTRA_FORK_EPOCH; // Then download, sign and publish a `SignedAggregateAndProof` for each // validator that is elected to aggregate for this `slot` and `committeeIndex`. await Promise.all( Array.from(dutiesByCommitteeIndex.entries()).map(([index, dutiesSameCommittee]) => { - const attestationData: phase0.AttestationData = {...attestationNoCommittee, index}; + const attestationData: phase0.AttestationData = {...attestationNoCommittee, index: isAfterElectra ? 0 : index}; return this.produceAndPublishAggregates(attestationData, index, dutiesSameCommittee); }) ); @@ -195,10 +196,11 @@ export class AttestationService { const signedAttestations: phase0.Attestation[] = []; const headRootHex = toHexString(attestationNoCommittee.beaconBlockRoot); const currentEpoch = computeEpochAtSlot(slot); + const isAfterElectra = currentEpoch >= this.config.ELECTRA_FORK_EPOCH; await Promise.all( duties.map(async ({duty}) => { - const index = duty.committeeIndex; + const index = isAfterElectra ? 0 : duty.committeeIndex; const attestationData: phase0.AttestationData = {...attestationNoCommittee, index}; const logCtxValidator = {slot, index, head: headRootHex, validatorIndex: duty.validatorIndex}; From 7a50b6ff14d8b6c4d9295ff5bc2dbc8110aad8b3 Mon Sep 17 00:00:00 2001 From: NC Date: Thu, 16 May 2024 15:08:22 +0300 Subject: [PATCH 059/145] fix: validator monitor summaries should not render during epoch 0 (#6791) Skip render summary at epoch 0 --- packages/beacon-node/src/metrics/validatorMonitor.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index a443bf761c93..6331c5dca550 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -644,7 +644,14 @@ export function createValidatorMonitor( } // Compute summaries of previous epoch attestation performance - const prevEpoch = Math.max(0, computeEpochAtSlot(headState.slot) - 1); + const prevEpoch = computeEpochAtSlot(headState.slot) - 1; + + // During the end of first epoch, the prev epoch with be -1 + // Skip this as there is no attestation and block proposal summary in epoch -1 + if (prevEpoch === -1) { + return; + } + const rootCache = new RootHexCache(headState); if (config.getForkSeq(headState.slot) >= ForkSeq.altair) { From d490b41a3cec9eeace7c4e78de22612652e46238 Mon Sep 17 00:00:00 2001 From: NC Date: Thu, 16 May 2024 15:11:37 +0300 Subject: [PATCH 060/145] fix: attestation duty validation (#6792) * fix attestation duty validation * Update packages/validator/src/services/validatorStore.ts Co-authored-by: twoeths * Update packages/validator/src/services/validatorStore.ts --------- Co-authored-by: twoeths Co-authored-by: Cayman --- packages/validator/src/services/validatorStore.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index a807c27569eb..62ebe339bff9 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -798,11 +798,18 @@ export class ValidatorStore { if (duty.slot !== data.slot) { throw Error(`Inconsistent duties during signing: duty.slot ${duty.slot} != att.slot ${data.slot}`); } - if (duty.committeeIndex != data.index) { + + const isAfterElectra = computeEpochAtSlot(duty.slot) >= this.config.ELECTRA_FORK_EPOCH; + if (!isAfterElectra && duty.committeeIndex != data.index) { throw Error( `Inconsistent duties during signing: duty.committeeIndex ${duty.committeeIndex} != att.committeeIndex ${data.index}` ); } + if (isAfterElectra && data.index !== 0) { + throw Error( + `Non-zero committee index post-electra during signing: att.committeeIndex ${data.index}` + ); + } if (this.config.getForkSeq(duty.slot) >= ForkSeq.electra && data.index !== 0) { throw Error(`Attestataion data index must be 0 post electra: index ${data.index}`); } From 296ce19a3b81cf9a0b27c2d8007e16683136301a Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 16 May 2024 08:12:40 -0400 Subject: [PATCH 061/145] fix: align BeaconBlockBody and BlindedBeaconBlockBody (#6782) * fix: align BeaconBlockBody and BlindedBeaconBlockBody * Remove type hacks in test --------- Co-authored-by: Nico Flaig --- packages/types/src/electra/sszTypes.ts | 1 + packages/types/test/unit/blinded.test.ts | 35 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 packages/types/test/unit/blinded.test.ts diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 3596bdbce5bf..0a0c0f6aeefb 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -223,6 +223,7 @@ export const BlindedBeaconBlockBody = new ContainerType( executionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, + consolidations: new ListCompositeType(SignedConsolidation, MAX_CONSOLIDATIONS), // [New in Electra] }, {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); diff --git a/packages/types/test/unit/blinded.test.ts b/packages/types/test/unit/blinded.test.ts new file mode 100644 index 000000000000..77005f1defc2 --- /dev/null +++ b/packages/types/test/unit/blinded.test.ts @@ -0,0 +1,35 @@ +import {describe, it, expect} from "vitest"; +import {ForkName, isForkExecution} from "@lodestar/params"; +import {ssz} from "../../src/index.js"; + +describe("blinded data structures", function () { + it("should have the same number of fields as non-blinded", () => { + const blindedTypes = [ + {a: "BlindedBeaconBlockBody" as const, b: "BeaconBlockBody" as const}, + {a: "ExecutionPayloadHeader" as const, b: "ExecutionPayload" as const}, + ]; + + for (const {a, b} of blindedTypes) { + for (const fork of Object.keys(ssz.allForks) as ForkName[]) { + if (!isForkExecution(fork)) { + continue; + } + + const blindedType = ssz[fork][a]; + if (blindedType === undefined) { + expect.fail(`fork: ${fork}, type ${a} is undefined`); + } + + const type = ssz[fork][b]; + if (type === undefined) { + expect.fail(`fork: ${fork}, type ${b} is undefined`); + } + + expect(Object.keys(blindedType.fields).length).toBeWithMessage( + Object.keys(type.fields).length, + `fork: ${fork}, types ${a} and ${b} have different number of fields` + ); + } + } + }); +}); From 7b76bfc18160347c5fd3bf65632b976b2cd053b7 Mon Sep 17 00:00:00 2001 From: Julien Date: Thu, 16 May 2024 05:17:49 -0700 Subject: [PATCH 062/145] test: improve ssz tests consistency (#6776) * test: improve ssz tests consistency * chore: address comments --- .../test/spec/presets/ssz_static.test.ts | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index ad828d5b5bb9..e4c5a1914827 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import {it, vi} from "vitest"; +import {expect, it, vi} from "vitest"; import {Type} from "@chainsafe/ssz"; <<<<<<< HEAD import {ssz, sszTypesFor} from "@lodestar/types"; @@ -56,23 +56,26 @@ const sszStatic = (ssz.bellatrix as Types)[typeName] || (ssz.altair as Types)[typeName] || (ssz.phase0 as Types)[typeName]; - if (!sszType) { - throw Error(`No type for ${typeName}`); - } - const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); + if (!sszType) { + expect.fail( + `Missing SSZ type definition for ${typeName}; this will prevent associated ssz_static tests to be executed` + ); + } else { + const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); - for (const testCase of fs.readdirSync(testSuiteDirpath)) { - // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts - it(testCase, function () { - // Mainnet must deal with big full states and hash each one multiple times - if (ACTIVE_PRESET === "mainnet") { - vi.setConfig({testTimeout: 30 * 1000}); - } + for (const testCase of fs.readdirSync(testSuiteDirpath)) { + // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts + it(testCase, function () { + // Mainnet must deal with big full states and hash each one multiple times + if (ACTIVE_PRESET === "mainnet") { + vi.setConfig({testTimeout: 30 * 1000}); + } - const testData = parseSszStaticTestcase(path.join(testSuiteDirpath, testCase)); - runValidSszTest(sszTypeNoUint, testData); - }); + const testData = parseSszStaticTestcase(path.join(testSuiteDirpath, testCase)); + runValidSszTest(sszTypeNoUint, testData); + }); + } } }; From dbd6a498a2b2cd960d2945131258ff73d46b5b7d Mon Sep 17 00:00:00 2001 From: twoeths Date: Thu, 16 May 2024 21:09:28 +0300 Subject: [PATCH 063/145] fix: batch validation for electra attestations (#6788) --- .../src/api/impl/beacon/pool/index.ts | 4 +- .../src/chain/opPools/attestationPool.ts | 29 ++-- .../chain/seenCache/seenAttestationData.ts | 24 ++-- .../src/chain/validation/aggregateAndProof.ts | 2 +- .../src/chain/validation/attestation.ts | 134 +++++++++++------- .../src/network/processor/gossipHandlers.ts | 12 +- .../network/processor/gossipQueues/index.ts | 8 +- .../network/processor/gossipQueues/indexed.ts | 1 + packages/beacon-node/src/util/sszBytes.ts | 30 +++- .../perf/chain/validation/attestation.test.ts | 6 +- .../attestation/validateAttestation.test.ts | 26 ++-- .../test/unit/util/sszBytes.test.ts | 6 +- .../validator/src/services/attestation.ts | 4 +- .../validator/src/services/validatorStore.ts | 2 +- 14 files changed, 179 insertions(+), 109 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 3f594fa4c3ae..19c11362c910 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -66,7 +66,7 @@ export function getBeaconPoolApi({ // when a validator is configured with multiple beacon node urls, this attestation data may come from another beacon node // and the block hasn't been in our forkchoice since we haven't seen / processing that block // see https://github.com/ChainSafe/lodestar/issues/5098 - const {indexedAttestation, subnet, attDataRootHex} = await validateGossipFnRetryUnknownRoot( + const {indexedAttestation, subnet, attDataRootHex, committeeIndex} = await validateGossipFnRetryUnknownRoot( validateFn, network, chain, @@ -75,7 +75,7 @@ export function getBeaconPoolApi({ ); if (network.shouldAggregate(subnet, slot)) { - const insertOutcome = chain.attestationPool.add(attestation, attDataRootHex); + const insertOutcome = chain.attestationPool.add(committeeIndex, attestation, attDataRootHex); metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index a9286ac4949a..dfb3bc7348a2 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -61,9 +61,10 @@ type CommitteeIndex = number; * receives and it can be triggered manually. */ export class AttestationPool { - private readonly attestationByRootBySlot = new MapDef>>( - () => new Map>() - ); + private readonly aggregateByIndexByRootBySlot = new MapDef< + Slot, + Map> + >(() => new Map>()); private lowestPermissibleSlot = 0; constructor( @@ -75,8 +76,10 @@ export class AttestationPool { /** Returns current count of pre-aggregated attestations with unique data */ getAttestationCount(): number { let attestationCount = 0; - for (const attestationByRoot of this.attestationByRootBySlot.values()) { - attestationCount += attestationByRoot.size; + for (const attestationByIndexByRoot of this.aggregateByIndexByRootBySlot.values()) { + for (const attestationByIndex of attestationByIndexByRoot.values()) { + attestationCount += attestationByIndex.size; + } } return attestationCount; } @@ -98,7 +101,7 @@ export class AttestationPool { * - Valid committeeIndex * - Valid data */ - add(attestation: allForks.Attestation, attDataRootHex: RootHex): InsertOutcome { + add(committeeIndex: CommitteeIndex, attestation: allForks.Attestation, attDataRootHex: RootHex): InsertOutcome { const slot = attestation.data.slot; const lowestPermissibleSlot = this.lowestPermissibleSlot; @@ -113,15 +116,11 @@ export class AttestationPool { } // Limit object per slot - const aggregateByRoot = this.attestationByRootBySlot.getOrDefault(slot); + const aggregateByRoot = this.aggregateByIndexByRootBySlot.getOrDefault(slot); if (aggregateByRoot.size >= MAX_ATTESTATIONS_PER_SLOT) { throw new OpPoolError({code: OpPoolErrorCode.REACHED_MAX_PER_SLOT}); } - const committeeIndex = isElectraAttestation(attestation) - ? // this attestation is added to pool after validation - attestation.committeeBits.getSingleTrueBit() - : attestation.data.index; // this should not happen because attestation should be validated before reaching this assert.notNull(committeeIndex, "Committee index should not be null in attestation pool"); @@ -146,7 +145,7 @@ export class AttestationPool { * For validator API to get an aggregate */ getAggregate(slot: Slot, committeeIndex: CommitteeIndex, dataRootHex: RootHex): allForks.Attestation | null { - const aggregate = this.attestationByRootBySlot.get(slot)?.get(dataRootHex)?.get(committeeIndex); + const aggregate = this.aggregateByIndexByRootBySlot.get(slot)?.get(dataRootHex)?.get(committeeIndex); if (!aggregate) { // TODO: Add metric for missing aggregates return null; @@ -160,7 +159,7 @@ export class AttestationPool { * By default, not interested in attestations in old slots, we only preaggregate attestations for the current slot. */ prune(clockSlot: Slot): void { - pruneBySlot(this.attestationByRootBySlot, clockSlot, SLOTS_RETAINED); + pruneBySlot(this.aggregateByIndexByRootBySlot, clockSlot, SLOTS_RETAINED); // by default preaggregateSlotDistance is 0, i.e only accept attestations in the same clock slot. this.lowestPermissibleSlot = Math.max(clockSlot - this.preaggregateSlotDistance, 0); } @@ -174,8 +173,8 @@ export class AttestationPool { const aggregateByRoots = bySlot === undefined - ? Array.from(this.attestationByRootBySlot.values()) - : [this.attestationByRootBySlot.get(bySlot)]; + ? Array.from(this.aggregateByIndexByRootBySlot.values()) + : [this.aggregateByIndexByRootBySlot.get(bySlot)]; for (const aggregateByRoot of aggregateByRoots) { if (aggregateByRoot) { diff --git a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts index 9312f3b517a7..17343e386fd7 100644 --- a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts +++ b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts @@ -1,12 +1,16 @@ -import {phase0, RootHex, Slot} from "@lodestar/types"; +import {BitArray} from "@chainsafe/ssz"; +import {CommitteeIndex, phase0, RootHex, Slot} from "@lodestar/types"; import {MapDef} from "@lodestar/utils"; import {Metrics} from "../../metrics/metrics.js"; -import {AttDataBase64} from "../../util/sszBytes.js"; +import {SeenAttDataKey} from "../../util/sszBytes.js"; import {InsertOutcome} from "../opPools/types.js"; export type AttestationDataCacheEntry = { // part of shuffling data, so this does not take memory - committeeIndices: Uint32Array; + committeeValidatorIndices: Uint32Array; + // undefined for phase0 Attestation + committeeBits?: BitArray; + committeeIndex: CommitteeIndex; // IndexedAttestationData signing root, 32 bytes signingRoot: Uint8Array; // to be consumed by forkchoice and oppool @@ -38,12 +42,14 @@ const DEFAULT_MAX_CACHE_SIZE_PER_SLOT = 200; const DEFAULT_CACHE_SLOT_DISTANCE = 2; /** + * Cached seen AttestationData to improve gossip validation. For Electra, this still take into account attestationIndex + * even through it is moved outside of AttestationData. * As of April 2023, validating gossip attestation takes ~12% of cpu time for a node subscribing to all subnets on mainnet. * Having this cache help saves a lot of cpu time since most of the gossip attestations are on the same slot. */ export class SeenAttestationDatas { - private cacheEntryByAttDataBase64BySlot = new MapDef>( - () => new Map() + private cacheEntryByAttDataBase64BySlot = new MapDef>( + () => new Map() ); private lowestPermissibleSlot = 0; @@ -57,14 +63,14 @@ export class SeenAttestationDatas { } // TODO: Move InsertOutcome type definition to a common place - add(slot: Slot, attDataBase64: AttDataBase64, cacheEntry: AttestationDataCacheEntry): InsertOutcome { + add(slot: Slot, attDataKey: SeenAttDataKey, cacheEntry: AttestationDataCacheEntry): InsertOutcome { if (slot < this.lowestPermissibleSlot) { this.metrics?.seenCache.attestationData.reject.inc({reason: RejectReason.too_old}); return InsertOutcome.Old; } const cacheEntryByAttDataBase64 = this.cacheEntryByAttDataBase64BySlot.getOrDefault(slot); - if (cacheEntryByAttDataBase64.has(attDataBase64)) { + if (cacheEntryByAttDataBase64.has(attDataKey)) { this.metrics?.seenCache.attestationData.reject.inc({reason: RejectReason.already_known}); return InsertOutcome.AlreadyKnown; } @@ -74,11 +80,11 @@ export class SeenAttestationDatas { return InsertOutcome.ReachLimit; } - cacheEntryByAttDataBase64.set(attDataBase64, cacheEntry); + cacheEntryByAttDataBase64.set(attDataKey, cacheEntry); return InsertOutcome.NewData; } - get(slot: Slot, attDataBase64: AttDataBase64): AttestationDataCacheEntry | null { + get(slot: Slot, attDataBase64: SeenAttDataKey): AttestationDataCacheEntry | null { const cacheEntryByAttDataBase64 = this.cacheEntryByAttDataBase64BySlot.get(slot); const cacheEntry = cacheEntryByAttDataBase64?.get(attDataBase64); if (cacheEntry) { diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index c2ecb95907e9..6cb4bb435cf4 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -168,7 +168,7 @@ async function validateAggregateAndProof( // [REJECT] The committee index is within the expected range // -- i.e. data.index < get_committee_count_per_slot(state, data.target.epoch) const committeeIndices = cachedAttData - ? cachedAttData.committeeIndices + ? cachedAttData.committeeValidatorIndices : getCommitteeIndices(shuffling, attSlot, attIndex); // [REJECT] The number of aggregation bits matches the committee size diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 046e1180309c..1876d9a3d883 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -1,4 +1,16 @@ -import {phase0, Epoch, Root, Slot, RootHex, ssz, allForks, electra} from "@lodestar/types"; +import {BitArray} from "@chainsafe/ssz"; +import { + phase0, + Epoch, + Root, + Slot, + RootHex, + ssz, + allForks, + electra, + isElectraAttestation, + CommitteeIndex, +} from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq, DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import { @@ -17,10 +29,9 @@ import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/in import {MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC} from "../../constants/index.js"; import {RegenCaller} from "../regen/index.js"; import { - AttDataBase64, + SeenAttDataKey, getAggregationBitsFromAttestationSerialized, - getAttDataBase64FromAttestationSerialized, - getCommitteeBitsFromAttestationSerialized, + getSeenAttDataKey, getSignatureFromAttestationSerialized, } from "../../util/sszBytes.js"; import {AttestationDataCacheEntry} from "../seenCache/seenAttestationData.js"; @@ -39,12 +50,13 @@ export type AttestationValidationResult = { indexedAttestation: allForks.IndexedAttestation; subnet: number; attDataRootHex: RootHex; + committeeIndex: CommitteeIndex; }; export type AttestationOrBytes = ApiAttestation | GossipAttestation; /** attestation from api */ -export type ApiAttestation = {attestation: phase0.Attestation; serializedData: null}; +export type ApiAttestation = {attestation: allForks.Attestation; serializedData: null}; /** attestation from gossip */ export type GossipAttestation = { @@ -52,7 +64,9 @@ export type GossipAttestation = { serializedData: Uint8Array; // available in NetworkProcessor since we check for unknown block root attestations attSlot: Slot; - attDataBase64?: string | null; + // for old LIFO linear gossip queue we don't have attDataBase64 + // for indexed gossip queue we have attDataBase64 + seenAttestationKey?: SeenAttDataKey | null; }; export type Step0Result = AttestationValidationResult & { @@ -83,7 +97,7 @@ export async function validateGossipAttestation( export async function validateGossipAttestationsSameAttData( fork: ForkName, chain: IBeaconChain, - attestationOrBytesArr: AttestationOrBytes[], + attestationOrBytesArr: GossipAttestation[], subnet: number, // for unit test, consumers do not need to pass this step0ValidationFn = validateGossipAttestationNoSignatureCheck @@ -251,15 +265,16 @@ async function validateGossipAttestationNoSignatureCheck( let attestationOrCache: | {attestation: allForks.Attestation; cache: null} | {attestation: null; cache: AttestationDataCacheEntry; serializedData: Uint8Array}; - let attDataBase64: AttDataBase64 | null = null; + let attDataKey: SeenAttDataKey | null = null; if (attestationOrBytes.serializedData) { // gossip const attSlot = attestationOrBytes.attSlot; - // for old LIFO linear gossip queue we don't have attDataBase64 - // for indexed gossip queue we have attDataBase64 - attDataBase64 = - attestationOrBytes.attDataBase64 ?? getAttDataBase64FromAttestationSerialized(attestationOrBytes.serializedData); - const cachedAttData = attDataBase64 !== null ? chain.seenAttestationDatas.get(attSlot, attDataBase64) : null; + attDataKey = + // we always have seenAttestationKey from the IndexedGossipQueue, getSeenAttDataKey() just for backward + // compatible in case beaconAttestationBatchValidation is false + // TODO: remove beaconAttestationBatchValidation flag since the batch attestation is stable + attestationOrBytes.seenAttestationKey ?? getSeenAttDataKey(ForkSeq[fork], attestationOrBytes.serializedData); + const cachedAttData = attDataKey !== null ? chain.seenAttestationDatas.get(attSlot, attDataKey) : null; if (cachedAttData === null) { const attestation = sszDeserializeAttestation(fork, attestationOrBytes.serializedData); // only deserialize on the first AttestationData that's not cached @@ -269,7 +284,7 @@ async function validateGossipAttestationNoSignatureCheck( } } else { // api - attDataBase64 = null; + attDataKey = null; attestationOrCache = {attestation: attestationOrBytes.attestation, cache: null}; } @@ -280,29 +295,33 @@ async function validateGossipAttestationNoSignatureCheck( const attEpoch = computeEpochAtSlot(attSlot); const attTarget = attData.target; const targetEpoch = attTarget.epoch; + let committeeIndex; + if (attestationOrCache.attestation) { + if (isElectraAttestation(attestationOrCache.attestation)) { + // api or first time validation of a gossip attestation + const {committeeBits} = attestationOrCache.attestation; + // throw in both in case of undefined and null + if (committeeBits == null) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.INVALID_SERIALIZED_BYTES}); + } - let attIndex; - if (ForkSeq[fork] >= ForkSeq.electra) { - const committeeBits = attestationOrCache.attestation - ? (attestationOrCache.attestation as electra.Attestation).committeeBits - : getCommitteeBitsFromAttestationSerialized(attestationOrCache.serializedData); - - if (committeeBits === null) { - throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.INVALID_SERIALIZED_BYTES}); - } - - attIndex = committeeBits.getSingleTrueBit(); - // [REJECT] len(committee_indices) == 1, where committee_indices = get_committee_indices(aggregate) - if (attIndex === null) { - throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_COMMITTEE_BIT_SET}); - } + committeeIndex = committeeBits.getSingleTrueBit(); + // [REJECT] len(committee_indices) == 1, where committee_indices = get_committee_indices(aggregate) + if (committeeIndex === null) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NOT_EXACTLY_ONE_COMMITTEE_BIT_SET}); + } - // [REJECT] aggregate.data.index == 0 - if (attData.index !== 0) { - throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NON_ZERO_ATTESTATION_DATA_INDEX}); + // [REJECT] aggregate.data.index == 0 + if (attData.index !== 0) { + throw new AttestationError(GossipAction.REJECT, {code: AttestationErrorCode.NON_ZERO_ATTESTATION_DATA_INDEX}); + } + } else { + // phase0 attestation + committeeIndex = attData.index; } } else { - attIndex = attData.index; + // found a seen AttestationData + committeeIndex = attestationOrCache.cache.committeeIndex; } chain.metrics?.gossipAttestation.attestationSlotToClockSlot.observe( @@ -343,11 +362,11 @@ async function validateGossipAttestationNoSignatureCheck( }); } - let committeeIndices: Uint32Array; + let committeeValidatorIndices: Uint32Array; let getSigningRoot: () => Uint8Array; let expectedSubnet: number; if (attestationOrCache.cache) { - committeeIndices = attestationOrCache.cache.committeeIndices; + committeeValidatorIndices = attestationOrCache.cache.committeeValidatorIndices; const signingRoot = attestationOrCache.cache.signingRoot; getSigningRoot = () => signingRoot; expectedSubnet = attestationOrCache.cache.subnet; @@ -389,17 +408,17 @@ async function validateGossipAttestationNoSignatureCheck( // [REJECT] The committee index is within the expected range // -- i.e. data.index < get_committee_count_per_slot(state, data.target.epoch) - committeeIndices = getCommitteeIndices(shuffling, attSlot, attIndex); + committeeValidatorIndices = getCommitteeIndices(shuffling, attSlot, committeeIndex); getSigningRoot = () => getAttestationDataSigningRoot(chain.config, attData); - expectedSubnet = computeSubnetForSlot(shuffling, attSlot, attIndex); + expectedSubnet = computeSubnetForSlot(shuffling, attSlot, committeeIndex); } - const validatorIndex = committeeIndices[bitIndex]; + const validatorIndex = committeeValidatorIndices[bitIndex]; // [REJECT] The number of aggregation bits matches the committee size // -- i.e. len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot, data.index)). // > TODO: Is this necessary? Lighthouse does not do this check. - if (aggregationBits.bitLen !== committeeIndices.length) { + if (aggregationBits.bitLen !== committeeValidatorIndices.length) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS, }); @@ -445,6 +464,7 @@ async function validateGossipAttestationNoSignatureCheck( }); } + let committeeBits: BitArray | undefined = undefined; if (attestationOrCache.cache) { // there could be up to 6% of cpu time to compute signing root if we don't clone the signature set signatureSet = createSingleSignatureSetFromComponents( @@ -453,6 +473,7 @@ async function validateGossipAttestationNoSignatureCheck( signature ); attDataRootHex = attestationOrCache.cache.attDataRootHex; + committeeBits = attestationOrCache.cache.committeeBits; } else { signatureSet = createSingleSignatureSetFromComponents( chain.index2pubkey[validatorIndex], @@ -462,9 +483,15 @@ async function validateGossipAttestationNoSignatureCheck( // add cached attestation data before verifying signature attDataRootHex = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(attData)); - if (attDataBase64) { - chain.seenAttestationDatas.add(attSlot, attDataBase64, { - committeeIndices, + // if attestation is phase0 the committeeBits is undefined anyway + committeeBits = isElectraAttestation(attestationOrCache.attestation) + ? attestationOrCache.attestation.committeeBits.clone() + : undefined; + if (attDataKey) { + chain.seenAttestationDatas.add(attSlot, attDataKey, { + committeeValidatorIndices, + committeeBits, + committeeIndex, signingRoot: signatureSet.signingRoot, subnet: expectedSubnet, // precompute this to be used in forkchoice @@ -486,14 +513,21 @@ async function validateGossipAttestationNoSignatureCheck( ? (indexedAttestationContent as electra.IndexedAttestation) : (indexedAttestationContent as phase0.IndexedAttestation); - const attestation: allForks.Attestation = attestationOrCache.attestation - ? attestationOrCache.attestation - : { - aggregationBits, - data: attData, - signature, - }; - return {attestation, indexedAttestation, subnet: expectedSubnet, attDataRootHex, signatureSet, validatorIndex}; + const attestation: allForks.Attestation = attestationOrCache.attestation ?? { + aggregationBits, + data: attData, + committeeBits, + signature, + }; + return { + attestation, + indexedAttestation, + subnet: expectedSubnet, + attDataRootHex, + signatureSet, + validatorIndex, + committeeIndex, + }; } /** diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 57697ba2643f..3f2d6368928d 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -37,8 +37,8 @@ import { AggregateAndProofValidationResult, validateGossipAttestationsSameAttData, validateGossipAttestation, - AttestationOrBytes, AttestationValidationResult, + GossipAttestation, } from "../../chain/validation/index.js"; import {NetworkEvent, NetworkEventBus} from "../events.js"; import {PeerAction} from "../peers/index.js"; @@ -487,14 +487,14 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } // Handler - const {indexedAttestation, attDataRootHex, attestation} = validationResult; + const {indexedAttestation, attDataRootHex, attestation, committeeIndex} = validationResult; metrics?.registerGossipUnaggregatedAttestation(seenTimestampSec, indexedAttestation); try { // Node may be subscribe to extra subnets (long-lived random subnets). For those, validate the messages // but don't add to attestation pool, to save CPU and RAM if (aggregatorTracker.shouldAggregate(subnet, indexedAttestation.data.slot)) { - const insertOutcome = chain.attestationPool.add(attestation, attDataRootHex); + const insertOutcome = chain.attestationPool.add(committeeIndex, attestation, attDataRootHex); metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } } catch (e) { @@ -679,7 +679,7 @@ function getBatchHandlers(modules: ValidatorFnsModules, options: GossipHandlerOp serializedData: param.gossipData.serializedData, attSlot: param.gossipData.msgSlot, attDataBase64: param.gossipData.indexed, - })) as AttestationOrBytes[]; + })) as GossipAttestation[]; const {results: validationResults, batchableBls} = await validateGossipAttestationsSameAttData( fork, chain, @@ -695,14 +695,14 @@ function getBatchHandlers(modules: ValidatorFnsModules, options: GossipHandlerOp results.push(null); // Handler - const {indexedAttestation, attDataRootHex, attestation} = validationResult.result; + const {indexedAttestation, attDataRootHex, attestation, committeeIndex} = validationResult.result; metrics?.registerGossipUnaggregatedAttestation(gossipHandlerParams[i].seenTimestampSec, indexedAttestation); try { // Node may be subscribe to extra subnets (long-lived random subnets). For those, validate the messages // but don't add to attestation pool, to save CPU and RAM if (aggregatorTracker.shouldAggregate(subnet, indexedAttestation.data.slot)) { - const insertOutcome = chain.attestationPool.add(attestation, attDataRootHex); + const insertOutcome = chain.attestationPool.add(committeeIndex, attestation, attDataRootHex); metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } } catch (e) { diff --git a/packages/beacon-node/src/network/processor/gossipQueues/index.ts b/packages/beacon-node/src/network/processor/gossipQueues/index.ts index 366b23b30679..b38ee74279ca 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues/index.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues/index.ts @@ -1,7 +1,8 @@ import {mapValues} from "@lodestar/utils"; +import {ForkSeq} from "@lodestar/params"; import {GossipType} from "../../gossip/interface.js"; import {PendingGossipsubMessage} from "../types.js"; -import {getAttDataBase64FromAttestationSerialized} from "../../../util/sszBytes.js"; +import {getSeenAttDataKey} from "../../../util/sszBytes.js"; import {LinearGossipQueue} from "./linear.js"; import { DropType, @@ -86,7 +87,10 @@ const indexedGossipQueueOpts: { } = { [GossipType.beacon_attestation]: { maxLength: 24576, - indexFn: (item: PendingGossipsubMessage) => getAttDataBase64FromAttestationSerialized(item.msg.data), + indexFn: (item: PendingGossipsubMessage) => { + const {topic, msg} = item; + return getSeenAttDataKey(ForkSeq[topic.fork], msg.data); + }, minChunkSize: MIN_SIGNATURE_SETS_TO_BATCH_VERIFY, maxChunkSize: MAX_GOSSIP_ATTESTATION_BATCH_SIZE, }, diff --git a/packages/beacon-node/src/network/processor/gossipQueues/indexed.ts b/packages/beacon-node/src/network/processor/gossipQueues/indexed.ts index 4e29a52173f0..8edba7dfaadb 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues/indexed.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues/indexed.ts @@ -84,6 +84,7 @@ export class IndexedGossipQueueMinSize= ForkSeq.electra ? getSeenAttDataKeyElectra(data) : getSeenAttDataKeyPhase0(data); +} + +/** + * Extract attestation data + committeeBits base64 from electra attestation serialized bytes. + * Return null if data is not long enough to extract attestation data. + */ +export function getSeenAttDataKeyElectra(electraAttestationBytes: Uint8Array): AttDataCommitteeBitsBase64 | null { + const startIndex = VARIABLE_FIELD_OFFSET; + const seenKeyLength = ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE; + + if (electraAttestationBytes.length < startIndex + seenKeyLength) { + return null; + } + + return Buffer.from(electraAttestationBytes.subarray(startIndex, startIndex + seenKeyLength)).toString("base64"); +} + +/** + * Extract attestation data base64 from phase0 attestation serialized bytes. * Return null if data is not long enough to extract attestation data. */ -export function getAttDataBase64FromAttestationSerialized(data: Uint8Array): AttDataBase64 | null { +export function getSeenAttDataKeyPhase0(data: Uint8Array): AttDataBase64 | null { if (data.length < VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE) { return null; } diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index 5fce9a342509..8f462609c063 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -5,7 +5,7 @@ import {ssz} from "@lodestar/types"; import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {validateAttestation, validateGossipAttestationsSameAttData} from "../../../../src/chain/validation/index.js"; import {getAttestationValidData} from "../../../utils/validationData/attestation.js"; -import {getAttDataBase64FromAttestationSerialized} from "../../../../src/util/sszBytes.js"; +import {getSeenAttDataKeyPhase0} from "../../../../src/util/sszBytes.js"; describe("validate gossip attestation", () => { setBenchOpts({ @@ -42,7 +42,7 @@ describe("validate gossip attestation", () => { attestation: null, serializedData, attSlot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet0 ); @@ -67,7 +67,7 @@ describe("validate gossip attestation", () => { attestation: null, serializedData, attSlot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + attDataBase64: getSeenAttDataKeyPhase0(serializedData), }; }); diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts index 56aab699f4f7..c7d9dc775c85 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts @@ -12,7 +12,7 @@ import { validateApiAttestation, validateAttestation, } from "../../../../../src/chain/validation/index.js"; -import {getAttDataBase64FromAttestationSerialized} from "../../../../../src/util/sszBytes.js"; +import {getSeenAttDataKeyPhase0} from "../../../../../src/util/sszBytes.js"; import {memoOnce} from "../../../../utils/cache.js"; import {expectRejectedWithLodestarError} from "../../../../utils/errors.js"; import {AttestationValidDataOpts, getAttestationValidData} from "../../../../utils/validationData/attestation.js"; @@ -52,7 +52,7 @@ describe("validateAttestation", () => { const {chain, subnet} = getValidData(); await expectGossipError( chain, - {attestation: null, serializedData: Buffer.alloc(0), attSlot: 0, attDataBase64: "invalid"}, + {attestation: null, serializedData: Buffer.alloc(0), attSlot: 0, seenAttestationDataKey: "invalid"}, subnet, GossipErrorCode.INVALID_SERIALIZED_BYTES_ERROR_CODE ); @@ -72,7 +72,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.BAD_TARGET_EPOCH @@ -91,7 +91,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.PAST_SLOT @@ -110,7 +110,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.FUTURE_SLOT @@ -135,7 +135,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -155,7 +155,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -179,7 +179,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT @@ -199,7 +199,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.INVALID_TARGET_ROOT @@ -226,7 +226,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS @@ -245,7 +245,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, invalidSubnet, AttestationErrorCode.INVALID_SUBNET_ID @@ -265,7 +265,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.ATTESTATION_ALREADY_KNOWN @@ -287,7 +287,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataBase64FromAttestationSerialized(serializedData), + seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.INVALID_SIGNATURE diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index dd1b5aa747bf..0c768bb3fbb3 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -4,7 +4,7 @@ import {allForks, deneb, Epoch, isElectraAttestation, phase0, RootHex, Slot, ssz import {fromHex, toHex} from "@lodestar/utils"; import {ForkName, MAX_COMMITTEES_PER_SLOT} from "@lodestar/params"; import { - getAttDataBase64FromAttestationSerialized, + getSeenAttDataKeyPhase0, getAttDataBase64FromSignedAggregateAndProofSerialized, getAggregationBitsFromAttestationSerialized, getBlockRootFromAttestationSerialized, @@ -62,7 +62,7 @@ describe("attestation SSZ serialized picking", () => { } const attDataBase64 = ssz.phase0.AttestationData.serialize(attestation.data); - expect(getAttDataBase64FromAttestationSerialized(bytes)).toBe(Buffer.from(attDataBase64).toString("base64")); + expect(getSeenAttDataKeyPhase0(bytes)).toBe(Buffer.from(attDataBase64).toString("base64")); }); } @@ -83,7 +83,7 @@ describe("attestation SSZ serialized picking", () => { it("getAttDataBase64FromAttestationSerialized - invalid data", () => { const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 131]; for (const size of invalidAttDataBase64DataSizes) { - expect(getAttDataBase64FromAttestationSerialized(Buffer.alloc(size))).toBeNull(); + expect(getSeenAttDataKeyPhase0(Buffer.alloc(size))).toBeNull(); } }); diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index f4aeeab15260..51fff4096282 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,5 +1,5 @@ import {toHexString} from "@chainsafe/ssz"; -import {allForks, BLSSignature, phase0, Slot, ssz} from "@lodestar/types"; +import {allForks, BLSSignature, electra, isElectraAttestation, phase0, Slot, ssz} from "@lodestar/types"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; @@ -193,7 +193,7 @@ export class AttestationService { attestationNoCommittee: phase0.AttestationData, duties: AttDutyAndProof[] ): Promise { - const signedAttestations: phase0.Attestation[] = []; + const signedAttestations: allForks.Attestation[] = []; const headRootHex = toHexString(attestationNoCommittee.beaconBlockRoot); const currentEpoch = computeEpochAtSlot(slot); const isAfterElectra = currentEpoch >= this.config.ELECTRA_FORK_EPOCH; diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 62ebe339bff9..ee40e80d811a 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -534,7 +534,7 @@ export class ValidatorStore { data: attestationData, committeeBits: BitArray.fromSingleBit(MAX_COMMITTEES_PER_SLOT, duty.committeeIndex), signature: await this.getSignature(duty.pubkey, signingRoot, signingSlot, signableMessage), - } as electra.Attestation; + }; } else { return { aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), From 9db912a6e68fe4de596ecba7be22a28f793ba127 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 16 May 2024 21:23:52 +0300 Subject: [PATCH 064/145] fix: update withdrawal request container to match consensus spec (#6797) --- packages/beacon-node/src/execution/engine/types.ts | 4 ++-- .../src/block/processExecutionLayerWithdrawalRequest.ts | 2 +- packages/types/src/electra/sszTypes.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index c7b0aefd07ad..8fcca29a026c 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -437,7 +437,7 @@ export function serializeExecutionLayerWithdrawalRequest( ): ExecutionLayerWithdrawalRequestRpc { return { sourceAddress: bytesToData(withdrawalRequest.sourceAddress), - validatorPublicKey: bytesToData(withdrawalRequest.validatorPublicKey), + validatorPublicKey: bytesToData(withdrawalRequest.validatorPubkey), amount: numToQuantity(withdrawalRequest.amount), }; } @@ -447,7 +447,7 @@ export function deserializeExecutionLayerWithdrawalRequest( ): electra.ExecutionLayerWithdrawalRequest { return { sourceAddress: dataToBytes(withdrawalRequest.sourceAddress, 20), - validatorPublicKey: dataToBytes(withdrawalRequest.validatorPublicKey, 48), + validatorPubkey: dataToBytes(withdrawalRequest.validatorPublicKey, 48), amount: quantityToNum(withdrawalRequest.amount), }; } diff --git a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts index 935acd522da1..e02129a09d05 100644 --- a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts @@ -32,7 +32,7 @@ export function processExecutionLayerWithdrawalRequest( // bail out if validator is not in beacon state // note that we don't need to check for 6110 unfinalized vals as they won't be eligible for withdraw/exit anyway - const validatorIndex = pubkey2index.get(executionLayerWithdrawalRequest.validatorPublicKey); + const validatorIndex = pubkey2index.get(executionLayerWithdrawalRequest.validatorPubkey); if (validatorIndex === undefined) { return; } diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 0a0c0f6aeefb..a8249156c7b9 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -128,7 +128,7 @@ export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT export const ExecutionLayerWithdrawalRequest = new ContainerType( { sourceAddress: ExecutionAddress, - validatorPublicKey: BLSPubkey, + validatorPubkey: BLSPubkey, amount: UintNum64, }, {typeName: "ExecutionLayerWithdrawalRequest", jsonCase: "eth2"} From 23868c66c74d2833698fd4b36f22a6be708a0190 Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 17 May 2024 18:47:58 +0300 Subject: [PATCH 065/145] fix: get seen AttData key from SignedAggregateAndProof electra (#6802) * fix: get seen AttData key from SignedAggregateAndProof electra * chore: revert the naming change to COMMITTEE_BITS_SIZE and add comment * fix: add toBase64() util --- .../src/chain/validation/aggregateAndProof.ts | 10 ++- packages/beacon-node/src/util/sszBytes.ts | 43 ++++++++-- .../test/unit/util/sszBytes.test.ts | 86 +++++++++++++++++-- 3 files changed, 121 insertions(+), 18 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 6cb4bb435cf4..176c90a665ac 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -9,7 +9,9 @@ import {toRootHex} from "@lodestar/utils"; import {IBeaconChain} from ".."; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {RegenCaller} from "../regen/index.js"; -import {getAttDataBase64FromSignedAggregateAndProofSerialized} from "../../util/sszBytes.js"; +import { + getSeenAttDataKeyFromSignedAggregateAndProof, +} from "../../util/sszBytes.js"; import {getSelectionProofSignatureSet, getAggregateAndProofSignatureSet} from "./signatureSets/index.js"; import { getAttestationDataSigningRoot, @@ -71,8 +73,10 @@ async function validateAggregateAndProof( const attData = aggregate.data; const attSlot = attData.slot; - const attDataBase64 = serializedData ? getAttDataBase64FromSignedAggregateAndProofSerialized(serializedData) : null; - const cachedAttData = attDataBase64 ? chain.seenAttestationDatas.get(attSlot, attDataBase64) : null; + const seenAttDataKey = serializedData + ? getSeenAttDataKeyFromSignedAggregateAndProof(ForkSeq[fork], serializedData) + : null; + const cachedAttData = seenAttDataKey ? chain.seenAttestationDatas.get(attSlot, seenAttDataKey) : null; let attIndex; if (ForkSeq[fork] >= ForkSeq.electra) { diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index 13b28212a04d..e82673a3167a 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -41,6 +41,7 @@ const ATTESTATION_BEACON_BLOCK_ROOT_OFFSET = VARIABLE_FIELD_OFFSET + 8 + 8; const ROOT_SIZE = 32; const SLOT_SIZE = 8; const ATTESTATION_DATA_SIZE = 128; +// MAX_COMMITTEES_PER_SLOT is in bit, need to convert to byte const COMMITTEE_BITS_SIZE = Math.max(Math.ceil(MAX_COMMITTEES_PER_SLOT / 8), 1); const SIGNATURE_SIZE = 96; @@ -94,7 +95,7 @@ export function getSeenAttDataKeyElectra(electraAttestationBytes: Uint8Array): A return null; } - return Buffer.from(electraAttestationBytes.subarray(startIndex, startIndex + seenKeyLength)).toString("base64"); + return toBase64(electraAttestationBytes.subarray(startIndex, startIndex + seenKeyLength)); } /** @@ -178,8 +179,9 @@ const SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET = AGGREGATE_OFFSET + VARIABLE_FIELD const SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET = SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + 8 + 8; /** - * Extract slot from signed aggregate and proof serialized bytes. - * Return null if data is not long enough to extract slot. + * Extract slot from signed aggregate and proof serialized bytes + * Return null if data is not long enough to extract slot + * This works for both phase + electra */ export function getSlotFromSignedAggregateAndProofSerialized(data: Uint8Array): Slot | null { if (data.length < SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + SLOT_SIZE) { @@ -190,8 +192,9 @@ export function getSlotFromSignedAggregateAndProofSerialized(data: Uint8Array): } /** - * Extract block root from signed aggregate and proof serialized bytes. - * Return null if data is not long enough to extract block root. + * Extract block root from signed aggregate and proof serialized bytes + * Return null if data is not long enough to extract block root + * This works for both phase + electra */ export function getBlockRootFromSignedAggregateAndProofSerialized(data: Uint8Array): BlockRootHex | null { if (data.length < SIGNED_AGGREGATE_AND_PROOF_BLOCK_ROOT_OFFSET + ROOT_SIZE) { @@ -207,11 +210,39 @@ export function getBlockRootFromSignedAggregateAndProofSerialized(data: Uint8Arr return "0x" + blockRootBuf.toString("hex"); } +/** + * Extract attestation data key from SignedAggregateAndProof Uint8Array to use cached data from SeenAttestationDatas + */ +export function getSeenAttDataKeyFromSignedAggregateAndProof( + forkSeq: ForkSeq, + data: Uint8Array +): SeenAttDataKey | null { + return forkSeq >= ForkSeq.electra + ? getSeenAttDataKeyFromSignedAggregateAndProofElectra(data) + : getSeenAttDataKeyFromSignedAggregateAndProofPhase0(data); +} + +/** + * Extract AttestationData + CommitteeBits from SignedAggregateAndProof for electra + * Return null if data is not long enough + */ +export function getSeenAttDataKeyFromSignedAggregateAndProofElectra(data: Uint8Array): SeenAttDataKey | null { + const startIndex = SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET; + const endIndex = startIndex + ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE; + + if (data.length < endIndex) { + return null; + } + + // base64 is a bit efficient than hex + return toBase64(data.subarray(startIndex, endIndex)); +} + /** * Extract attestation data base64 from signed aggregate and proof serialized bytes. * Return null if data is not long enough to extract attestation data. */ -export function getAttDataBase64FromSignedAggregateAndProofSerialized(data: Uint8Array): AttDataBase64 | null { +export function getSeenAttDataKeyFromSignedAggregateAndProofPhase0(data: Uint8Array): AttDataBase64 | null { if (data.length < SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE) { return null; } diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 0c768bb3fbb3..d6ca3ec24621 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -1,12 +1,12 @@ import {describe, it, expect} from "vitest"; import {BitArray} from "@chainsafe/ssz"; -import {allForks, deneb, Epoch, isElectraAttestation, phase0, RootHex, Slot, ssz} from "@lodestar/types"; +import {allForks, deneb, electra, Epoch, isElectraAttestation, phase0, RootHex, Slot, ssz} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; import {ForkName, MAX_COMMITTEES_PER_SLOT} from "@lodestar/params"; import { getSeenAttDataKeyPhase0, - getAttDataBase64FromSignedAggregateAndProofSerialized, - getAggregationBitsFromAttestationSerialized, + getSeenAttDataKeyFromSignedAggregateAndProofPhase0, + getAggregationBitsFromAttestationSerialized as getAggregationBitsFromAttestationSerialized, getBlockRootFromAttestationSerialized, getBlockRootFromSignedAggregateAndProofSerialized, getSlotFromAttestationSerialized, @@ -15,6 +15,7 @@ import { getSlotFromSignedBeaconBlockSerialized, getSlotFromBlobSidecarSerialized, getCommitteeBitsFromAttestationSerialized, + getSeenAttDataKeyFromSignedAggregateAndProofElectra, } from "../../../src/util/sszBytes.js"; describe("attestation SSZ serialized picking", () => { @@ -104,16 +105,15 @@ describe("attestation SSZ serialized picking", () => { }); }); -describe("aggregateAndProof SSZ serialized picking", () => { +describe("phase0 SignedAggregateAndProof SSZ serialized picking", () => { const testCases: phase0.SignedAggregateAndProof[] = [ ssz.phase0.SignedAggregateAndProof.defaultValue(), - signedAggregateAndProofFromValues( + phase0SignedAggregateAndProofFromValues( 4_000_000, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 200_00, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffff" ), - ssz.electra.SignedAggregateAndProof.defaultValue(), ]; for (const [i, signedAggregateAndProof] of testCases.entries()) { @@ -128,7 +128,7 @@ describe("aggregateAndProof SSZ serialized picking", () => { ); const attDataBase64 = ssz.phase0.AttestationData.serialize(signedAggregateAndProof.message.aggregate.data); - expect(getAttDataBase64FromSignedAggregateAndProofSerialized(bytes)).toBe( + expect(getSeenAttDataKeyFromSignedAggregateAndProofPhase0(bytes)).toBe( Buffer.from(attDataBase64).toString("base64") ); }); @@ -151,7 +151,60 @@ describe("aggregateAndProof SSZ serialized picking", () => { it("getAttDataBase64FromSignedAggregateAndProofSerialized - invalid data", () => { const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 339]; for (const size of invalidAttDataBase64DataSizes) { - expect(getAttDataBase64FromSignedAggregateAndProofSerialized(Buffer.alloc(size))).toBeNull(); + expect(getSeenAttDataKeyFromSignedAggregateAndProofPhase0(Buffer.alloc(size))).toBeNull(); + } + }); +}); + +describe("electra SignedAggregateAndProof SSZ serialized picking", () => { + const testCases: electra.SignedAggregateAndProof[] = [ + ssz.electra.SignedAggregateAndProof.defaultValue(), + electraSignedAggregateAndProofFromValues( + 4_000_000, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 200_00, + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffffffffffff" + ), + ]; + + for (const [i, signedAggregateAndProof] of testCases.entries()) { + it(`signedAggregateAndProof ${i}`, () => { + const bytes = ssz.electra.SignedAggregateAndProof.serialize(signedAggregateAndProof); + + expect(getSlotFromSignedAggregateAndProofSerialized(bytes)).toBe( + signedAggregateAndProof.message.aggregate.data.slot + ); + expect(getBlockRootFromSignedAggregateAndProofSerialized(bytes)).toBe( + toHex(signedAggregateAndProof.message.aggregate.data.beaconBlockRoot) + ); + + const attDataBase64 = ssz.phase0.AttestationData.serialize(signedAggregateAndProof.message.aggregate.data); + const committeeBits = ssz.electra.CommitteeBits.serialize( + signedAggregateAndProof.message.aggregate.committeeBits + ); + const seenKey = Buffer.concat([attDataBase64, committeeBits]).toString("base64"); + expect(getSeenAttDataKeyFromSignedAggregateAndProofElectra(bytes)).toBe(seenKey); + }); + } + + it("getSlotFromSignedAggregateAndProofSerialized - invalid data", () => { + const invalidSlotDataSizes = [0, 4, 11]; + for (const size of invalidSlotDataSizes) { + expect(getSlotFromSignedAggregateAndProofSerialized(Buffer.alloc(size))).toBeNull(); + } + }); + + it("getBlockRootFromSignedAggregateAndProofSerialized - invalid data", () => { + const invalidBlockRootDataSizes = [0, 4, 20, 227]; + for (const size of invalidBlockRootDataSizes) { + expect(getBlockRootFromSignedAggregateAndProofSerialized(Buffer.alloc(size))).toBeNull(); + } + }); + + it("getAttDataBase64FromSignedAggregateAndProofSerialized - invalid data", () => { + const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 339]; + for (const size of invalidAttDataBase64DataSizes) { + expect(getSeenAttDataKeyFromSignedAggregateAndProofPhase0(Buffer.alloc(size))).toBeNull(); } }); it("getSlotFromSignedAggregateAndProofSerialized - invalid data - large slots", () => { @@ -215,7 +268,7 @@ function attestationFromValues( return attestation; } -function signedAggregateAndProofFromValues( +function phase0SignedAggregateAndProofFromValues( slot: Slot, blockRoot: RootHex, targetEpoch: Epoch, @@ -229,6 +282,21 @@ function signedAggregateAndProofFromValues( return signedAggregateAndProof; } +function electraSignedAggregateAndProofFromValues( + slot: Slot, + blockRoot: RootHex, + targetEpoch: Epoch, + targetRoot: RootHex +): electra.SignedAggregateAndProof { + const signedAggregateAndProof = ssz.electra.SignedAggregateAndProof.defaultValue(); + signedAggregateAndProof.message.aggregate.data.slot = slot; + signedAggregateAndProof.message.aggregate.data.beaconBlockRoot = fromHex(blockRoot); + signedAggregateAndProof.message.aggregate.data.target.epoch = targetEpoch; + signedAggregateAndProof.message.aggregate.data.target.root = fromHex(targetRoot); + signedAggregateAndProof.message.aggregate.committeeBits = BitArray.fromSingleBit(MAX_COMMITTEES_PER_SLOT, 1); + return signedAggregateAndProof; +} + function signedBeaconBlockFromValues(slot: Slot): phase0.SignedBeaconBlock { const signedBeaconBlock = ssz.phase0.SignedBeaconBlock.defaultValue(); signedBeaconBlock.message.slot = slot; From bd5c8b7bf654e986c7ba3fdff2e54e96604b2fc2 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 23 May 2024 21:56:47 +0200 Subject: [PATCH 066/145] test: only skip ssz_static tests associated to missing type (#6798) * test: only skip ssz_static tests associated to missing type * More detailed error message if type is not defined --- .../test/spec/presets/ssz_static.test.ts | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index e4c5a1914827..bf7a3b5aad06 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -57,25 +57,28 @@ const sszStatic = (ssz.altair as Types)[typeName] || (ssz.phase0 as Types)[typeName]; + it(`${fork} - ${typeName} type exists`, function () { + expect(sszType).toEqualWithMessage(expect.any(Type), `SSZ type ${typeName} for fork ${fork} is not defined`); + }); + if (!sszType) { - expect.fail( - `Missing SSZ type definition for ${typeName}; this will prevent associated ssz_static tests to be executed` - ); - } else { - const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); + // Return instead of throwing an error to only skip ssz_static tests associated to missing type + return; + } + + const sszTypeNoUint = replaceUintTypeWithUintBigintType(sszType); - for (const testCase of fs.readdirSync(testSuiteDirpath)) { - // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts - it(testCase, function () { - // Mainnet must deal with big full states and hash each one multiple times - if (ACTIVE_PRESET === "mainnet") { - vi.setConfig({testTimeout: 30 * 1000}); - } + for (const testCase of fs.readdirSync(testSuiteDirpath)) { + // Do not manually skip tests here, do it in packages/beacon-node/test/spec/presets/index.test.ts + it(testCase, function () { + // Mainnet must deal with big full states and hash each one multiple times + if (ACTIVE_PRESET === "mainnet") { + vi.setConfig({testTimeout: 30 * 1000}); + } - const testData = parseSszStaticTestcase(path.join(testSuiteDirpath, testCase)); - runValidSszTest(sszTypeNoUint, testData); - }); - } + const testData = parseSszStaticTestcase(path.join(testSuiteDirpath, testCase)); + runValidSszTest(sszTypeNoUint, testData); + }); } }; From 3d19cb487717047caeaf3064790f45698eb4ca65 Mon Sep 17 00:00:00 2001 From: g11tech Date: Sat, 25 May 2024 13:45:36 +0530 Subject: [PATCH 067/145] chore: types and lint fixes (#6819) --- .../src/chain/validation/aggregateAndProof.ts | 4 +--- .../perf/chain/validation/attestation.test.ts | 2 +- .../attestation/validateAttestation.test.ts | 24 +++++++++---------- .../validator/src/services/attestation.ts | 2 +- .../validator/src/services/validatorStore.ts | 5 +--- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 176c90a665ac..3c8b102eb5ac 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -9,9 +9,7 @@ import {toRootHex} from "@lodestar/utils"; import {IBeaconChain} from ".."; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {RegenCaller} from "../regen/index.js"; -import { - getSeenAttDataKeyFromSignedAggregateAndProof, -} from "../../util/sszBytes.js"; +import {getSeenAttDataKeyFromSignedAggregateAndProof} from "../../util/sszBytes.js"; import {getSelectionProofSignatureSet, getAggregateAndProofSignatureSet} from "./signatureSets/index.js"; import { getAttestationDataSigningRoot, diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index 8f462609c063..e0e0a1e51169 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -42,7 +42,7 @@ describe("validate gossip attestation", () => { attestation: null, serializedData, attSlot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet0 ); diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts index c7d9dc775c85..4a1c3badae50 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts @@ -52,7 +52,7 @@ describe("validateAttestation", () => { const {chain, subnet} = getValidData(); await expectGossipError( chain, - {attestation: null, serializedData: Buffer.alloc(0), attSlot: 0, seenAttestationDataKey: "invalid"}, + {attestation: null, serializedData: Buffer.alloc(0), attSlot: 0, seenAttestationKey: "invalid"}, subnet, GossipErrorCode.INVALID_SERIALIZED_BYTES_ERROR_CODE ); @@ -72,7 +72,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.BAD_TARGET_EPOCH @@ -91,7 +91,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.PAST_SLOT @@ -110,7 +110,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.FUTURE_SLOT @@ -135,7 +135,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -155,7 +155,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -179,7 +179,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT @@ -199,7 +199,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.INVALID_TARGET_ROOT @@ -226,7 +226,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS @@ -245,7 +245,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, invalidSubnet, AttestationErrorCode.INVALID_SUBNET_ID @@ -265,7 +265,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.ATTESTATION_ALREADY_KNOWN @@ -287,7 +287,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationDataKey: getSeenAttDataKeyPhase0(serializedData), + seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), }, subnet, AttestationErrorCode.INVALID_SIGNATURE diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 51fff4096282..61bd169afeda 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,5 +1,5 @@ import {toHexString} from "@chainsafe/ssz"; -import {allForks, BLSSignature, electra, isElectraAttestation, phase0, Slot, ssz} from "@lodestar/types"; +import {allForks, BLSSignature, phase0, Slot, ssz} from "@lodestar/types"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index ee40e80d811a..a85dd52a7c0d 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -29,7 +29,6 @@ import { BlindedBeaconBlock, BLSPubkey, BLSSignature, - electra, Epoch, phase0, Root, @@ -806,9 +805,7 @@ export class ValidatorStore { ); } if (isAfterElectra && data.index !== 0) { - throw Error( - `Non-zero committee index post-electra during signing: att.committeeIndex ${data.index}` - ); + throw Error(`Non-zero committee index post-electra during signing: att.committeeIndex ${data.index}`); } if (this.config.getForkSeq(duty.slot) >= ForkSeq.electra && data.index !== 0) { throw Error(`Attestataion data index must be 0 post electra: index ${data.index}`); From e75d90631fb3d33f3a65ddd8881de1757120d339 Mon Sep 17 00:00:00 2001 From: NC Date: Sat, 8 Jun 2024 12:46:01 +0300 Subject: [PATCH 068/145] feat: add engine_getPayloadBodiesByHash and ByRange V2 (#6852) * Add ByHash and ByRange V2 * Fix build issue * Fix CI error --- packages/beacon-node/src/chain/chain.ts | 2 +- .../beacon-node/src/eth1/eth1DepositDataTracker.ts | 4 ++-- packages/beacon-node/src/execution/engine/http.ts | 9 ++++++--- .../beacon-node/src/execution/engine/interface.ts | 6 ++++-- packages/beacon-node/src/execution/engine/mock.ts | 2 ++ packages/beacon-node/src/execution/engine/types.ts | 10 +++++++--- .../beacon-node/test/sim/electra-interop.test.ts | 10 +++++----- .../test/spec/presets/operations.test.ts | 2 +- .../test/unit/executionEngine/http.test.ts | 4 ++-- packages/beacon-node/test/utils/state.ts | 2 +- packages/light-client/src/spec/utils.ts | 6 +++--- .../src/block/processDepositRequest.ts | 4 ++-- .../src/block/processOperations.ts | 2 +- .../src/slot/upgradeStateToElectra.ts | 14 +++++++------- packages/state-transition/src/util/deposit.ts | 4 ++-- packages/state-transition/src/util/execution.ts | 4 ++-- packages/state-transition/src/util/genesis.ts | 2 +- .../test/unit/util/deposit.test.ts | 4 ++-- packages/types/src/electra/sszTypes.ts | 12 ++++++------ packages/types/src/electra/types.ts | 4 ++-- 20 files changed, 59 insertions(+), 48 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 9f67857fe530..9f30936305f0 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1147,7 +1147,7 @@ export class BeaconChain implements IBeaconChain { // Will resolve this later // if (cpEpoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity)) { // // finalizedState can be safely casted to Electra state since cp is already post-Electra - // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositReceiptsStartIndex) { + // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositRequestsStartIndex) { // // Signal eth1 to stop polling eth1Data // this.eth1.stopPollingEth1Data(); // } diff --git a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts index c0b3ab35a73a..d0578718f29f 100644 --- a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts +++ b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts @@ -129,9 +129,9 @@ export class Eth1DepositDataTracker { async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { if ( state.epochCtx.isAfterElectra() && - state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositReceiptsStartIndex + state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositRequestsStartIndex ) { - // No need to poll eth1Data since Electra deprecates the mechanism after depositReceiptsStartIndex is reached + // No need to poll eth1Data since Electra deprecates the mechanism after depositRequestsStartIndex is reached return {eth1Data: state.eth1Data, deposits: []}; } const eth1Data = this.forcedEth1DataVote ?? (await this.getEth1Data(state)); diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 10a805e26d66..a69a5b94bd65 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -418,8 +418,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { this.payloadIdCache.prune(); } - async getPayloadBodiesByHash(blockHashes: RootHex[]): Promise<(ExecutionPayloadBody | null)[]> { - const method = "engine_getPayloadBodiesByHashV1"; + async getPayloadBodiesByHash(fork: ForkName, blockHashes: RootHex[]): Promise<(ExecutionPayloadBody | null)[]> { + const method = + ForkSeq[fork] >= ForkSeq.electra ? "engine_getPayloadBodiesByHashV2" : "engine_getPayloadBodiesByHashV1"; assertReqSizeLimit(blockHashes.length, 32); const response = await this.rpc.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], @@ -429,10 +430,12 @@ export class ExecutionEngineHttp implements IExecutionEngine { } async getPayloadBodiesByRange( + fork: ForkName, startBlockNumber: number, blockCount: number ): Promise<(ExecutionPayloadBody | null)[]> { - const method = "engine_getPayloadBodiesByRangeV1"; + const method = + ForkSeq[fork] >= ForkSeq.electra ? "engine_getPayloadBodiesByRangeV2" : "engine_getPayloadBodiesByRangeV1"; assertReqSizeLimit(blockCount, 32); const start = numToQuantity(startBlockNumber); const count = numToQuantity(blockCount); diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index c2e44901afbb..a835fa76f509 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -174,7 +174,9 @@ export interface IExecutionEngine { shouldOverrideBuilder?: boolean; }>; - getPayloadBodiesByHash(blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>; + getPayloadBodiesByHash(fork: ForkName, blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>; - getPayloadBodiesByRange(start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>; + getPayloadBodiesByRange(fork: ForkName, start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>; + + getState(): ExecutionEngineState; } diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index 4cff2a8d00f4..9fed781fa523 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -98,8 +98,10 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_getPayloadV3: this.getPayload.bind(this), engine_getPayloadV4: this.getPayload.bind(this), engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this), + engine_getPayloadBodiesByHashV2: this.getPayloadBodiesByHash.bind(this), engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this), engine_getClientVersionV1: this.getClientVersionV1.bind(this), + engine_getPayloadBodiesByRangeV2: this.getPayloadBodiesByRange.bind(this), }; } diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 8fcca29a026c..7d57b45d134d 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -58,6 +58,7 @@ export type EngineApiRpcParamTypes = { * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure * */ engine_getPayloadBodiesByHashV1: DATA[][]; + engine_getPayloadBodiesByHashV2: DATA[][]; /** * 1. start: QUANTITY, 64 bits - Starting block number @@ -69,6 +70,7 @@ export type EngineApiRpcParamTypes = { * Object - Instance of ClientVersion */ engine_getClientVersionV1: [ClientVersionRpc]; + engine_getPayloadBodiesByRangeV2: [start: QUANTITY, count: QUANTITY]; }; export type PayloadStatus = { @@ -107,10 +109,12 @@ export type EngineApiRpcReturnTypes = { engine_getPayloadV4: ExecutionPayloadResponse; engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[]; + engine_getPayloadBodiesByHashV2: (ExecutionPayloadBodyRpc | null)[]; engine_getPayloadBodiesByRangeV1: (ExecutionPayloadBodyRpc | null)[]; engine_getClientVersionV1: ClientVersionRpc[]; + engine_getPayloadBodiesByRangeV2: (ExecutionPayloadBodyRpc | null)[]; }; type ExecutionPayloadRpcWithValue = { @@ -235,8 +239,8 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload // ELECTRA adds depositRequests/depositRequests to the ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositReceipts, withdrawalRequests} = data as electra.ExecutionPayload; - payload.depositRequests = depositReceipts.map(serializeDepositRequest); + const {depositRequests, withdrawalRequests} = data as electra.ExecutionPayload; + payload.depositRequests = depositRequests.map(serializeDepositRequest); payload.withdrawalRequests = withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest); } @@ -334,7 +338,7 @@ export function parseExecutionPayload( `depositRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` ); } - (executionPayload as electra.ExecutionPayload).depositReceipts = depositRequests.map(deserializeDepositRequest); + (executionPayload as electra.ExecutionPayload).depositRequests = depositRequests.map(deserializeDepositRequest); if (withdrawalRequests == null) { throw Error( diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 962e0aa5dea9..2dfa5ee0f77c 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -219,11 +219,11 @@ describe("executionEngine / ExecutionEngineHttp", function () { } } - if (payload.depositReceipts.length !== 1) { - throw Error(`Number of depositRequests mismatched. Expected: 1, actual: ${payload.depositReceipts.length}`); + if (payload.depositRequests.length !== 1) { + throw Error(`Number of depositRequests mismatched. Expected: 1, actual: ${payload.depositRequests.length}`); } - const actualDepositRequest = payload.depositReceipts[0]; + const actualDepositRequest = payload.depositRequests[0]; assert.deepStrictEqual( actualDepositRequest, depositRequestB, @@ -431,8 +431,8 @@ describe("executionEngine / ExecutionEngineHttp", function () { throw Error("Historical validator length for epoch 1 or 2 is not dropped properly"); } - if (headState.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { - throw Error("state.depositReceiptsStartIndex is not set upon processing new deposit receipt"); + if (headState.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + throw Error("state.depositRequestsStartIndex is not set upon processing new deposit receipt"); } // wait for 1 slot to print current epoch stats diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index de8a32a6056a..552e0cfa44b6 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -143,7 +143,7 @@ const operations: TestRunnerFn = (fork, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, - deposit_receipt: ssz.electra.DepositReceipt, + deposit_receipt: ssz.electra.DepositRequest, proposer_slashing: ssz.phase0.ProposerSlashing, voluntary_exit: ssz.phase0.SignedVoluntaryExit, // Altair diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index 2cc08c8c78d2..90ca6595d8a2 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -223,7 +223,7 @@ describe("ExecutionEngine / http", () => { returnValue = response; - const res = await executionEngine.getPayloadBodiesByHash(reqBlockHashes); + const res = await executionEngine.getPayloadBodiesByHash(ForkName.bellatrix, reqBlockHashes); expect(reqJsonRpcPayload).toEqual(request); expect(res.map(serializeExecutionPayloadBody)).toEqual(response.result); @@ -276,7 +276,7 @@ describe("ExecutionEngine / http", () => { returnValue = response; - const res = await executionEngine.getPayloadBodiesByRange(startBlockNumber, blockCount); + const res = await executionEngine.getPayloadBodiesByRange(ForkName.bellatrix, startBlockNumber, blockCount); expect(reqJsonRpcPayload).toEqual(request); expect(res.map(serializeExecutionPayloadBody)).toEqual(response.result); diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index ad4dd4b88dac..bb55adca1eb0 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -97,7 +97,7 @@ export function generateState( if (forkSeq >= ForkSeq.electra) { const stateElectra = state as electra.BeaconState; - stateElectra.depositReceiptsStartIndex = 2023n; + stateElectra.depositRequestsStartIndex = 2023n; stateElectra.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue(); } diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index b1145860833f..679371a54d6c 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -115,8 +115,8 @@ export function upgradeLightClientHeader( // eslint-disable-next-line no-fallthrough case ForkName.electra: - (upgradedHeader as LightClientHeader).execution.depositReceiptsRoot = - ssz.electra.LightClientHeader.fields.execution.fields.depositReceiptsRoot.defaultValue(); + (upgradedHeader as LightClientHeader).execution.depositRequestsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.depositRequestsRoot.defaultValue(); (upgradedHeader as LightClientHeader).execution.withdrawalRequestsRoot = ssz.electra.LightClientHeader.fields.execution.fields.withdrawalRequestsRoot.defaultValue(); @@ -157,7 +157,7 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC if (epoch < config.ELECTRA_FORK_EPOCH) { if ( - (header as LightClientHeader).execution.depositReceiptsRoot !== undefined || + (header as LightClientHeader).execution.depositRequestsRoot !== undefined || (header as LightClientHeader).execution.withdrawalRequestsRoot !== undefined ) { return false; diff --git a/packages/state-transition/src/block/processDepositRequest.ts b/packages/state-transition/src/block/processDepositRequest.ts index d5525f3cd544..ca6fe4206188 100644 --- a/packages/state-transition/src/block/processDepositRequest.ts +++ b/packages/state-transition/src/block/processDepositRequest.ts @@ -9,8 +9,8 @@ export function processDepositRequest( state: CachedBeaconStateElectra, depositRequest: electra.DepositRequest ): void { - if (state.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { - state.depositReceiptsStartIndex = BigInt(depositRequest.index); + if (state.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + state.depositRequestsStartIndex = BigInt(depositRequest.index); } applyDeposit(fork, state, depositRequest); diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 86672a6572e9..a2bd691337ad 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -71,7 +71,7 @@ export function processOperations( processExecutionLayerWithdrawalRequest(fork, state as CachedBeaconStateElectra, elWithdrawalRequest); } - for (const depositRequest of bodyElectra.executionPayload.depositReceipts) { + for (const depositRequest of bodyElectra.executionPayload.depositRequests) { processDepositRequest(fork, stateElectra, depositRequest); } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 72255ccddee3..3d77366cbe57 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -50,16 +50,16 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache stateElectraView.nextSyncCommittee = stateElectraCloned.nextSyncCommittee; stateElectraView.latestExecutionPayloadHeader = ssz.electra.BeaconState.fields.latestExecutionPayloadHeader.toViewDU({ ...stateElectraCloned.latestExecutionPayloadHeader.toValue(), - depositReceiptsRoot: ssz.Root.defaultValue(), + depositRequestsRoot: ssz.Root.defaultValue(), withdrawalRequestsRoot: ssz.Root.defaultValue(), }); stateElectraView.nextWithdrawalIndex = stateDeneb.nextWithdrawalIndex; stateElectraView.nextWithdrawalValidatorIndex = stateDeneb.nextWithdrawalValidatorIndex; stateElectraView.historicalSummaries = stateElectraCloned.historicalSummaries; - // latestExecutionPayloadHeader's depositReceiptsRoot and withdrawalRequestsRoot set to zeros by default - // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX - stateElectraView.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default + // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectraView.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; stateElectraView.depositBalanceToConsume = BigInt(0); stateElectraView.exitBalanceToConsume = BigInt(0); @@ -136,9 +136,9 @@ export function upgradeStateToElectraOriginal(stateDeneb: CachedBeaconStateDeneb epoch: stateDeneb.epochCtx.epoch, }); - // latestExecutionPayloadHeader's depositReceiptsRoot and withdrawalRequestsRoot set to zeros by default - // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX - stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default + // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; const validatorsArr = stateElectra.validators.getAllReadonly(); diff --git a/packages/state-transition/src/util/deposit.ts b/packages/state-transition/src/util/deposit.ts index 493099fdd982..e8ef93c515d2 100644 --- a/packages/state-transition/src/util/deposit.ts +++ b/packages/state-transition/src/util/deposit.ts @@ -9,9 +9,9 @@ export function getEth1DepositCount(state: CachedBeaconStateAllForks, eth1Data?: // eth1DataIndexLimit = min(UintNum64, UintBn64) can be safely casted as UintNum64 // since the result lies within upper and lower bound of UintNum64 const eth1DataIndexLimit: UintNum64 = - eth1DataToUse.depositCount < electraState.depositReceiptsStartIndex + eth1DataToUse.depositCount < electraState.depositRequestsStartIndex ? eth1DataToUse.depositCount - : Number(electraState.depositReceiptsStartIndex); + : Number(electraState.depositRequestsStartIndex); if (state.eth1DepositIndex < eth1DataIndexLimit) { return Math.min(MAX_DEPOSITS, eth1DataIndexLimit - state.eth1DepositIndex); diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index c7f0ec2f395c..2c55dc61a502 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -172,8 +172,8 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio } if (fork >= ForkSeq.electra) { - (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositReceiptsRoot = - ssz.electra.DepositReceipts.hashTreeRoot((payload as electra.ExecutionPayload).depositReceipts); + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositRequestsRoot = + ssz.electra.DepositRequests.hashTreeRoot((payload as electra.ExecutionPayload).depositRequests); (bellatrixPayloadFields as electra.ExecutionPayloadHeader).withdrawalRequestsRoot = ssz.electra.ExecutionLayerWithdrawalRequests.hashTreeRoot( (payload as electra.ExecutionPayload).withdrawalRequests diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index e1003dca1946..ca894fd5b4e5 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -308,7 +308,7 @@ export function initializeBeaconStateFromEth1( stateElectra.latestExecutionPayloadHeader = (executionPayloadHeader as CompositeViewDU) ?? ssz.electra.ExecutionPayloadHeader.defaultViewDU(); - stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; } state.commit(); diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts index e8f7f7a86af8..677a15724ece 100644 --- a/packages/state-transition/test/unit/util/deposit.test.ts +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -43,7 +43,7 @@ describe("getEth1DepositCount", () => { throw Error("Not a post-Electra state"); } - postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.depositRequestsStartIndex = 1000n; postElectraState.eth1Data.depositCount = 995; // 1. Should get less than MAX_DEPOSIT @@ -77,7 +77,7 @@ describe("getEth1DepositCount", () => { throw Error("Not a post-Electra state"); } - postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.depositRequestsStartIndex = 1000n; postElectraState.eth1Data.depositCount = 1005; // Before eth1DepositIndex reaching the start index diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index a8249156c7b9..b03be1697fd1 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -112,7 +112,7 @@ export const SignedAggregateAndProof = new ContainerType( {typeName: "SignedAggregateAndProof", jsonCase: "eth2"} ); -export const DepositReceipt = new ContainerType( +export const DepositRequest = new ContainerType( { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, @@ -120,10 +120,10 @@ export const DepositReceipt = new ContainerType( signature: BLSSignature, index: DepositIndex, }, - {typeName: "DepositReceipt", jsonCase: "eth2"} + {typeName: "DepositRequest", jsonCase: "eth2"} ); -export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); +export const DepositRequests = new ListCompositeType(DepositRequest, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); export const ExecutionLayerWithdrawalRequest = new ContainerType( { @@ -141,7 +141,7 @@ export const ExecutionLayerWithdrawalRequests = new ListCompositeType( export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, - depositReceipts: DepositReceipts, // New in ELECTRA + depositRequests: DepositRequests, // New in ELECTRA withdrawalRequests: ExecutionLayerWithdrawalRequests, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} @@ -150,7 +150,7 @@ export const ExecutionPayload = new ContainerType( export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, - depositReceiptsRoot: Root, // New in ELECTRA + depositRequestsRoot: Root, // New in ELECTRA withdrawalRequestsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} @@ -340,7 +340,7 @@ export const BeaconState = new ContainerType( nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, // Deep history valid from Capella onwards historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, - depositReceiptsStartIndex: UintBn64, // New in ELECTRA:EIP6110 + depositRequestsStartIndex: UintBn64, // New in ELECTRA:EIP6110 depositBalanceToConsume: Gwei, // New in Electra:EIP7251 exitBalanceToConsume: Gwei, // New in Electra:EIP7251 earliestExitEpoch: Epoch, // New in Electra:EIP7251 diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index fc95ecde562d..994e5a732243 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -9,8 +9,8 @@ export type AttesterSlashing = ValueOf; export type AggregateAndProof = ValueOf; export type SignedAggregateAndProof = ValueOf; -export type DepositRequest = ValueOf; -export type DepositRequests = ValueOf; +export type DepositRequest = ValueOf; +export type DepositRequests = ValueOf; export type ExecutionLayerWithdrawalRequest = ValueOf; export type ExecutionLayerWithdrawalRequests = ValueOf; From d87de02064a2d4ed2a10d142a6497f2ca183dc4f Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 12 Jun 2024 09:22:40 +0100 Subject: [PATCH 069/145] fix: align WithdrawalRequestV1 field names with latest spec (#6877) --- packages/beacon-node/src/execution/engine/payloadIdCache.ts | 2 +- packages/beacon-node/src/execution/engine/types.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index 35a50e97cfb3..f79c28582459 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -28,7 +28,7 @@ export type DepositRequestV1 = { export type ExecutionLayerWithdrawalRequestV1 = { sourceAddress: DATA; - validatorPublicKey: DATA; + validatorPubkey: DATA; amount: QUANTITY; }; diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 7d57b45d134d..447d14992526 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -441,7 +441,7 @@ export function serializeExecutionLayerWithdrawalRequest( ): ExecutionLayerWithdrawalRequestRpc { return { sourceAddress: bytesToData(withdrawalRequest.sourceAddress), - validatorPublicKey: bytesToData(withdrawalRequest.validatorPubkey), + validatorPubkey: bytesToData(withdrawalRequest.validatorPubkey), amount: numToQuantity(withdrawalRequest.amount), }; } @@ -451,7 +451,7 @@ export function deserializeExecutionLayerWithdrawalRequest( ): electra.ExecutionLayerWithdrawalRequest { return { sourceAddress: dataToBytes(withdrawalRequest.sourceAddress, 20), - validatorPubkey: dataToBytes(withdrawalRequest.validatorPublicKey, 48), + validatorPubkey: dataToBytes(withdrawalRequest.validatorPubkey, 48), amount: quantityToNum(withdrawalRequest.amount), }; } From 44e004156c9cd62733d2ead8ad8871645afe410d Mon Sep 17 00:00:00 2001 From: NC Date: Fri, 14 Jun 2024 14:11:23 +0300 Subject: [PATCH 070/145] feat: move attestation committee at the end of attestation (#6883) --- .../src/chain/validation/attestation.ts | 2 +- packages/beacon-node/src/util/sszBytes.ts | 26 ++++++++++--------- .../test/unit/util/sszBytes.test.ts | 8 +++--- packages/types/src/electra/sszTypes.ts | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 1876d9a3d883..1d5568fbc868 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -457,7 +457,7 @@ async function validateGossipAttestationNoSignatureCheck( let attDataRootHex: RootHex; const signature = attestationOrCache.attestation ? attestationOrCache.attestation.signature - : getSignatureFromAttestationSerialized(fork, attestationOrCache.serializedData); + : getSignatureFromAttestationSerialized(attestationOrCache.serializedData); if (signature === null) { throw new AttestationError(GossipAction.REJECT, { code: AttestationErrorCode.INVALID_SERIALIZED_BYTES, diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index e82673a3167a..bac66ddf3a0d 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -25,8 +25,8 @@ export type AttDataCommitteeBitsBase64 = string; // class Attestation(Container): // aggregation_bits: BitList[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT] - offset 4 // data: AttestationData - target data - 128 -// committee_bits: BitVector[MAX_COMMITTEES_PER_SLOT] // signature: BLSSignature - 96 +// committee_bits: BitVector[MAX_COMMITTEES_PER_SLOT] // // for all forks // class AttestationData(Container): 128 bytes fixed size @@ -88,14 +88,19 @@ export function getSeenAttDataKey(forkSeq: ForkSeq, data: Uint8Array): SeenAttDa * Return null if data is not long enough to extract attestation data. */ export function getSeenAttDataKeyElectra(electraAttestationBytes: Uint8Array): AttDataCommitteeBitsBase64 | null { - const startIndex = VARIABLE_FIELD_OFFSET; - const seenKeyLength = ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE; + const attestationData = getSeenAttDataKeyPhase0(electraAttestationBytes); - if (electraAttestationBytes.length < startIndex + seenKeyLength) { + if (attestationData === null) { return null; } - return toBase64(electraAttestationBytes.subarray(startIndex, startIndex + seenKeyLength)); + const committeeBits = getCommitteeBitsFromAttestationSerialized(electraAttestationBytes); + + if (committeeBits === null) { + return null; + } + + return attestationData + toBase64(committeeBits.uint8Array); } /** @@ -119,7 +124,7 @@ export function getSeenAttDataKeyPhase0(data: Uint8Array): AttDataBase64 | null export function getAggregationBitsFromAttestationSerialized(fork: ForkName, data: Uint8Array): BitArray | null { const aggregationBitsStartIndex = ForkSeq[fork] >= ForkSeq.electra - ? VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE + SIGNATURE_SIZE + ? VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE + COMMITTEE_BITS_SIZE : VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE; if (data.length < aggregationBitsStartIndex) { @@ -134,11 +139,8 @@ export function getAggregationBitsFromAttestationSerialized(fork: ForkName, data * Extract signature from attestation serialized bytes. * Return null if data is not long enough to extract signature. */ -export function getSignatureFromAttestationSerialized(fork: ForkName, data: Uint8Array): BLSSignature | null { - const signatureStartIndex = - ForkSeq[fork] >= ForkSeq.electra - ? VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE - : VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE; +export function getSignatureFromAttestationSerialized(data: Uint8Array): BLSSignature | null { + const signatureStartIndex = VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE; if (data.length < signatureStartIndex + SIGNATURE_SIZE) { return null; @@ -152,7 +154,7 @@ export function getSignatureFromAttestationSerialized(fork: ForkName, data: Uint * Return null if data is not long enough to extract committee bits. */ export function getCommitteeBitsFromAttestationSerialized(data: Uint8Array): BitArray | null { - const committeeBitsStartIndex = VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE; + const committeeBitsStartIndex = VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE; if (data.length < committeeBitsStartIndex + COMMITTEE_BITS_SIZE) { return null; diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index d6ca3ec24621..2d138111a05f 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -54,12 +54,12 @@ describe("attestation SSZ serialized picking", () => { attestation.aggregationBits.toBoolArray() ); expect(getCommitteeBitsFromAttestationSerialized(bytes)).toEqual(attestation.committeeBits); - expect(getSignatureFromAttestationSerialized(ForkName.electra, bytes)).toEqual(attestation.signature); + expect(getSignatureFromAttestationSerialized(bytes)).toEqual(attestation.signature); } else { expect(getAggregationBitsFromAttestationSerialized(ForkName.phase0, bytes)?.toBoolArray()).toEqual( attestation.aggregationBits.toBoolArray() ); - expect(getSignatureFromAttestationSerialized(ForkName.phase0, bytes)).toEqual(attestation.signature); + expect(getSignatureFromAttestationSerialized(bytes)).toEqual(attestation.signature); } const attDataBase64 = ssz.phase0.AttestationData.serialize(attestation.data); @@ -99,8 +99,8 @@ describe("attestation SSZ serialized picking", () => { it("getSignatureFromAttestationSerialized - invalid data", () => { const invalidSignatureDataSizes = [0, 4, 100, 128, 227]; for (const size of invalidSignatureDataSizes) { - expect(getSignatureFromAttestationSerialized(ForkName.phase0, Buffer.alloc(size))).toBeNull(); - expect(getSignatureFromAttestationSerialized(ForkName.electra, Buffer.alloc(size))).toBeNull(); + expect(getSignatureFromAttestationSerialized(Buffer.alloc(size))).toBeNull(); + expect(getSignatureFromAttestationSerialized(Buffer.alloc(size))).toBeNull(); } }); }); diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index b03be1697fd1..ab9cbbaacd80 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -62,8 +62,8 @@ export const Attestation = new ContainerType( { aggregationBits: AggregationBits, // Modified in ELECTRA data: phase0Ssz.AttestationData, - committeeBits: CommitteeBits, // New in ELECTRA signature: BLSSignature, + committeeBits: CommitteeBits, // New in ELECTRA }, {typeName: "Attestation", jsonCase: "eth2"} ); From 43fe8de1871882d671a3bdb8cbd8fe26dbd2aa81 Mon Sep 17 00:00:00 2001 From: harkamal Date: Wed, 19 Jun 2024 19:27:59 +0530 Subject: [PATCH 071/145] rebase fixes chore: fix bls and blst versioning fix: add ForkName.electra to ForkBlobsInfo some api header lookup fixes more api fixes make the api data safe Co-authored-by: matthewkeil --- packages/api/src/beacon/routes/beacon/pool.ts | 11 ++++++++-- packages/api/src/beacon/routes/validator.ts | 21 ++++++++++++++----- .../api/test/unit/beacon/testData/beacon.ts | 2 +- .../test/unit/beacon/testData/validator.ts | 2 +- .../test/unit/webEsmBundle.browser.test.ts | 4 ++-- packages/state-transition/package.json | 1 + .../test/unit/services/attestation.test.ts | 5 ++++- 7 files changed, 34 insertions(+), 12 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/pool.ts b/packages/api/src/beacon/routes/beacon/pool.ts index 3c390aa41481..499e8af432d7 100644 --- a/packages/api/src/beacon/routes/beacon/pool.ts +++ b/packages/api/src/beacon/routes/beacon/pool.ts @@ -17,6 +17,7 @@ import { } from "../../../utils/codecs.js"; import {MetaHeader, VersionCodec, VersionMeta} from "../../../utils/metadata.js"; import {toForkName} from "../../../utils/fork.js"; +import {fromHeaders} from "../../../utils/headers.js"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -247,7 +248,12 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = toForkName(headers[MetaHeader.Version]); + const versionHeader = fromHeaders(headers, MetaHeader.Version, false); + const fork = + versionHeader !== undefined + ? toForkName(versionHeader) + : config.getForkName(Number((body as {data: {slot: string}}[])[0]?.data.slot ?? 0)); + return { signedAttestations: ForkSeq[fork] >= ForkSeq.electra @@ -266,7 +272,8 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = toForkName(headers[MetaHeader.Version]); + const versionHeader = fromHeaders(headers, MetaHeader.Version, true); + const fork = toForkName(versionHeader); return { signedAttestations: ForkSeq[fork] >= ForkSeq.electra diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 8c8355f63d07..ac8526947c3f 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -41,6 +41,7 @@ import { VersionMeta, VersionType, } from "../../utils/metadata.js"; +import {fromHeaders} from "../../utils/headers.js"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -811,7 +812,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ attestationDataRoot: fromHexString(query.attestation_data_root), slot: query.slot, - committeeIndex: query.slot, + committeeIndex: query.committeeIndex, }), schema: { query: { @@ -833,7 +834,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = config.getForkName(signedAggregateAndProofs[0].message.aggregate.data.slot); + const fork = config.getForkName(signedAggregateAndProofs[0]?.message.aggregate.data.slot ?? 0); return { body: ForkSeq[fork] >= ForkSeq.electra @@ -847,7 +848,16 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = toForkName(headers[MetaHeader.Version]); + const versionHeader = fromHeaders(headers, MetaHeader.Version, false); + const fork = + versionHeader !== undefined + ? toForkName(versionHeader) + : config.getForkName( + Number( + (body as {message: {aggregate: {data: {slot: string}}}}[])[0]?.message.aggregate.data.slot ?? 0 + ) + ); + return { signedAggregateAndProofs: ForkSeq[fork] >= ForkSeq.electra @@ -856,7 +866,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = config.getForkName(signedAggregateAndProofs[0].message.aggregate.data.slot); + const fork = config.getForkName(signedAggregateAndProofs[0]?.message.aggregate.data.slot ?? 0); return { body: ForkSeq[fork] >= ForkSeq.electra @@ -870,7 +880,8 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = toForkName(headers[MetaHeader.Version]); + const versionHeader = fromHeaders(headers, MetaHeader.Version, true); + const fork = toForkName(versionHeader); return { signedAggregateAndProofs: ForkSeq[fork] >= ForkSeq.electra diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index 9a89abd68a14..da13d4c7783c 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -92,7 +92,7 @@ export const testData: GenericServerTestCases = { getPoolAttestations: { args: {slot: 1, committeeIndex: 2}, - res: {data: [ssz.phase0.Attestation.defaultValue()]}, + res: {data: [ssz.phase0.Attestation.defaultValue()], meta: {version: ForkName.deneb}}, }, getPoolAttesterSlashings: { args: undefined, diff --git a/packages/api/test/unit/beacon/testData/validator.ts b/packages/api/test/unit/beacon/testData/validator.ts index 5f82d4a818b0..b27fc9c38733 100644 --- a/packages/api/test/unit/beacon/testData/validator.ts +++ b/packages/api/test/unit/beacon/testData/validator.ts @@ -99,7 +99,7 @@ export const testData: GenericServerTestCases = { res: {data: ssz.altair.SyncCommitteeContribution.defaultValue()}, }, getAggregatedAttestation: { - args: {attestationDataRoot: ZERO_HASH, slot: 32000, index: 2}, + args: {attestationDataRoot: ZERO_HASH, slot: 32000, committeeIndex: 2}, res: {data: ssz.phase0.Attestation.defaultValue(), meta: {version: ForkName.phase0}}, }, publishAggregateAndProofs: { diff --git a/packages/light-client/test/unit/webEsmBundle.browser.test.ts b/packages/light-client/test/unit/webEsmBundle.browser.test.ts index defc421d7071..49b0f877d46c 100644 --- a/packages/light-client/test/unit/webEsmBundle.browser.test.ts +++ b/packages/light-client/test/unit/webEsmBundle.browser.test.ts @@ -1,7 +1,7 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access */ import {expect, describe, it, vi, beforeAll} from "vitest"; import {sleep} from "@lodestar/utils"; -import {Lightclient, LightclientEvent, utils, transport} from "../../dist/lightclient.min.mjs"; +import {Lightclient, LightclientEvent, utils, transport} from "../../src/index.js"; describe("web bundle for lightclient", () => { vi.setConfig({testTimeout: 20_000}); diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 61b73ef7efc1..c9551195e87e 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -59,6 +59,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/as-sha256": "^0.5.0", + "@chainsafe/bls": "7.1.3", "@chainsafe/blst": "^2.0.3", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/persistent-ts": "^0.19.1", diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index f02b15f38853..fdfc34c857b9 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -5,6 +5,7 @@ import {ssz} from "@lodestar/types"; import {routes} from "@lodestar/api"; import {createChainForkConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; +import {ForkName} from "@lodestar/params"; import {AttestationService, AttestationServiceOpts} from "../../../src/services/attestation.js"; import {AttDutyAndProof} from "../../../src/services/attestationDuties.js"; import {ValidatorStore} from "../../../src/services/validatorStore.js"; @@ -105,7 +106,9 @@ describe("AttestationService", function () { // Mock beacon's attestation and aggregates endpoints api.validator.produceAttestationData.mockResolvedValue(mockApiResponse({data: attestation.data})); - api.validator.getAggregatedAttestation.mockResolvedValue(mockApiResponse({data: attestation})); + api.validator.getAggregatedAttestation.mockResolvedValue( + mockApiResponse({data: attestation, meta: {version: ForkName.phase0}}) + ); api.beacon.submitPoolAttestations.mockResolvedValue(mockApiResponse({})); api.validator.publishAggregateAndProofs.mockResolvedValue(mockApiResponse({})); From 4dfe326c6ba1807e285ab21f535543b611828b76 Mon Sep 17 00:00:00 2001 From: NC Date: Fri, 21 Jun 2024 19:32:37 +0300 Subject: [PATCH 072/145] feat: handle exited/exiting validators during top up (#6880) * Handle exiting validataor * lint * Add todo --- .../epoch/processPendingBalanceDeposits.ts | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts index 95928d66df2d..112832c920d7 100644 --- a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts @@ -1,6 +1,8 @@ +import {FAR_FUTURE_EPOCH} from "@lodestar/params"; import {CachedBeaconStateElectra} from "../types.js"; import {increaseBalance} from "../util/balance.js"; import {getActivationExitChurnLimit} from "../util/validator.js"; +import {getCurrentEpoch} from "../util/epoch.js"; /** * Starting from Electra: @@ -8,19 +10,40 @@ import {getActivationExitChurnLimit} from "../util/validator.js"; * For each eligible `deposit`, call `increaseBalance()`. * Remove the processed deposits from `state.pendingBalanceDeposits`. * Update `state.depositBalanceToConsume` for the next epoch + * + * TODO Electra: Update ssz library to support batch push to `pendingBalanceDeposits` */ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): void { const availableForProcessing = state.depositBalanceToConsume + BigInt(getActivationExitChurnLimit(state)); + const currentEpoch = getCurrentEpoch(state); let processedAmount = 0n; let nextDepositIndex = 0; + const depositsToPostpone = []; for (const deposit of state.pendingBalanceDeposits.getAllReadonly()) { - const {amount} = deposit; - if (processedAmount + amount > availableForProcessing) { - break; + const {amount, index: depositIndex} = deposit; + const validator = state.validators.get(depositIndex); + + // Validator is exiting, postpone the deposit until after withdrawable epoch + if (validator.exitEpoch < FAR_FUTURE_EPOCH) { + if (currentEpoch <= validator.withdrawableEpoch) { + depositsToPostpone.push(deposit); + } else { + // Deposited balance will never become active. Increase balance but do not consume churn + increaseBalance(state, deposit.index, Number(amount)); + } + } else { + // Validator is not exiting, attempt to process deposit + if (processedAmount + amount > availableForProcessing) { + // Deposit does not fit in the churn, no more deposit processing in this epoch. + break; + } else { + // Deposit fits in the churn, process it. Increase balance and consume churn. + increaseBalance(state, deposit.index, Number(amount)); + processedAmount = processedAmount + amount; + } } - increaseBalance(state, deposit.index, Number(amount)); - processedAmount = processedAmount + amount; + // Regardless of how the deposit was handled, we move on in the queue. nextDepositIndex++; } @@ -32,4 +55,9 @@ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): } else { state.depositBalanceToConsume = availableForProcessing - processedAmount; } + + // TODO Electra: add a function in ListCompositeTreeView to support batch push operation + for (const deposit of depositsToPostpone) { + state.pendingBalanceDeposits.push(deposit); + } } From 39edbfdfcd6170dc5ebfa1194b5973b8ce55102f Mon Sep 17 00:00:00 2001 From: NC Date: Fri, 21 Jun 2024 19:33:42 +0300 Subject: [PATCH 073/145] feat: add EL triggered consolidation and remove `ExecutionLayer` prefix (#6865) * init commit * Add consolidation request * fix lint * Remove ExecutionLayer prefix * Address comment * Fix verification logic * lint * Add todo --- .../chain/produceBlock/produceBlockBody.ts | 4 - .../src/execution/engine/payloadIdCache.ts | 2 +- .../beacon-node/src/execution/engine/types.ts | 33 ++---- .../test/spec/presets/operations.test.ts | 20 ++-- packages/params/src/index.ts | 2 +- packages/params/src/presets/mainnet.ts | 2 +- packages/params/src/presets/minimal.ts | 2 +- packages/params/src/types.ts | 4 +- .../src/block/processConsolidation.ts | 111 ------------------ .../src/block/processConsolidationRequest.ts | 74 ++++++++++++ .../src/block/processOperations.ts | 20 ++-- ...Request.ts => processWithdrawalRequest.ts} | 10 +- .../src/signatureSets/consolidation.ts | 51 -------- .../src/signatureSets/index.ts | 12 -- .../src/slot/upgradeStateToElectra.ts | 1 + .../state-transition/src/util/execution.ts | 4 +- packages/types/src/electra/sszTypes.ts | 44 +++---- packages/types/src/electra/types.ts | 10 +- packages/validator/src/util/params.ts | 2 +- .../test/unit/utils/interopConfigs.ts | 8 +- 20 files changed, 144 insertions(+), 272 deletions(-) delete mode 100644 packages/state-transition/src/block/processConsolidation.ts create mode 100644 packages/state-transition/src/block/processConsolidationRequest.ts rename packages/state-transition/src/block/{processExecutionLayerWithdrawalRequest.ts => processWithdrawalRequest.ts} (89%) delete mode 100644 packages/state-transition/src/signatureSets/consolidation.ts diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 53f59850b647..638df42cba1b 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -343,10 +343,6 @@ export async function produceBlockBody( } } - if (ForkSeq[fork] >= ForkSeq.electra) { - (blockBody as electra.BeaconBlockBody).consolidations = []; - } - Object.assign(logMeta, {executionPayloadValue}); this.logger.verbose("Produced beacon block body", logMeta); diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index f79c28582459..8cb5b4e4f8e9 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -26,7 +26,7 @@ export type DepositRequestV1 = { index: QUANTITY; }; -export type ExecutionLayerWithdrawalRequestV1 = { +export type WithdrawalRequestV1 = { sourceAddress: DATA; validatorPubkey: DATA; amount: QUANTITY; diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 447d14992526..6b37f3854b8f 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1, DepositRequestV1, ExecutionLayerWithdrawalRequestV1} from "./payloadIdCache.js"; +import {WithdrawalV1, DepositRequestV1, WithdrawalRequestV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -132,14 +132,14 @@ export type ExecutionPayloadBodyRpc = { // currently there is a discepancy between EL and CL field name references for deposit requests // its likely CL receipt will be renamed to requests depositRequests: DepositRequestV1[] | null | undefined; - withdrawalRequests: ExecutionLayerWithdrawalRequestV1[] | null | undefined; + withdrawalRequests: WithdrawalRequestV1[] | null | undefined; }; export type ExecutionPayloadBody = { transactions: bellatrix.Transaction[]; withdrawals: capella.Withdrawals | null; depositRequests: electra.DepositRequests | null; - withdrawalRequests: electra.ExecutionLayerWithdrawalRequests | null; + withdrawalRequests: electra.WithdrawalRequests | null; }; export type ExecutionPayloadRpc = { @@ -162,7 +162,7 @@ export type ExecutionPayloadRpc = { excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB depositRequests?: DepositRequestRpc[]; // ELECTRA - withdrawalRequests?: ExecutionLayerWithdrawalRequestRpc[]; // ELECTRA + withdrawalRequests?: WithdrawalRequestRpc[]; // ELECTRA }; export type WithdrawalRpc = { @@ -173,7 +173,7 @@ export type WithdrawalRpc = { }; export type DepositRequestRpc = DepositRequestV1; -export type ExecutionLayerWithdrawalRequestRpc = ExecutionLayerWithdrawalRequestV1; +export type WithdrawalRequestRpc = WithdrawalRequestV1; export type VersionedHashesRpc = DATA[]; @@ -241,7 +241,7 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { const {depositRequests, withdrawalRequests} = data as electra.ExecutionPayload; payload.depositRequests = depositRequests.map(serializeDepositRequest); - payload.withdrawalRequests = withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest); + payload.withdrawalRequests = withdrawalRequests.map(serializeWithdrawalRequest); } return payload; @@ -345,9 +345,8 @@ export function parseExecutionPayload( `withdrawalRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` ); } - (executionPayload as electra.ExecutionPayload).withdrawalRequests = withdrawalRequests.map( - deserializeExecutionLayerWithdrawalRequest - ); + (executionPayload as electra.ExecutionPayload).withdrawalRequests = + withdrawalRequests.map(deserializeWithdrawalRequest); } return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; @@ -436,9 +435,7 @@ export function deserializeDepositRequest(serialized: DepositRequestRpc): electr } as electra.DepositRequest; } -export function serializeExecutionLayerWithdrawalRequest( - withdrawalRequest: electra.ExecutionLayerWithdrawalRequest -): ExecutionLayerWithdrawalRequestRpc { +export function serializeWithdrawalRequest(withdrawalRequest: electra.WithdrawalRequest): WithdrawalRequestRpc { return { sourceAddress: bytesToData(withdrawalRequest.sourceAddress), validatorPubkey: bytesToData(withdrawalRequest.validatorPubkey), @@ -446,9 +443,7 @@ export function serializeExecutionLayerWithdrawalRequest( }; } -export function deserializeExecutionLayerWithdrawalRequest( - withdrawalRequest: ExecutionLayerWithdrawalRequestRpc -): electra.ExecutionLayerWithdrawalRequest { +export function deserializeWithdrawalRequest(withdrawalRequest: WithdrawalRequestRpc): electra.WithdrawalRequest { return { sourceAddress: dataToBytes(withdrawalRequest.sourceAddress, 20), validatorPubkey: dataToBytes(withdrawalRequest.validatorPubkey, 48), @@ -462,9 +457,7 @@ export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, depositRequests: data.depositRequests ? data.depositRequests.map(deserializeDepositRequest) : null, - withdrawalRequests: data.withdrawalRequests - ? data.withdrawalRequests.map(deserializeExecutionLayerWithdrawalRequest) - : null, + withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(deserializeWithdrawalRequest) : null, } : null; } @@ -475,9 +468,7 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, depositRequests: data.depositRequests ? data.depositRequests.map(serializeDepositRequest) : null, - withdrawalRequests: data.withdrawalRequests - ? data.withdrawalRequests.map(serializeExecutionLayerWithdrawalRequest) - : null, + withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(serializeWithdrawalRequest) : null, } : null; } diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 552e0cfa44b6..47895540ac1a 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -92,20 +92,14 @@ const operationFns: Record> = blockFns.processWithdrawals(ForkSeq.capella, state as CachedBeaconStateCapella, testCase.execution_payload); }, - consolidation: (state, testCase: {consolidation: electra.SignedConsolidation}) => { - blockFns.processConsolidation(state as CachedBeaconStateElectra, testCase.consolidation); + withdrawal_request: (state, testCase: {withdrawal_request: electra.WithdrawalRequest}) => { + blockFns.processWithdrawalRequest(ForkSeq.electra, state as CachedBeaconStateElectra, testCase.withdrawal_request); }, - execution_layer_withdrawal_request: ( - state, - testCase: {execution_layer_withdrawal_request: electra.ExecutionLayerWithdrawalRequest} - ) => { - blockFns.processExecutionLayerWithdrawalRequest( - ForkSeq.electra, - state as CachedBeaconStateElectra, - testCase.execution_layer_withdrawal_request - ); + consolidation_request: (state, testCase: {consolidation_request: electra.ConsolidationRequest}) => { + blockFns.processConsolidationRequest(state as CachedBeaconStateElectra, testCase.consolidation_request); }, + }; export type BlockProcessFn = (state: T, testCase: any) => void; @@ -156,8 +150,8 @@ const operations: TestRunnerFn = (fork, // Capella address_change: ssz.capella.SignedBLSToExecutionChange, // Electra - consolidation: ssz.electra.SignedConsolidation, - execution_layer_withdrawal_request: ssz.electra.ExecutionLayerWithdrawalRequest, + withdrawal_request: ssz.electra.WithdrawalRequest, + consolidation_request: ssz.electra.ConsolidationRequest, }, shouldError: (testCase) => testCase.post === undefined, getExpected: (testCase) => testCase.post, diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 0c2e3b34794b..a3dfb65da350 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -100,7 +100,7 @@ export const { PENDING_PARTIAL_WITHDRAWALS_LIMIT, PENDING_CONSOLIDATIONS_LIMIT, MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA, - MAX_CONSOLIDATIONS, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 2495f7ef97a1..072936492bd3 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -133,6 +133,6 @@ export const mainnetPreset: BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728, PENDING_CONSOLIDATIONS_LIMIT: 262144, - MAX_CONSOLIDATIONS: 1, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1, WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 8e71407965d7..4a1e4af50665 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -134,6 +134,6 @@ export const minimalPreset: BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 64, PENDING_CONSOLIDATIONS_LIMIT: 64, - MAX_CONSOLIDATIONS: 1, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1, WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index dffd98518006..14e8f8c172d8 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -95,7 +95,7 @@ export type BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: number; PENDING_PARTIAL_WITHDRAWALS_LIMIT: number; PENDING_CONSOLIDATIONS_LIMIT: number; - MAX_CONSOLIDATIONS: number; + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: number; WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: number; }; @@ -195,7 +195,7 @@ export const beaconPresetTypes: BeaconPresetTypes = { PENDING_BALANCE_DEPOSITS_LIMIT: "number", PENDING_PARTIAL_WITHDRAWALS_LIMIT: "number", PENDING_CONSOLIDATIONS_LIMIT: "number", - MAX_CONSOLIDATIONS: "number", + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: "number", WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: "number", }; diff --git a/packages/state-transition/src/block/processConsolidation.ts b/packages/state-transition/src/block/processConsolidation.ts deleted file mode 100644 index 846b0e2521b4..000000000000 --- a/packages/state-transition/src/block/processConsolidation.ts +++ /dev/null @@ -1,111 +0,0 @@ -import {toHexString} from "@chainsafe/ssz"; -import {electra, ssz} from "@lodestar/types"; -import {FAR_FUTURE_EPOCH, MIN_ACTIVATION_BALANCE, PENDING_CONSOLIDATIONS_LIMIT} from "@lodestar/params"; -import {verifyConsolidationSignature} from "../signatureSets/index.js"; - -import {CachedBeaconStateElectra} from "../types.js"; -import {getConsolidationChurnLimit, isActiveValidator} from "../util/validator.js"; -import {hasExecutionWithdrawalCredential} from "../util/electra.js"; -import {computeConsolidationEpochAndUpdateChurn} from "../util/epoch.js"; - -export function processConsolidation( - state: CachedBeaconStateElectra, - signedConsolidation: electra.SignedConsolidation -): void { - assertValidConsolidation(state, signedConsolidation); - - // Initiate source validator exit and append pending consolidation - const {sourceIndex, targetIndex} = signedConsolidation.message; - const sourceValidator = state.validators.get(sourceIndex); - - const exitEpoch = computeConsolidationEpochAndUpdateChurn(state, BigInt(sourceValidator.effectiveBalance)); - sourceValidator.exitEpoch = exitEpoch; - sourceValidator.withdrawableEpoch = exitEpoch + state.config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; - - const pendingConsolidation = ssz.electra.PendingConsolidation.toViewDU({ - sourceIndex, - targetIndex, - }); - state.pendingConsolidations.push(pendingConsolidation); -} - -function assertValidConsolidation( - state: CachedBeaconStateElectra, - signedConsolidation: electra.SignedConsolidation -): void { - // If the pending consolidations queue is full, no consolidations are allowed in the block - if (state.pendingConsolidations.length >= PENDING_CONSOLIDATIONS_LIMIT) { - throw new Error("Pending consolidation queue is full"); - } - - // If there is too little available consolidation churn limit, no consolidations are allowed in the block - // assert get_consolidation_churn_limit(state) > MIN_ACTIVATION_BALANCE - if (getConsolidationChurnLimit(state) <= MIN_ACTIVATION_BALANCE) { - throw new Error(`Consolidation churn limit too low. consolidationChurnLimit=${getConsolidationChurnLimit(state)}`); - } - - const consolidation = signedConsolidation.message; - const {sourceIndex, targetIndex} = consolidation; - - // Verify that source != target, so a consolidation cannot be used as an exit. - if (sourceIndex === targetIndex) { - throw new Error( - `Consolidation source and target index cannot be the same: sourceIndex=${sourceIndex} targetIndex=${targetIndex}` - ); - } - - const sourceValidator = state.validators.getReadonly(sourceIndex); - const targetValidator = state.validators.getReadonly(targetIndex); - const currentEpoch = state.epochCtx.epoch; - - // Verify the source and the target are active - if (!isActiveValidator(sourceValidator, currentEpoch)) { - throw new Error(`Consolidation source validator is not active: sourceIndex=${sourceIndex}`); - } - - if (!isActiveValidator(targetValidator, currentEpoch)) { - throw new Error(`Consolidation target validator is not active: targetIndex=${targetIndex}`); - } - - // Verify exits for source and target have not been initiated - if (sourceValidator.exitEpoch !== FAR_FUTURE_EPOCH) { - throw new Error(`Consolidation source validator has initialized exit: sourceIndex=${sourceIndex}`); - } - if (targetValidator.exitEpoch !== FAR_FUTURE_EPOCH) { - throw new Error(`Consolidation target validator has initialized exit: targetIndex=${targetIndex}`); - } - - // Consolidations must specify an epoch when they become valid; they are not valid before then - if (currentEpoch < consolidation.epoch) { - throw new Error( - `Consolidation epoch is after the current epoch: consolidationEpoch=${consolidation.epoch} currentEpoch=${currentEpoch}` - ); - } - - // Verify the source and the target have Execution layer withdrawal credentials - if (!hasExecutionWithdrawalCredential(sourceValidator.withdrawalCredentials)) { - throw new Error( - `Consolidation source validator does not have execution withdrawal credentials: sourceIndex=${sourceIndex}` - ); - } - if (!hasExecutionWithdrawalCredential(targetValidator.withdrawalCredentials)) { - throw new Error( - `Consolidation target validator does not have execution withdrawal credentials: targetIndex=${targetIndex}` - ); - } - - // Verify the same withdrawal address - const sourceWithdrawalAddress = toHexString(sourceValidator.withdrawalCredentials.subarray(12)); - const targetWithdrawalAddress = toHexString(targetValidator.withdrawalCredentials.subarray(12)); - - if (sourceWithdrawalAddress !== targetWithdrawalAddress) { - throw new Error( - `Consolidation source and target withdrawal address are different: source: ${sourceWithdrawalAddress} target: ${targetWithdrawalAddress}` - ); - } - - // Verify consolidation is signed by the source and the target - if (!verifyConsolidationSignature(state, signedConsolidation)) { - throw new Error("Consolidation not valid"); - } -} diff --git a/packages/state-transition/src/block/processConsolidationRequest.ts b/packages/state-transition/src/block/processConsolidationRequest.ts new file mode 100644 index 000000000000..4b6a65e036ad --- /dev/null +++ b/packages/state-transition/src/block/processConsolidationRequest.ts @@ -0,0 +1,74 @@ +import {electra, ssz} from "@lodestar/types"; +import {FAR_FUTURE_EPOCH, MIN_ACTIVATION_BALANCE, PENDING_CONSOLIDATIONS_LIMIT} from "@lodestar/params"; + +import {CachedBeaconStateElectra} from "../types.js"; +import {getConsolidationChurnLimit, isActiveValidator} from "../util/validator.js"; +import {hasExecutionWithdrawalCredential} from "../util/electra.js"; +import {computeConsolidationEpochAndUpdateChurn} from "../util/epoch.js"; + +export function processConsolidationRequest( + state: CachedBeaconStateElectra, + consolidationRequest: electra.ConsolidationRequest +): void { + + // If the pending consolidations queue is full, consolidation requests are ignored + if (state.pendingConsolidations.length >= PENDING_CONSOLIDATIONS_LIMIT) { + return; + } + + // If there is too little available consolidation churn limit, consolidation requests are ignored + if (getConsolidationChurnLimit(state) <= MIN_ACTIVATION_BALANCE) { + return; + } + + const {sourcePubkey, targetPubkey} = consolidationRequest; + const sourceIndex = state.epochCtx.getValidatorIndex(sourcePubkey); + const targetIndex = state.epochCtx.getValidatorIndex(targetPubkey); + + if (sourceIndex === undefined || targetIndex === undefined) { + return; + } + + // Verify that source != target, so a consolidation cannot be used as an exit. + if (sourceIndex === targetIndex){ + return; + } + + const sourceValidator = state.validators.getReadonly(sourceIndex); + const targetValidator = state.validators.getReadonly(targetIndex); + const sourceWithdrawalAddress = sourceValidator.withdrawalCredentials.subarray(12); + const currentEpoch = state.epochCtx.epoch; + + // Verify withdrawal credentials + if ( + !hasExecutionWithdrawalCredential(sourceValidator.withdrawalCredentials) || + !hasExecutionWithdrawalCredential(targetValidator.withdrawalCredentials) + ) { + return; + } + + if (Buffer.compare(sourceWithdrawalAddress, consolidationRequest.sourceAddress) !== 0) { + return; + } + + // Verify the source and the target are active + if (!isActiveValidator(sourceValidator, currentEpoch) || !isActiveValidator(targetValidator, currentEpoch)) { + return; + } + + // Verify exits for source and target have not been initiated + if (sourceValidator.exitEpoch !== FAR_FUTURE_EPOCH || targetValidator.exitEpoch !== FAR_FUTURE_EPOCH) { + return; + } + + // TODO Electra: See if we can get rid of big int + const exitEpoch = computeConsolidationEpochAndUpdateChurn(state, BigInt(sourceValidator.effectiveBalance)); + sourceValidator.exitEpoch = exitEpoch; + sourceValidator.withdrawableEpoch = exitEpoch + state.config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; + + const pendingConsolidation = ssz.electra.PendingConsolidation.toViewDU({ + sourceIndex, + targetIndex, + }); + state.pendingConsolidations.push(pendingConsolidation); +} \ No newline at end of file diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index a2bd691337ad..8290771d2d8e 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -9,10 +9,10 @@ import {processAttesterSlashing} from "./processAttesterSlashing.js"; import {processDeposit} from "./processDeposit.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; -import {processExecutionLayerWithdrawalRequest} from "./processExecutionLayerWithdrawalRequest.js"; +import {processWithdrawalRequest} from "./processWithdrawalRequest.js"; import {processDepositRequest} from "./processDepositRequest.js"; import {ProcessBlockOpts} from "./types.js"; -import {processConsolidation} from "./processConsolidation.js"; +import {processConsolidationRequest} from "./processConsolidationRequest.js"; export { processProposerSlashing, @@ -20,10 +20,10 @@ export { processAttestations, processDeposit, processVoluntaryExit, - processExecutionLayerWithdrawalRequest, + processWithdrawalRequest, processBlsToExecutionChange, processDepositRequest, - processConsolidation, + processConsolidationRequest, }; export function processOperations( @@ -67,16 +67,16 @@ export function processOperations( const stateElectra = state as CachedBeaconStateElectra; const bodyElectra = body as electra.BeaconBlockBody; - for (const elWithdrawalRequest of bodyElectra.executionPayload.withdrawalRequests) { - processExecutionLayerWithdrawalRequest(fork, state as CachedBeaconStateElectra, elWithdrawalRequest); + for (const depositRequest of bodyElectra.executionPayload.depositReceipts) { + processDepositRequest(fork, stateElectra, depositRequest); } - for (const depositRequest of bodyElectra.executionPayload.depositRequests) { - processDepositRequest(fork, stateElectra, depositRequest); + for (const elWithdrawalRequest of bodyElectra.executionPayload.withdrawalRequests) { + processWithdrawalRequest(fork, state as CachedBeaconStateElectra, elWithdrawalRequest); } - for (const consolidation of bodyElectra.consolidations) { - processConsolidation(stateElectra, consolidation); + for (const elConsolidationRequest of bodyElectra.executionPayload.consolidationRequests) { + processConsolidationRequest(stateElectra, elConsolidationRequest); } } } diff --git a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts b/packages/state-transition/src/block/processWithdrawalRequest.ts similarity index 89% rename from packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts rename to packages/state-transition/src/block/processWithdrawalRequest.ts index e02129a09d05..cff1ea03bd80 100644 --- a/packages/state-transition/src/block/processExecutionLayerWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processWithdrawalRequest.ts @@ -14,12 +14,12 @@ import {getPendingBalanceToWithdraw, isActiveValidator} from "../util/validator. import {computeExitEpochAndUpdateChurn} from "../util/epoch.js"; import {initiateValidatorExit} from "./initiateValidatorExit.js"; -export function processExecutionLayerWithdrawalRequest( +export function processWithdrawalRequest( fork: ForkSeq, state: CachedBeaconStateElectra, - executionLayerWithdrawalRequest: electra.ExecutionLayerWithdrawalRequest + withdrawalRequest: electra.WithdrawalRequest ): void { - const amount = Number(executionLayerWithdrawalRequest.amount); + const amount = Number(withdrawalRequest.amount); const {pendingPartialWithdrawals, validators, epochCtx} = state; // no need to use unfinalized pubkey cache from 6110 as validator won't be active anyway const {pubkey2index, config} = epochCtx; @@ -32,13 +32,13 @@ export function processExecutionLayerWithdrawalRequest( // bail out if validator is not in beacon state // note that we don't need to check for 6110 unfinalized vals as they won't be eligible for withdraw/exit anyway - const validatorIndex = pubkey2index.get(executionLayerWithdrawalRequest.validatorPubkey); + const validatorIndex = pubkey2index.get(withdrawalRequest.validatorPubkey); if (validatorIndex === undefined) { return; } const validator = validators.get(validatorIndex); - if (!isValidatorEligibleForWithdrawOrExit(validator, executionLayerWithdrawalRequest.sourceAddress, state)) { + if (!isValidatorEligibleForWithdrawOrExit(validator, withdrawalRequest.sourceAddress, state)) { return; } diff --git a/packages/state-transition/src/signatureSets/consolidation.ts b/packages/state-transition/src/signatureSets/consolidation.ts deleted file mode 100644 index 0aef47d417a5..000000000000 --- a/packages/state-transition/src/signatureSets/consolidation.ts +++ /dev/null @@ -1,51 +0,0 @@ -import {DOMAIN_CONSOLIDATION, ForkName} from "@lodestar/params"; -import {electra, ssz} from "@lodestar/types"; - -import { - computeSigningRoot, - createAggregateSignatureSetFromComponents, - ISignatureSet, - verifySignatureSet, -} from "../util/index.js"; -import {CachedBeaconStateElectra} from "../types.js"; - -export function verifyConsolidationSignature( - state: CachedBeaconStateElectra, - signedConsolidation: electra.SignedConsolidation -): boolean { - return verifySignatureSet(getConsolidationSignatureSet(state, signedConsolidation)); -} - -/** - * Extract signatures to allow validating all block signatures at once - */ -export function getConsolidationSignatureSet( - state: CachedBeaconStateElectra, - signedConsolidation: electra.SignedConsolidation -): ISignatureSet { - const {config} = state; - const {index2pubkey} = state.epochCtx; // TODO Electra: Use 6110 pubkey cache - const {sourceIndex, targetIndex} = signedConsolidation.message; - const sourcePubkey = index2pubkey[sourceIndex]; - const targetPubkey = index2pubkey[targetIndex]; - - // signatureFork for signing domain is fixed - const signatureFork = ForkName.phase0; - const domain = config.getDomainAtFork(signatureFork, DOMAIN_CONSOLIDATION); - const signingRoot = computeSigningRoot(ssz.electra.Consolidation, signedConsolidation.message, domain); - - return createAggregateSignatureSetFromComponents( - [sourcePubkey, targetPubkey], - signingRoot, - signedConsolidation.signature - ); -} - -export function getConsolidationSignatureSets( - state: CachedBeaconStateElectra, - signedBlock: electra.SignedBeaconBlock -): ISignatureSet[] { - return signedBlock.message.body.consolidations.map((consolidation) => - getConsolidationSignatureSet(state, consolidation) - ); -} diff --git a/packages/state-transition/src/signatureSets/index.ts b/packages/state-transition/src/signatureSets/index.ts index 5f063235735c..eae1d434a121 100644 --- a/packages/state-transition/src/signatureSets/index.ts +++ b/packages/state-transition/src/signatureSets/index.ts @@ -10,7 +10,6 @@ import {getBlockProposerSignatureSet} from "./proposer.js"; import {getRandaoRevealSignatureSet} from "./randao.js"; import {getVoluntaryExitsSignatureSets} from "./voluntaryExits.js"; import {getBlsToExecutionChangeSignatureSets} from "./blsToExecutionChange.js"; -import {getConsolidationSignatureSets} from "./consolidation.js"; export * from "./attesterSlashings.js"; export * from "./indexedAttestation.js"; @@ -19,7 +18,6 @@ export * from "./proposerSlashings.js"; export * from "./randao.js"; export * from "./voluntaryExits.js"; export * from "./blsToExecutionChange.js"; -export * from "./consolidation.js"; /** * Includes all signatures on the block (except the deposit signatures) for verification. @@ -71,15 +69,5 @@ export function getBlockSignatureSets( } } - if (fork >= ForkSeq.electra) { - const consolidationSignatureSets = getConsolidationSignatureSets( - state as CachedBeaconStateElectra, - signedBlock as electra.SignedBeaconBlock - ); - if (consolidationSignatureSets.length > 0) { - signatureSets.push(...consolidationSignatureSets); - } - } - return signatureSets; } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 3d77366cbe57..7031aaa76fce 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -52,6 +52,7 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache ...stateElectraCloned.latestExecutionPayloadHeader.toValue(), depositRequestsRoot: ssz.Root.defaultValue(), withdrawalRequestsRoot: ssz.Root.defaultValue(), + consolidationRequestsRoot: ssz.Root.defaultValue(), }); stateElectraView.nextWithdrawalIndex = stateDeneb.nextWithdrawalIndex; stateElectraView.nextWithdrawalValidatorIndex = stateDeneb.nextWithdrawalValidatorIndex; diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 2c55dc61a502..2f9d5ae0f6d7 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -175,9 +175,7 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositRequestsRoot = ssz.electra.DepositRequests.hashTreeRoot((payload as electra.ExecutionPayload).depositRequests); (bellatrixPayloadFields as electra.ExecutionPayloadHeader).withdrawalRequestsRoot = - ssz.electra.ExecutionLayerWithdrawalRequests.hashTreeRoot( - (payload as electra.ExecutionPayload).withdrawalRequests - ); + ssz.electra.WithdrawalRequests.hashTreeRoot((payload as electra.ExecutionPayload).withdrawalRequests); } return bellatrixPayloadFields; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index ab9cbbaacd80..f02a1a27aefe 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -17,7 +17,7 @@ import { MAX_ATTESTATIONS_ELECTRA, MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, - MAX_CONSOLIDATIONS, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, PENDING_BALANCE_DEPOSITS_LIMIT, PENDING_PARTIAL_WITHDRAWALS_LIMIT, PENDING_CONSOLIDATIONS_LIMIT, @@ -125,24 +125,34 @@ export const DepositRequest = new ContainerType( export const DepositRequests = new ListCompositeType(DepositRequest, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); -export const ExecutionLayerWithdrawalRequest = new ContainerType( +export const WithdrawalRequest = new ContainerType( { sourceAddress: ExecutionAddress, validatorPubkey: BLSPubkey, amount: UintNum64, }, - {typeName: "ExecutionLayerWithdrawalRequest", jsonCase: "eth2"} + {typeName: "WithdrawalRequest", jsonCase: "eth2"} ); -export const ExecutionLayerWithdrawalRequests = new ListCompositeType( - ExecutionLayerWithdrawalRequest, - MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD +export const WithdrawalRequests = new ListCompositeType(WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD); +export const ConsolidationRequest = new ContainerType( + { + sourceAddress: ExecutionAddress, + sourcePubkey: BLSPubkey, + targetPubkey: BLSPubkey, + }, + {typeName: "ConsolidationRequest", jsonCase: "eth2"} +); +export const ConsolidationRequests = new ListCompositeType( + ConsolidationRequest, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD ); export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, depositRequests: DepositRequests, // New in ELECTRA - withdrawalRequests: ExecutionLayerWithdrawalRequests, // New in ELECTRA + withdrawalRequests: WithdrawalRequests, // New in ELECTRA + consolidationRequests: ConsolidationRequests, // [New in Electra] }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -152,27 +162,11 @@ export const ExecutionPayloadHeader = new ContainerType( ...denebSsz.ExecutionPayloadHeader.fields, depositRequestsRoot: Root, // New in ELECTRA withdrawalRequestsRoot: Root, // New in ELECTRA + consolidationRequestsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); -export const Consolidation = new ContainerType( - { - sourceIndex: ValidatorIndex, - targetIndex: ValidatorIndex, - epoch: Epoch, - }, - {typeName: "Consolidation", jsonCase: "eth2"} -); - -export const SignedConsolidation = new ContainerType( - { - message: Consolidation, - signature: BLSSignature, - }, - {typeName: "SignedConsolidation", jsonCase: "eth2"} -); - // We have to preserve Fields ordering while changing the type of ExecutionPayload export const BeaconBlockBody = new ContainerType( { @@ -188,7 +182,6 @@ export const BeaconBlockBody = new ContainerType( executionPayload: ExecutionPayload, // Modified in ELECTRA blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, - consolidations: new ListCompositeType(SignedConsolidation, MAX_CONSOLIDATIONS), // [New in Electra] }, {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -223,7 +216,6 @@ export const BlindedBeaconBlockBody = new ContainerType( executionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, - consolidations: new ListCompositeType(SignedConsolidation, MAX_CONSOLIDATIONS), // [New in Electra] }, {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 994e5a732243..728ffc56ac83 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -12,8 +12,11 @@ export type SignedAggregateAndProof = ValueOf; export type DepositRequests = ValueOf; -export type ExecutionLayerWithdrawalRequest = ValueOf; -export type ExecutionLayerWithdrawalRequests = ValueOf; +export type WithdrawalRequest = ValueOf; +export type WithdrawalRequests = ValueOf; + +export type ConsolidationRequest = ValueOf; +export type ConsolidationRequests = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; @@ -43,9 +46,6 @@ export type LightClientFinalityUpdate = ValueOf; export type LightClientStore = ValueOf; -export type Consolidation = ValueOf; -export type SignedConsolidation = ValueOf; - export type PendingBalanceDeposit = ValueOf; export type PendingPartialWithdrawal = ValueOf; export type PendingConsolidation = ValueOf; diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 3a497555361a..d5369515d47b 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -234,7 +234,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Tue, 25 Jun 2024 12:48:06 +0300 Subject: [PATCH 074/145] feat: support electra devnet-1 (#6892) * Update spec test version * Use new max effective balance * Relax loop breaking condition for `computeProposerIndex` * fix remaining * check-types & lint * Skip invalid test * Fix rebase + lint * Remove early return statement in `computeProposers` * Address comment * Address comment --- .../chain/produceBlock/produceBlockBody.ts | 1 - .../spec/presets/epoch_processing.test.ts | 6 +++++- .../test/spec/presets/operations.test.ts | 6 +++++- .../test/spec/specTestVersioning.ts | 2 +- .../test/spec/utils/specTestIterator.ts | 3 ++- packages/config/src/forkConfig/index.ts | 8 +++++++- packages/config/src/forkConfig/types.ts | 5 ++++- .../src/block/processConsolidationRequest.ts | 7 +++---- .../src/block/processOperations.ts | 2 +- .../state-transition/src/cache/epochCache.ts | 15 +++++++++++++-- packages/state-transition/src/epoch/index.ts | 2 +- .../epoch/processPendingBalanceDeposits.ts | 2 +- .../src/epoch/processSyncCommitteeUpdates.ts | 5 +++-- .../src/signatureSets/index.ts | 4 ++-- .../src/slot/upgradeStateToAltair.ts | 1 + .../state-transition/src/util/execution.ts | 2 ++ packages/state-transition/src/util/genesis.ts | 8 ++++++-- packages/state-transition/src/util/seed.ts | 19 ++++++++++++++----- .../src/util/syncCommittee.ts | 4 +++- .../test/perf/epoch/epochAltair.test.ts | 2 +- .../epoch/processSyncCommitteeUpdates.test.ts | 4 ++-- packages/state-transition/test/perf/util.ts | 8 +++++++- .../test/perf/util/shufflings.test.ts | 5 ++++- 23 files changed, 88 insertions(+), 33 deletions(-) diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 638df42cba1b..ba560d5a7ff0 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -17,7 +17,6 @@ import { BlindedBeaconBlockBody, BlindedBeaconBlock, sszTypesFor, - electra, } from "@lodestar/types"; import { CachedBeaconStateAllForks, diff --git a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts index f159612e416c..604243400aa0 100644 --- a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts +++ b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts @@ -5,6 +5,7 @@ import { EpochTransitionCache, BeaconStateAllForks, beforeProcessEpoch, + CachedBeaconStateAltair, } from "@lodestar/state-transition"; import * as epochFns from "@lodestar/state-transition/epoch"; import {ssz} from "@lodestar/types"; @@ -40,7 +41,10 @@ const epochTransitionFns: Record = { rewards_and_penalties: epochFns.processRewardsAndPenalties, slashings: epochFns.processSlashings, slashings_reset: epochFns.processSlashingsReset, - sync_committee_updates: epochFns.processSyncCommitteeUpdates as EpochTransitionFn, + sync_committee_updates: (state, _) => { + const fork = state.config.getForkSeq(state.slot); + epochFns.processSyncCommitteeUpdates(fork, state as CachedBeaconStateAltair); + }, historical_summaries_update: epochFns.processHistoricalSummariesUpdate as EpochTransitionFn, pending_balance_deposits: epochFns.processPendingBalanceDeposits as EpochTransitionFn, pending_consolidations: epochFns.processPendingConsolidations as EpochTransitionFn, diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 47895540ac1a..1128cdd11ba9 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -96,10 +96,13 @@ const operationFns: Record> = blockFns.processWithdrawalRequest(ForkSeq.electra, state as CachedBeaconStateElectra, testCase.withdrawal_request); }, + deposit_request: (state, testCase: {deposit_request: electra.DepositRequest}) => { + blockFns.processDepositRequest(ForkSeq.electra, state as CachedBeaconStateElectra, testCase.deposit_request); + }, + consolidation_request: (state, testCase: {consolidation_request: electra.ConsolidationRequest}) => { blockFns.processConsolidationRequest(state as CachedBeaconStateElectra, testCase.consolidation_request); }, - }; export type BlockProcessFn = (state: T, testCase: any) => void; @@ -151,6 +154,7 @@ const operations: TestRunnerFn = (fork, address_change: ssz.capella.SignedBLSToExecutionChange, // Electra withdrawal_request: ssz.electra.WithdrawalRequest, + deposit_request: ssz.electra.DepositRequest, consolidation_request: ssz.electra.ConsolidationRequest, }, shouldError: (testCase) => testCase.post === undefined, diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index 9195532379a0..e97770e8146c 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.5.0-alpha.2", + specVersion: "v1.5.0-alpha.3", // Target directory is the host package root: 'packages/*/spec-tests' outputDir: path.join(__dirname, "../../spec-tests"), specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests", diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 63fe0cc10442..ac0d70307d33 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -66,7 +66,8 @@ export const defaultSkipOpts: SkipOpts = { /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, ], - skippedTests: [], + // TODO Electra: Review this test in the next spec test release + skippedTests: [/incorrect_not_enough_consolidation_churn_available/], skippedRunners: ["merkle_proof", "networking"], }; diff --git a/packages/config/src/forkConfig/index.ts b/packages/config/src/forkConfig/index.ts index 54edc8b618e0..dd2a1773c390 100644 --- a/packages/config/src/forkConfig/index.ts +++ b/packages/config/src/forkConfig/index.ts @@ -11,7 +11,7 @@ import { ForkLightClient, ForkBlobs, } from "@lodestar/params"; -import {Slot, Version, SSZTypesFor, sszTypesFor} from "@lodestar/types"; +import {Slot, Version, SSZTypesFor, sszTypesFor, Epoch} from "@lodestar/types"; import {ChainConfig} from "../chainConfig/index.js"; import {ForkConfig, ForkInfo} from "./types.js"; @@ -84,6 +84,9 @@ export function createForkConfig(config: ChainConfig): ForkConfig { // Fork convenience methods getForkInfo(slot: Slot): ForkInfo { const epoch = Math.floor(Math.max(slot, 0) / SLOTS_PER_EPOCH); + return this.getForkInfoFromEpoch(epoch); + }, + getForkInfoFromEpoch(epoch: Epoch): ForkInfo { // NOTE: forks must be sorted by descending epoch, latest fork first for (const fork of forksDescendingEpochOrder) { if (epoch >= fork.epoch) return fork; @@ -96,6 +99,9 @@ export function createForkConfig(config: ChainConfig): ForkConfig { getForkSeq(slot: Slot): ForkSeq { return this.getForkInfo(slot).seq; }, + getForkSeqFromEpoch(epoch: Epoch): ForkSeq { + return this.getForkInfoFromEpoch(epoch).seq; + }, getForkVersion(slot: Slot): Version { return this.getForkInfo(slot).version; }, diff --git a/packages/config/src/forkConfig/types.ts b/packages/config/src/forkConfig/types.ts index 2905e6f03c34..dd0381f4e9bf 100644 --- a/packages/config/src/forkConfig/types.ts +++ b/packages/config/src/forkConfig/types.ts @@ -21,11 +21,14 @@ export type ForkConfig = { /** Get the hard-fork info for the active fork at `slot` */ getForkInfo(slot: Slot): ForkInfo; - + /** Get the hard-fork info for the active fork at `epoch` */ + getForkInfoFromEpoch(epoch: Epoch): ForkInfo; /** Get the hard-fork name at a given slot */ getForkName(slot: Slot): ForkName; /** Get the hard-fork sequence number at a given slot */ getForkSeq(slot: Slot): ForkSeq; + /** Get the hard-fork sequence number at a given epoch */ + getForkSeqFromEpoch(epoch: Epoch): ForkSeq; /** Get the hard-fork version at a given slot */ getForkVersion(slot: Slot): Version; /** Get SSZ types by hard-fork */ diff --git a/packages/state-transition/src/block/processConsolidationRequest.ts b/packages/state-transition/src/block/processConsolidationRequest.ts index 4b6a65e036ad..2f12065fde41 100644 --- a/packages/state-transition/src/block/processConsolidationRequest.ts +++ b/packages/state-transition/src/block/processConsolidationRequest.ts @@ -10,7 +10,6 @@ export function processConsolidationRequest( state: CachedBeaconStateElectra, consolidationRequest: electra.ConsolidationRequest ): void { - // If the pending consolidations queue is full, consolidation requests are ignored if (state.pendingConsolidations.length >= PENDING_CONSOLIDATIONS_LIMIT) { return; @@ -30,11 +29,11 @@ export function processConsolidationRequest( } // Verify that source != target, so a consolidation cannot be used as an exit. - if (sourceIndex === targetIndex){ + if (sourceIndex === targetIndex) { return; } - const sourceValidator = state.validators.getReadonly(sourceIndex); + const sourceValidator = state.validators.get(sourceIndex); const targetValidator = state.validators.getReadonly(targetIndex); const sourceWithdrawalAddress = sourceValidator.withdrawalCredentials.subarray(12); const currentEpoch = state.epochCtx.epoch; @@ -71,4 +70,4 @@ export function processConsolidationRequest( targetIndex, }); state.pendingConsolidations.push(pendingConsolidation); -} \ No newline at end of file +} diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 8290771d2d8e..6d4fb25b9f5b 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -67,7 +67,7 @@ export function processOperations( const stateElectra = state as CachedBeaconStateElectra; const bodyElectra = body as electra.BeaconBlockBody; - for (const depositRequest of bodyElectra.executionPayload.depositReceipts) { + for (const depositRequest of bodyElectra.executionPayload.depositRequests) { processDepositRequest(fork, stateElectra, depositRequest); } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index dbcacec0062d..1f95da1ad5fc 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -398,7 +398,12 @@ export class EpochCache { // Allow to create CachedBeaconState for empty states, or no active validators const proposers = currentShuffling.activeIndices.length > 0 - ? computeProposers(currentProposerSeed, currentShuffling, effectiveBalanceIncrements) + ? computeProposers( + config.getForkSeqFromEpoch(currentEpoch), + currentProposerSeed, + currentShuffling, + effectiveBalanceIncrements + ) : []; const proposersNextEpoch: ProposersDeferred = { @@ -577,7 +582,12 @@ export class EpochCache { this.proposersPrevEpoch = this.proposers; const currentProposerSeed = getSeed(state, this.currentShuffling.epoch, DOMAIN_BEACON_PROPOSER); - this.proposers = computeProposers(currentProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements); + this.proposers = computeProposers( + this.config.getForkSeqFromEpoch(currEpoch), + currentProposerSeed, + this.currentShuffling, + this.effectiveBalanceIncrements + ); // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. this.proposersNextEpoch = {computed: false, seed: getSeed(state, this.nextShuffling.epoch, DOMAIN_BEACON_PROPOSER)}; @@ -774,6 +784,7 @@ export class EpochCache { getBeaconProposersNextEpoch(): ValidatorIndex[] { if (!this.proposersNextEpoch.computed) { const indexes = computeProposers( + this.config.getForkSeqFromEpoch(this.epoch + 1), this.proposersNextEpoch.seed, this.nextShuffling, this.effectiveBalanceIncrements diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index 6e736fdae2cc..21455521897b 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -178,7 +178,7 @@ export function processEpoch( const timer = metrics?.epochTransitionStepTime.startTimer({ step: EpochTransitionStep.processSyncCommitteeUpdates, }); - processSyncCommitteeUpdates(state as CachedBeaconStateAltair); + processSyncCommitteeUpdates(fork, state as CachedBeaconStateAltair); timer?.(); } } diff --git a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts index 112832c920d7..e6e43bbaa089 100644 --- a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts @@ -10,7 +10,7 @@ import {getCurrentEpoch} from "../util/epoch.js"; * For each eligible `deposit`, call `increaseBalance()`. * Remove the processed deposits from `state.pendingBalanceDeposits`. * Update `state.depositBalanceToConsume` for the next epoch - * + * * TODO Electra: Update ssz library to support batch push to `pendingBalanceDeposits` */ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): void { diff --git a/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts b/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts index b3fd9b45053c..aa22bb4f7bcf 100644 --- a/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts +++ b/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts @@ -1,5 +1,5 @@ import {aggregateSerializedPublicKeys} from "@chainsafe/blst"; -import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@lodestar/params"; +import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, ForkSeq} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {getNextSyncCommitteeIndices} from "../util/seed.js"; import {CachedBeaconStateAltair} from "../types.js"; @@ -10,7 +10,7 @@ import {CachedBeaconStateAltair} from "../types.js"; * PERF: Once every `EPOCHS_PER_SYNC_COMMITTEE_PERIOD`, do an expensive operation to compute the next committee. * Calculating the next sync committee has a proportional cost to $VALIDATOR_COUNT */ -export function processSyncCommitteeUpdates(state: CachedBeaconStateAltair): void { +export function processSyncCommitteeUpdates(fork: ForkSeq, state: CachedBeaconStateAltair): void { const nextEpoch = state.epochCtx.epoch + 1; if (nextEpoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD === 0) { @@ -18,6 +18,7 @@ export function processSyncCommitteeUpdates(state: CachedBeaconStateAltair): voi const {effectiveBalanceIncrements} = state.epochCtx; const nextSyncCommitteeIndices = getNextSyncCommitteeIndices( + fork, state, activeValidatorIndices, effectiveBalanceIncrements diff --git a/packages/state-transition/src/signatureSets/index.ts b/packages/state-transition/src/signatureSets/index.ts index eae1d434a121..c883bb0587f8 100644 --- a/packages/state-transition/src/signatureSets/index.ts +++ b/packages/state-transition/src/signatureSets/index.ts @@ -1,7 +1,7 @@ import {ForkSeq} from "@lodestar/params"; -import {SignedBeaconBlock, altair, capella, electra} from "@lodestar/types"; +import {SignedBeaconBlock, altair, capella} from "@lodestar/types"; import {ISignatureSet} from "../util/index.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStateElectra} from "../types.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; import {getSyncCommitteeSignatureSet} from "../block/processSyncCommittee.js"; import {getProposerSlashingsSignatureSets} from "./proposerSlashings.js"; import {getAttesterSlashingsSignatureSets} from "./attesterSlashings.js"; diff --git a/packages/state-transition/src/slot/upgradeStateToAltair.ts b/packages/state-transition/src/slot/upgradeStateToAltair.ts index 0afa43930ef0..fa7e6fbeba8e 100644 --- a/packages/state-transition/src/slot/upgradeStateToAltair.ts +++ b/packages/state-transition/src/slot/upgradeStateToAltair.ts @@ -70,6 +70,7 @@ export function upgradeStateToAltair(statePhase0: CachedBeaconStatePhase0): Cach stateAltair.inactivityScores = ssz.altair.InactivityScores.toViewDU(newZeroedArray(validatorCount)); const {syncCommittee, indices} = getNextSyncCommittee( + ForkSeq.altair, stateAltair, stateAltair.epochCtx.nextShuffling.activeIndices, stateAltair.epochCtx.effectiveBalanceIncrements diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 2f9d5ae0f6d7..06e654f9f1d2 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -176,6 +176,8 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio ssz.electra.DepositRequests.hashTreeRoot((payload as electra.ExecutionPayload).depositRequests); (bellatrixPayloadFields as electra.ExecutionPayloadHeader).withdrawalRequestsRoot = ssz.electra.WithdrawalRequests.hashTreeRoot((payload as electra.ExecutionPayload).withdrawalRequests); + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).consolidationRequestsRoot = + ssz.electra.ConsolidationRequests.hashTreeRoot((payload as electra.ExecutionPayload).consolidationRequests); } return bellatrixPayloadFields; diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index ca894fd5b4e5..d1f09a9796e1 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -18,7 +18,7 @@ import {EpochCacheImmutableData} from "../cache/epochCache.js"; import {processDeposit} from "../block/processDeposit.js"; import {increaseBalance} from "../index.js"; import {computeEpochAtSlot} from "./epoch.js"; -import {getActiveValidatorIndices} from "./validator.js"; +import {getActiveValidatorIndices, getValidatorMaxEffectiveBalance} from "./validator.js"; import {getTemporaryBlockHeader} from "./blockRoot.js"; import {newFilledArray} from "./array.js"; import {getNextSyncCommittee} from "./syncCommittee.js"; @@ -193,7 +193,10 @@ export function applyDeposits( } const balance = balancesArr[i]; - const effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); + const effectiveBalance = Math.min( + balance - (balance % EFFECTIVE_BALANCE_INCREMENT), + getValidatorMaxEffectiveBalance(validator.withdrawalCredentials) + ); validator.effectiveBalance = effectiveBalance; epochCtx.effectiveBalanceIncrementsSet(i, effectiveBalance); @@ -263,6 +266,7 @@ export function initializeBeaconStateFromEth1( if (fork >= ForkSeq.altair) { const {syncCommittee} = getNextSyncCommittee( + fork, state, activeValidatorIndices, state.epochCtx.effectiveBalanceIncrements diff --git a/packages/state-transition/src/util/seed.ts b/packages/state-transition/src/util/seed.ts index cf48fda8bec4..a5a0028d6c17 100644 --- a/packages/state-transition/src/util/seed.ts +++ b/packages/state-transition/src/util/seed.ts @@ -5,7 +5,9 @@ import { DOMAIN_SYNC_COMMITTEE, EFFECTIVE_BALANCE_INCREMENT, EPOCHS_PER_HISTORICAL_VECTOR, + ForkSeq, MAX_EFFECTIVE_BALANCE, + MAX_EFFECTIVE_BALANCE_ELECTRA, MIN_SEED_LOOKAHEAD, SHUFFLE_ROUND_COUNT, SLOTS_PER_EPOCH, @@ -20,6 +22,7 @@ import {computeEpochAtSlot} from "./epoch.js"; * Compute proposer indices for an epoch */ export function computeProposers( + fork: ForkSeq, epochSeed: Uint8Array, shuffling: {epoch: Epoch; activeIndices: ArrayLike}, effectiveBalanceIncrements: EffectiveBalanceIncrements @@ -29,6 +32,7 @@ export function computeProposers( for (let slot = startSlot; slot < startSlot + SLOTS_PER_EPOCH; slot++) { proposers.push( computeProposerIndex( + fork, effectiveBalanceIncrements, shuffling.activeIndices, digest(Buffer.concat([epochSeed, intToBytes(slot, 8)])) @@ -44,6 +48,7 @@ export function computeProposers( * SLOW CODE - 🐢 */ export function computeProposerIndex( + fork: ForkSeq, effectiveBalanceIncrements: EffectiveBalanceIncrements, indices: ArrayLike, seed: Uint8Array @@ -54,7 +59,10 @@ export function computeProposerIndex( // TODO: Inline outside this function const MAX_RANDOM_BYTE = 2 ** 8 - 1; - const MAX_EFFECTIVE_BALANCE_INCREMENT = MAX_EFFECTIVE_BALANCE / EFFECTIVE_BALANCE_INCREMENT; + const MAX_EFFECTIVE_BALANCE_INCREMENT = + fork >= ForkSeq.electra + ? MAX_EFFECTIVE_BALANCE_ELECTRA / EFFECTIVE_BALANCE_INCREMENT + : MAX_EFFECTIVE_BALANCE / EFFECTIVE_BALANCE_INCREMENT; let i = 0; /* eslint-disable-next-line no-constant-condition */ @@ -73,9 +81,6 @@ export function computeProposerIndex( return candidateIndex; } i += 1; - if (i === indices.length) { - return -1; - } } } @@ -90,13 +95,17 @@ export function computeProposerIndex( * SLOW CODE - 🐢 */ export function getNextSyncCommitteeIndices( + fork: ForkSeq, state: BeaconStateAllForks, activeValidatorIndices: ArrayLike, effectiveBalanceIncrements: EffectiveBalanceIncrements ): ValidatorIndex[] { // TODO: Bechmark if it's necessary to inline outside of this function const MAX_RANDOM_BYTE = 2 ** 8 - 1; - const MAX_EFFECTIVE_BALANCE_INCREMENT = MAX_EFFECTIVE_BALANCE / EFFECTIVE_BALANCE_INCREMENT; + const MAX_EFFECTIVE_BALANCE_INCREMENT = + fork >= ForkSeq.electra + ? MAX_EFFECTIVE_BALANCE_ELECTRA / EFFECTIVE_BALANCE_INCREMENT + : MAX_EFFECTIVE_BALANCE / EFFECTIVE_BALANCE_INCREMENT; const epoch = computeEpochAtSlot(state.slot) + 1; diff --git a/packages/state-transition/src/util/syncCommittee.ts b/packages/state-transition/src/util/syncCommittee.ts index b6de821d5406..c1f53632e521 100644 --- a/packages/state-transition/src/util/syncCommittee.ts +++ b/packages/state-transition/src/util/syncCommittee.ts @@ -2,6 +2,7 @@ import {aggregateSerializedPublicKeys} from "@chainsafe/blst"; import { BASE_REWARD_FACTOR, EFFECTIVE_BALANCE_INCREMENT, + ForkSeq, SLOTS_PER_EPOCH, SYNC_COMMITTEE_SIZE, SYNC_REWARD_WEIGHT, @@ -19,11 +20,12 @@ import {getNextSyncCommitteeIndices} from "./seed.js"; * SLOW CODE - 🐢 */ export function getNextSyncCommittee( + fork: ForkSeq, state: BeaconStateAllForks, activeValidatorIndices: ArrayLike, effectiveBalanceIncrements: EffectiveBalanceIncrements ): {indices: ValidatorIndex[]; syncCommittee: altair.SyncCommittee} { - const indices = getNextSyncCommitteeIndices(state, activeValidatorIndices, effectiveBalanceIncrements); + const indices = getNextSyncCommitteeIndices(fork, state, activeValidatorIndices, effectiveBalanceIncrements); // Using the index2pubkey cache is slower because it needs the serialized pubkey. const pubkeys = indices.map((index) => state.validators.getReadonly(index).pubkey); diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 6c43151cc137..39e0a1b4c5c3 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -172,7 +172,7 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue id: `${stateId} - altair processSyncCommitteeUpdates`, convergeFactor: 1 / 100, // Very unstable make it converge faster beforeEach: () => stateOg.value.clone() as CachedBeaconStateAltair, - fn: (state) => processSyncCommitteeUpdates(state), + fn: (state) => processSyncCommitteeUpdates(ForkSeq.altair, state), }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts b/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts index ffde30e1302c..4497dc16be0c 100644 --- a/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processSyncCommitteeUpdates.test.ts @@ -1,5 +1,5 @@ import {itBench} from "@dapplion/benchmark"; -import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@lodestar/params"; +import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, ForkSeq} from "@lodestar/params"; import {processSyncCommitteeUpdates} from "../../../src/epoch/processSyncCommitteeUpdates.js"; import {StateAltair} from "../types.js"; import {generatePerfTestCachedStateAltair, numValidators} from "../util.js"; @@ -21,7 +21,7 @@ describe("altair processSyncCommitteeUpdates", () => { }, fn: (state) => { const nextSyncCommitteeBefore = state.nextSyncCommittee; - processSyncCommitteeUpdates(state); + processSyncCommitteeUpdates(ForkSeq.altair, state); if (state.nextSyncCommittee === nextSyncCommitteeBefore) { throw Error("nextSyncCommittee instance has not changed"); } diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/test/perf/util.ts index 4b2a7da4a50e..f3c2eaef91e5 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/test/perf/util.ts @@ -7,6 +7,7 @@ import { EPOCHS_PER_ETH1_VOTING_PERIOD, EPOCHS_PER_HISTORICAL_VECTOR, ForkName, + ForkSeq, MAX_ATTESTATIONS, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH, @@ -273,7 +274,12 @@ export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): Beaco const activeValidatorIndices = getActiveValidatorIndices(altairState, epoch); const effectiveBalanceIncrements = getEffectiveBalanceIncrements(altairState); - const {syncCommittee} = getNextSyncCommittee(altairState, activeValidatorIndices, effectiveBalanceIncrements); + const {syncCommittee} = getNextSyncCommittee( + ForkSeq.altair, + altairState, + activeValidatorIndices, + effectiveBalanceIncrements + ); state.currentSyncCommittee = syncCommittee; state.nextSyncCommittee = syncCommittee; diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index 96c7878a46ac..24be96c4676d 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -28,7 +28,8 @@ describe("epoch shufflings", () => { id: `computeProposers - vc ${numValidators}`, fn: () => { const epochSeed = getSeed(state, state.epochCtx.nextShuffling.epoch, DOMAIN_BEACON_PROPOSER); - computeProposers(epochSeed, state.epochCtx.nextShuffling, state.epochCtx.effectiveBalanceIncrements); + const fork = state.config.getForkSeq(state.slot); + computeProposers(fork, epochSeed, state.epochCtx.nextShuffling, state.epochCtx.effectiveBalanceIncrements); }, }); @@ -43,7 +44,9 @@ describe("epoch shufflings", () => { itBench({ id: `getNextSyncCommittee - vc ${numValidators}`, fn: () => { + const fork = state.config.getForkSeq(state.slot); getNextSyncCommittee( + fork, state, state.epochCtx.nextShuffling.activeIndices, state.epochCtx.effectiveBalanceIncrements From 1d9c3a3a7de3ff0720cee224fdb7e4c1858da446 Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 25 Jun 2024 18:37:55 +0530 Subject: [PATCH 075/145] fix: electra rebase fixes on the new generic typing model lint & type fix lint --- packages/api/src/beacon/routes/events.ts | 11 +++--- packages/api/src/beacon/routes/validator.ts | 3 +- .../opPools/aggregatedAttestationPool.ts | 22 +++++------- .../src/chain/opPools/attestationPool.ts | 18 +++++----- .../beacon-node/src/chain/opPools/opPool.ts | 6 ++-- .../src/chain/validation/aggregateAndProof.ts | 8 ++--- .../src/chain/validation/attestation.ts | 13 +++---- .../signatureSets/aggregateAndProof.ts | 6 ++-- .../src/metrics/validatorMonitor.ts | 2 +- .../src/network/gossip/interface.ts | 12 +++---- .../beacon-node/src/network/gossip/topic.ts | 4 +-- packages/beacon-node/src/network/interface.ts | 3 +- packages/beacon-node/src/network/network.ts | 3 +- .../test/unit/util/sszBytes.test.ts | 4 +-- .../fork-choice/src/forkChoice/interface.ts | 4 +-- packages/params/src/index.ts | 2 +- packages/params/src/presets/mainnet.ts | 2 +- packages/params/src/presets/minimal.ts | 2 +- packages/params/src/types.ts | 4 +-- .../src/block/processAttestationPhase0.ts | 8 ++--- .../src/block/processAttestations.ts | 4 +-- .../src/block/processAttestationsAltair.ts | 4 +-- .../state-transition/src/cache/epochCache.ts | 7 ++-- .../src/signatureSets/attesterSlashings.ts | 6 ++-- packages/state-transition/src/util/epoch.ts | 2 +- packages/types/src/electra/sszTypes.ts | 7 ++-- packages/types/src/types.ts | 35 +++++++++++++++++++ .../validator/src/services/attestation.ts | 6 ++-- .../validator/src/services/validatorStore.ts | 11 +++--- packages/validator/src/util/params.ts | 2 +- 30 files changed, 128 insertions(+), 93 deletions(-) diff --git a/packages/api/src/beacon/routes/events.ts b/packages/api/src/beacon/routes/events.ts index 04131750dbf2..e15790fd3bea 100644 --- a/packages/api/src/beacon/routes/events.ts +++ b/packages/api/src/beacon/routes/events.ts @@ -13,6 +13,9 @@ import { LightClientOptimisticUpdate, LightClientFinalityUpdate, SSEPayloadAttributes, + Attestation, + AttesterSlashing, + sszTypesFor, } from "@lodestar/types"; import {ForkName} from "@lodestar/params"; @@ -104,10 +107,10 @@ export type EventData = { block: RootHex; executionOptimistic: boolean; }; - [EventType.attestation]: {version: ForkName; data: allForks.Attestation}; + [EventType.attestation]: {version: ForkName; data: Attestation}; [EventType.voluntaryExit]: phase0.SignedVoluntaryExit; [EventType.proposerSlashing]: phase0.ProposerSlashing; - [EventType.attesterSlashing]: {version: ForkName; data: allForks.AttesterSlashing}; + [EventType.attesterSlashing]: {version: ForkName; data: AttesterSlashing}; [EventType.blsToExecutionChange]: capella.SignedBLSToExecutionChange; [EventType.finalizedCheckpoint]: { block: RootHex; @@ -225,10 +228,10 @@ export function getTypeByEvent(): {[K in EventType]: TypeJson} { {jsonCase: "eth2"} ), - [EventType.attestation]: WithVersion((fork) => (ssz.allForks[fork] as allForks.AllForksSSZTypes).Attestation), + [EventType.attestation]: WithVersion((fork) => sszTypesFor(fork).Attestation), [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit, [EventType.proposerSlashing]: ssz.phase0.ProposerSlashing, - [EventType.attesterSlashing]: WithVersion((fork) => ssz.allForks[fork].AttesterSlashing), + [EventType.attesterSlashing]: WithVersion((fork) => sszTypesFor(fork).AttesterSlashing), [EventType.blsToExecutionChange]: ssz.capella.SignedBLSToExecutionChange, [EventType.finalizedCheckpoint]: new ContainerType( diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index ac8526947c3f..154397107d47 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -17,6 +17,7 @@ import { stringType, BeaconBlockOrContents, BlindedBeaconBlock, + Attestation, } from "@lodestar/types"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; import {fromGraffitiHex, toBoolean, toGraffitiHex} from "../../utils/serdes.js"; @@ -413,7 +414,7 @@ export type Endpoints = { committeeIndex: number; }, {query: {attestation_data_root: string; slot: number; committeeIndex: number}}, - allForks.Attestation, + Attestation, VersionMeta >; diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 31d461766947..491b5fd6f72a 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -17,9 +17,9 @@ import { ssz, ValidatorIndex, RootHex, - allForks, electra, isElectraAttestation, + Attestation, } from "@lodestar/types"; import { CachedBeaconStateAllForks, @@ -40,7 +40,7 @@ type DataRootHex = string; type CommitteeIndex = number; // for pre-electra -type AttestationWithScore = {attestation: allForks.Attestation; score: number}; +type AttestationWithScore = {attestation: Attestation; score: number}; /** * for electra, this is to consolidate aggregated attestations of the same attestation data into a single attestation to be included in block * note that this is local definition in this file and it's NOT validator consolidation @@ -118,7 +118,7 @@ export class AggregatedAttestationPool { } add( - attestation: allForks.Attestation, + attestation: Attestation, dataRootHex: RootHex, attestingIndicesCount: number, committee: Uint32Array @@ -162,11 +162,7 @@ export class AggregatedAttestationPool { this.lowestPermissibleSlot = Math.max(clockSlot - SLOTS_PER_EPOCH, 0); } - getAttestationsForBlock( - fork: ForkName, - forkChoice: IForkChoice, - state: CachedBeaconStateAllForks - ): allForks.Attestation[] { + getAttestationsForBlock(fork: ForkName, forkChoice: IForkChoice, state: CachedBeaconStateAllForks): Attestation[] { const forkSeq = ForkSeq[fork]; return forkSeq >= ForkSeq.electra ? this.getAttestationsForBlockElectra(fork, forkChoice, state) @@ -397,7 +393,7 @@ export class AggregatedAttestationPool { * Get all attestations optionally filtered by `attestation.data.slot` * @param bySlot slot to filter, `bySlot === attestation.data.slot` */ - getAll(bySlot?: Slot): allForks.Attestation[] { + getAll(bySlot?: Slot): Attestation[] { let attestationGroupsArr: Map[]; if (bySlot === undefined) { attestationGroupsArr = Array.from(this.attestationGroupByIndexByDataHexBySlot.values()).flatMap((byIndex) => @@ -409,7 +405,7 @@ export class AggregatedAttestationPool { attestationGroupsArr = Array.from(attestationGroupsByIndex.values()); } - const attestations: allForks.Attestation[] = []; + const attestations: Attestation[] = []; for (const attestationGroups of attestationGroupsArr) { for (const attestationGroup of attestationGroups.values()) { attestations.push(...attestationGroup.getAttestations()); @@ -420,12 +416,12 @@ export class AggregatedAttestationPool { } interface AttestationWithIndex { - attestation: allForks.Attestation; + attestation: Attestation; trueBitsCount: number; } type AttestationNonParticipant = { - attestation: allForks.Attestation; + attestation: Attestation; // this is <= attestingIndices.count since some attesters may be seen by the chain // this is only updated and used in removeBySeenValidators function notSeenAttesterCount: number; @@ -534,7 +530,7 @@ export class MatchingDataAttestationGroup { } /** Get attestations for API. */ - getAttestations(): allForks.Attestation[] { + getAttestations(): Attestation[] { return this.attestations.map((attestation) => attestation.attestation); } } diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index dfb3bc7348a2..8daae6b663c7 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -1,6 +1,6 @@ import {BitArray} from "@chainsafe/ssz"; import {Signature, aggregateSignatures} from "@chainsafe/blst"; -import {Slot, RootHex, allForks, isElectraAttestation} from "@lodestar/types"; +import {Slot, RootHex, isElectraAttestation, Attestation} from "@lodestar/types"; import {MapDef, assert} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; @@ -23,7 +23,7 @@ const SLOTS_RETAINED = 3; const MAX_ATTESTATIONS_PER_SLOT = 16_384; type AggregateFastPhase0 = { - data: allForks.Attestation["data"]; + data: Attestation["data"]; aggregationBits: BitArray; signature: Signature; }; @@ -101,7 +101,7 @@ export class AttestationPool { * - Valid committeeIndex * - Valid data */ - add(committeeIndex: CommitteeIndex, attestation: allForks.Attestation, attDataRootHex: RootHex): InsertOutcome { + add(committeeIndex: CommitteeIndex, attestation: Attestation, attDataRootHex: RootHex): InsertOutcome { const slot = attestation.data.slot; const lowestPermissibleSlot = this.lowestPermissibleSlot; @@ -144,7 +144,7 @@ export class AttestationPool { /** * For validator API to get an aggregate */ - getAggregate(slot: Slot, committeeIndex: CommitteeIndex, dataRootHex: RootHex): allForks.Attestation | null { + getAggregate(slot: Slot, committeeIndex: CommitteeIndex, dataRootHex: RootHex): Attestation | null { const aggregate = this.aggregateByIndexByRootBySlot.get(slot)?.get(dataRootHex)?.get(committeeIndex); if (!aggregate) { // TODO: Add metric for missing aggregates @@ -168,8 +168,8 @@ export class AttestationPool { * Get all attestations optionally filtered by `attestation.data.slot` * @param bySlot slot to filter, `bySlot === attestation.data.slot` */ - getAll(bySlot?: Slot): allForks.Attestation[] { - const attestations: allForks.Attestation[] = []; + getAll(bySlot?: Slot): Attestation[] { + const attestations: Attestation[] = []; const aggregateByRoots = bySlot === undefined @@ -196,7 +196,7 @@ export class AttestationPool { /** * Aggregate a new attestation into `aggregate` mutating it */ -function aggregateAttestationInto(aggregate: AggregateFast, attestation: allForks.Attestation): InsertOutcome { +function aggregateAttestationInto(aggregate: AggregateFast, attestation: Attestation): InsertOutcome { const bitIndex = attestation.aggregationBits.getSingleTrueBit(); // Should never happen, attestations are verified against this exact condition before @@ -214,7 +214,7 @@ function aggregateAttestationInto(aggregate: AggregateFast, attestation: allFork /** * Format `contribution` into an efficient `aggregate` to add more contributions in with aggregateContributionInto() */ -function attestationToAggregate(attestation: allForks.Attestation): AggregateFast { +function attestationToAggregate(attestation: Attestation): AggregateFast { if (isElectraAttestation(attestation)) { return { data: attestation.data, @@ -235,6 +235,6 @@ function attestationToAggregate(attestation: allForks.Attestation): AggregateFas /** * Unwrap AggregateFast to phase0.Attestation */ -function fastToAttestation(aggFast: AggregateFast): allForks.Attestation { +function fastToAttestation(aggFast: AggregateFast): Attestation { return {...aggFast, signature: aggFast.signature.toBytes()}; } diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index 8eb3cebcffdb..54fec380de86 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -16,8 +16,8 @@ import { ForkSeq, MAX_ATTESTER_SLASHINGS_ELECTRA, } from "@lodestar/params"; -import {Epoch, phase0, capella, ssz, ValidatorIndex, SignedBeaconBlock} from "@lodestar/types"; import {toRootHex} from "@lodestar/utils"; +import {Epoch, phase0, capella, ssz, ValidatorIndex, SignedBeaconBlock, AttesterSlashing} from "@lodestar/types"; import {IBeaconDb} from "../../db/index.js"; import {SignedBLSToExecutionChangeVersioned} from "../../util/types.js"; import {BlockType} from "../interface.js"; @@ -175,7 +175,7 @@ export class OpPool { blockType: BlockType, metrics: Metrics | null ): [ - allForks.AttesterSlashing[], + AttesterSlashing[], phase0.ProposerSlashing[], phase0.SignedVoluntaryExit[], capella.SignedBLSToExecutionChange[], @@ -209,7 +209,7 @@ export class OpPool { }); const endAttesterSlashings = stepsMetrics?.startTimer(); - const attesterSlashings: allForks.AttesterSlashing[] = []; + const attesterSlashings: AttesterSlashing[] = []; const maxAttesterSlashing = stateFork >= ForkSeq.electra ? MAX_ATTESTER_SLASHINGS_ELECTRA : MAX_ATTESTER_SLASHINGS; attesterSlashing: for (const attesterSlashing of this.attesterSlashings.values()) { /** Indices slashable in this attester slashing */ diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 3c8b102eb5ac..839f7850f75d 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -1,5 +1,5 @@ import {ForkName, ForkSeq} from "@lodestar/params"; -import {allForks, electra, phase0, RootHex, ssz} from "@lodestar/types"; +import {electra, phase0, RootHex, ssz, IndexedAttestation, SignedAggregateAndProof} from "@lodestar/types"; import { computeEpochAtSlot, isAggregatorFromCommitteeLength, @@ -20,7 +20,7 @@ import { } from "./attestation.js"; export type AggregateAndProofValidationResult = { - indexedAttestation: allForks.IndexedAttestation; + indexedAttestation: IndexedAttestation; committeeIndices: Uint32Array; attDataRootHex: RootHex; }; @@ -41,7 +41,7 @@ export async function validateApiAggregateAndProof( export async function validateGossipAggregateAndProof( fork: ForkName, chain: IBeaconChain, - signedAggregateAndProof: allForks.SignedAggregateAndProof, + signedAggregateAndProof: SignedAggregateAndProof, serializedData: Uint8Array ): Promise { return validateAggregateAndProof(fork, chain, signedAggregateAndProof, serializedData); @@ -50,7 +50,7 @@ export async function validateGossipAggregateAndProof( async function validateAggregateAndProof( fork: ForkName, chain: IBeaconChain, - signedAggregateAndProof: allForks.SignedAggregateAndProof, + signedAggregateAndProof: SignedAggregateAndProof, serializedData: Uint8Array | null = null, opts: {skipValidationKnownAttesters: boolean; prioritizeBls: boolean} = { skipValidationKnownAttesters: false, diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 1d5568fbc868..1dcf76cdc5ae 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -6,10 +6,11 @@ import { Slot, RootHex, ssz, - allForks, electra, isElectraAttestation, CommitteeIndex, + Attestation, + IndexedAttestation, } from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq, DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; @@ -46,8 +47,8 @@ export type BatchResult = { }; export type AttestationValidationResult = { - attestation: allForks.Attestation; - indexedAttestation: allForks.IndexedAttestation; + attestation: Attestation; + indexedAttestation: IndexedAttestation; subnet: number; attDataRootHex: RootHex; committeeIndex: CommitteeIndex; @@ -56,7 +57,7 @@ export type AttestationValidationResult = { export type AttestationOrBytes = ApiAttestation | GossipAttestation; /** attestation from api */ -export type ApiAttestation = {attestation: allForks.Attestation; serializedData: null}; +export type ApiAttestation = {attestation: Attestation; serializedData: null}; /** attestation from gossip */ export type GossipAttestation = { @@ -263,7 +264,7 @@ async function validateGossipAttestationNoSignatureCheck( // Run the checks that happen before an indexed attestation is constructed. let attestationOrCache: - | {attestation: allForks.Attestation; cache: null} + | {attestation: Attestation; cache: null} | {attestation: null; cache: AttestationDataCacheEntry; serializedData: Uint8Array}; let attDataKey: SeenAttDataKey | null = null; if (attestationOrBytes.serializedData) { @@ -513,7 +514,7 @@ async function validateGossipAttestationNoSignatureCheck( ? (indexedAttestationContent as electra.IndexedAttestation) : (indexedAttestationContent as phase0.IndexedAttestation); - const attestation: allForks.Attestation = attestationOrCache.attestation ?? { + const attestation: Attestation = attestationOrCache.attestation ?? { aggregationBits, data: attData, committeeBits, diff --git a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts index 411f33f68e66..2945b4d445d0 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -1,6 +1,6 @@ import type {PublicKey} from "@chainsafe/bls/types"; import {DOMAIN_AGGREGATE_AND_PROOF, ForkSeq} from "@lodestar/params"; -import {allForks, ssz} from "@lodestar/types"; +import {ssz, SignedAggregateAndProof} from "@lodestar/types"; import {Epoch} from "@lodestar/types"; import { computeSigningRoot, @@ -13,7 +13,7 @@ import {BeaconConfig} from "@lodestar/config"; export function getAggregateAndProofSigningRoot( config: BeaconConfig, epoch: Epoch, - aggregateAndProof: allForks.SignedAggregateAndProof + aggregateAndProof: SignedAggregateAndProof ): Uint8Array { // previously, we call `const aggregatorDomain = state.config.getDomain(state.slot, DOMAIN_AGGREGATE_AND_PROOF, slot);` // at fork boundary, it's required to dial to target epoch https://github.com/ChainSafe/lodestar/blob/v1.11.3/packages/beacon-node/src/chain/validation/attestation.ts#L573 @@ -29,7 +29,7 @@ export function getAggregateAndProofSignatureSet( config: BeaconConfig, epoch: Epoch, aggregator: PublicKey, - aggregateAndProof: allForks.SignedAggregateAndProof + aggregateAndProof: SignedAggregateAndProof ): ISignatureSet { return createSingleSignatureSetFromComponents( aggregator, diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index 6331c5dca550..34dfa6b72e03 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -13,7 +13,7 @@ import {BeaconBlock, RootHex, altair, deneb} from "@lodestar/types"; import {ChainConfig, ChainForkConfig} from "@lodestar/config"; import {ForkSeq, INTERVALS_PER_SLOT, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, Slot, ValidatorIndex} from "@lodestar/types"; -import {IndexedAttestation, SignedAggregateAndProof} from "@lodestar/types/allForks"; +import {IndexedAttestation, SignedAggregateAndProof} from "@lodestar/types"; import {GENESIS_SLOT} from "../constants/constants.js"; import {LodestarMetrics} from "./metrics/lodestar.js"; diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index 649bfd455387..ab9b8a65978d 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -11,6 +11,8 @@ import { phase0, SignedBeaconBlock, Slot, + Attestation, + SignedAggregateAndProof, } from "@lodestar/types"; import {BeaconConfig} from "@lodestar/config"; import {Logger} from "@lodestar/utils"; @@ -80,8 +82,8 @@ export type SSZTypeOfGossipTopic = T extends {type: infer export type GossipTypeMap = { [GossipType.beacon_block]: SignedBeaconBlock; [GossipType.blob_sidecar]: deneb.BlobSidecar; - [GossipType.beacon_aggregate_and_proof]: allForks.SignedAggregateAndProof; - [GossipType.beacon_attestation]: allForks.Attestation; + [GossipType.beacon_aggregate_and_proof]: SignedAggregateAndProof; + [GossipType.beacon_attestation]: Attestation; [GossipType.voluntary_exit]: phase0.SignedVoluntaryExit; [GossipType.proposer_slashing]: phase0.ProposerSlashing; [GossipType.attester_slashing]: phase0.AttesterSlashing; @@ -95,10 +97,8 @@ export type GossipTypeMap = { export type GossipFnByType = { [GossipType.beacon_block]: (signedBlock: SignedBeaconBlock) => Promise | void; [GossipType.blob_sidecar]: (blobSidecar: deneb.BlobSidecar) => Promise | void; - [GossipType.beacon_aggregate_and_proof]: ( - aggregateAndProof: allForks.SignedAggregateAndProof - ) => Promise | void; - [GossipType.beacon_attestation]: (attestation: allForks.Attestation) => Promise | void; + [GossipType.beacon_aggregate_and_proof]: (aggregateAndProof: SignedAggregateAndProof) => Promise | void; + [GossipType.beacon_attestation]: (attestation: Attestation) => Promise | void; [GossipType.voluntary_exit]: (voluntaryExit: phase0.SignedVoluntaryExit) => Promise | void; [GossipType.proposer_slashing]: (proposerSlashing: phase0.ProposerSlashing) => Promise | void; [GossipType.attester_slashing]: (attesterSlashing: phase0.AttesterSlashing) => Promise | void; diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index ffbad28cd936..2ccde0044037 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -1,4 +1,4 @@ -import {ssz, sszTypesFor} from "@lodestar/types"; +import {ssz, Attestation, sszTypesFor} from "@lodestar/types"; import {ForkDigestContext} from "@lodestar/config"; import { ATTESTATION_SUBNET_COUNT, @@ -128,7 +128,7 @@ export function sszDeserialize(topic: T, serializedData: /** * Deserialize a gossip serialized data into an Attestation object. */ -export function sszDeserializeAttestation(fork: ForkName, serializedData: Uint8Array): allForks.Attestation { +export function sszDeserializeAttestation(fork: ForkName, serializedData: Uint8Array): Attestation { const sszType = ssz.allForks[fork].Attestation; try { return sszType.deserialize(serializedData); diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 4fd1235fe91a..8d73379af221 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -26,6 +26,7 @@ import { capella, deneb, phase0, + SignedAggregateAndProof, } from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; import {INetworkEventBus} from "./events.js"; @@ -71,7 +72,7 @@ export interface INetwork extends INetworkCorePublic { // Gossip publishBeaconBlock(signedBlock: SignedBeaconBlock): Promise; publishBlobSidecar(blobSidecar: deneb.BlobSidecar): Promise; - publishBeaconAggregateAndProof(aggregateAndProof: allForks.SignedAggregateAndProof): Promise; + publishBeaconAggregateAndProof(aggregateAndProof: SignedAggregateAndProof): Promise; publishBeaconAttestation(attestation: phase0.Attestation, subnet: number): Promise; publishVoluntaryExit(voluntaryExit: phase0.SignedVoluntaryExit): Promise; publishBlsToExecutionChange(blsToExecutionChange: capella.SignedBLSToExecutionChange): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index f9550d4c0ec2..1b3ccaaaf75a 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -17,6 +17,7 @@ import { LightClientFinalityUpdate, LightClientOptimisticUpdate, LightClientUpdate, + SignedAggregateAndProof, } from "@lodestar/types"; import {routes} from "@lodestar/api"; import {ResponseIncoming} from "@lodestar/reqresp"; @@ -316,7 +317,7 @@ export class Network implements INetwork { }); } - async publishBeaconAggregateAndProof(aggregateAndProof: allForks.SignedAggregateAndProof): Promise { + async publishBeaconAggregateAndProof(aggregateAndProof: SignedAggregateAndProof): Promise { const fork = this.config.getForkName(aggregateAndProof.message.aggregate.data.slot); return this.publishGossip( {type: GossipType.beacon_aggregate_and_proof, fork}, diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 2d138111a05f..643b6b1bac0f 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect} from "vitest"; import {BitArray} from "@chainsafe/ssz"; -import {allForks, deneb, electra, Epoch, isElectraAttestation, phase0, RootHex, Slot, ssz} from "@lodestar/types"; +import {deneb, electra, Epoch, isElectraAttestation, phase0, RootHex, Slot, ssz} from "@lodestar/types"; import {fromHex, toHex} from "@lodestar/utils"; import {ForkName, MAX_COMMITTEES_PER_SLOT} from "@lodestar/params"; import { @@ -19,7 +19,7 @@ import { } from "../../../src/util/sszBytes.js"; describe("attestation SSZ serialized picking", () => { - const testCases: allForks.Attestation[] = [ + const testCases: (phase0.Attestation | electra.Attestation)[] = [ ssz.phase0.Attestation.defaultValue(), attestationFromValues( 4_000_000, diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index c9d1aa627b03..0b6d56a88bf2 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -1,6 +1,6 @@ import {EffectiveBalanceIncrements} from "@lodestar/state-transition"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; -import {Epoch, Slot, ValidatorIndex, phase0, Root, RootHex, BeaconBlock} from "@lodestar/types"; +import {Epoch, Slot, ValidatorIndex, phase0, Root, RootHex, BeaconBlock, IndexedAttestation} from "@lodestar/types"; import { ProtoBlock, MaybeValidExecutionStatus, @@ -156,7 +156,7 @@ export interface IForkChoice { * The supplied `attestation` **must** pass the `in_valid_indexed_attestation` function as it * will not be run here. */ - onAttestation(attestation: allForks.IndexedAttestation, attDataRoot: string, forceImport?: boolean): void; + onAttestation(attestation: IndexedAttestation, attDataRoot: string, forceImport?: boolean): void; /** * Register attester slashing in order not to consider their votes in `getHead` * diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index a3dfb65da350..0c2e3b34794b 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -100,7 +100,7 @@ export const { PENDING_PARTIAL_WITHDRAWALS_LIMIT, PENDING_CONSOLIDATIONS_LIMIT, MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA, - MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, + MAX_CONSOLIDATIONS, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 072936492bd3..2495f7ef97a1 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -133,6 +133,6 @@ export const mainnetPreset: BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728, PENDING_CONSOLIDATIONS_LIMIT: 262144, - MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1, + MAX_CONSOLIDATIONS: 1, WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 4a1e4af50665..8e71407965d7 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -134,6 +134,6 @@ export const minimalPreset: BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 64, PENDING_CONSOLIDATIONS_LIMIT: 64, - MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1, + MAX_CONSOLIDATIONS: 1, WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index 14e8f8c172d8..dffd98518006 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -95,7 +95,7 @@ export type BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: number; PENDING_PARTIAL_WITHDRAWALS_LIMIT: number; PENDING_CONSOLIDATIONS_LIMIT: number; - MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: number; + MAX_CONSOLIDATIONS: number; WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: number; }; @@ -195,7 +195,7 @@ export const beaconPresetTypes: BeaconPresetTypes = { PENDING_BALANCE_DEPOSITS_LIMIT: "number", PENDING_PARTIAL_WITHDRAWALS_LIMIT: "number", PENDING_CONSOLIDATIONS_LIMIT: "number", - MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: "number", + MAX_CONSOLIDATIONS: "number", WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: "number", }; diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index 3bb5b71fa267..ab21841069ec 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -1,5 +1,5 @@ import {toRootHex} from "@lodestar/utils"; -import {Slot, allForks, electra, phase0, ssz} from "@lodestar/types"; +import {Slot, Attestation, electra, phase0, ssz} from "@lodestar/types"; import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, ForkSeq} from "@lodestar/params"; import {assert} from "@lodestar/utils"; import {computeEpochAtSlot} from "../util/index.js"; @@ -56,11 +56,7 @@ export function processAttestationPhase0( } } -export function validateAttestation( - fork: ForkSeq, - state: CachedBeaconStateAllForks, - attestation: allForks.Attestation -): void { +export function validateAttestation(fork: ForkSeq, state: CachedBeaconStateAllForks, attestation: Attestation): void { const {epochCtx} = state; const slot = state.slot; const data = attestation.data; diff --git a/packages/state-transition/src/block/processAttestations.ts b/packages/state-transition/src/block/processAttestations.ts index 991ba5621905..844bda768570 100644 --- a/packages/state-transition/src/block/processAttestations.ts +++ b/packages/state-transition/src/block/processAttestations.ts @@ -1,4 +1,4 @@ -import {allForks} from "@lodestar/types"; +import {Attestation} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../types.js"; import {processAttestationPhase0} from "./processAttestationPhase0.js"; @@ -10,7 +10,7 @@ import {processAttestationsAltair} from "./processAttestationsAltair.js"; export function processAttestations( fork: ForkSeq, state: CachedBeaconStateAllForks, - attestations: allForks.Attestation[], + attestations: Attestation[], verifySignatures = true ): void { if (fork === ForkSeq.phase0) { diff --git a/packages/state-transition/src/block/processAttestationsAltair.ts b/packages/state-transition/src/block/processAttestationsAltair.ts index 48d304c08133..f3d4a3e16c21 100644 --- a/packages/state-transition/src/block/processAttestationsAltair.ts +++ b/packages/state-transition/src/block/processAttestationsAltair.ts @@ -1,5 +1,5 @@ import {byteArrayEquals} from "@chainsafe/ssz"; -import {Epoch, allForks, phase0} from "@lodestar/types"; +import {Epoch, Attestation, phase0} from "@lodestar/types"; import {intSqrt} from "@lodestar/utils"; import { @@ -32,7 +32,7 @@ const SLOTS_PER_EPOCH_SQRT = intSqrt(SLOTS_PER_EPOCH); export function processAttestationsAltair( fork: ForkSeq, state: CachedBeaconStateAltair, - attestations: allForks.Attestation[], + attestations: Attestation[], verifySignature = true ): void { const {epochCtx} = state; diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 1f95da1ad5fc..cf463a93a92b 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -9,7 +9,8 @@ import { ValidatorIndex, phase0, SyncPeriod, - allForks, + Attestation, + IndexedAttestation, electra, } from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainConfig} from "@lodestar/config"; @@ -798,7 +799,7 @@ export class EpochCache { /** * Return the indexed attestation corresponding to ``attestation``. */ - getIndexedAttestation(fork: ForkSeq, attestation: allForks.Attestation): allForks.IndexedAttestation { + getIndexedAttestation(fork: ForkSeq, attestation: Attestation): IndexedAttestation { const {data} = attestation; const attestingIndices = this.getAttestingIndices(fork, attestation); @@ -814,7 +815,7 @@ export class EpochCache { /** * Return indices of validators who attestested in `attestation` */ - getAttestingIndices(fork: ForkSeq, attestation: allForks.Attestation): number[] { + getAttestingIndices(fork: ForkSeq, attestation: Attestation): number[] { if (fork < ForkSeq.electra) { const {aggregationBits, data} = attestation; const validatorIndices = this.getBeaconCommittee(data.slot, data.index); diff --git a/packages/state-transition/src/signatureSets/attesterSlashings.ts b/packages/state-transition/src/signatureSets/attesterSlashings.ts index 10e7a1991e4f..8088c2522282 100644 --- a/packages/state-transition/src/signatureSets/attesterSlashings.ts +++ b/packages/state-transition/src/signatureSets/attesterSlashings.ts @@ -1,4 +1,4 @@ -import {SignedBeaconBlock, ssz} from "@lodestar/types"; +import {SignedBeaconBlock, ssz, AttesterSlashing, IndexedAttestationBigint} from "@lodestar/types"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import {computeSigningRoot, computeStartSlotAtEpoch, ISignatureSet, SignatureSetType} from "../util/index.js"; import {CachedBeaconStateAllForks} from "../types.js"; @@ -16,7 +16,7 @@ export function getAttesterSlashingsSignatureSets( /** Get signature sets from a single AttesterSlashing object */ export function getAttesterSlashingSignatureSets( state: CachedBeaconStateAllForks, - attesterSlashing: allForks.AttesterSlashing + attesterSlashing: AttesterSlashing ): ISignatureSet[] { return [attesterSlashing.attestation1, attesterSlashing.attestation2].map((attestation) => getIndexedAttestationBigintSignatureSet(state, attestation) @@ -25,7 +25,7 @@ export function getAttesterSlashingSignatureSets( export function getIndexedAttestationBigintSignatureSet( state: CachedBeaconStateAllForks, - indexedAttestation: allForks.IndexedAttestationBigint + indexedAttestation: IndexedAttestationBigint ): ISignatureSet { const slot = computeStartSlotAtEpoch(Number(indexedAttestation.data.target.epoch as bigint)); const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_ATTESTER, slot); diff --git a/packages/state-transition/src/util/epoch.ts b/packages/state-transition/src/util/epoch.ts index b9b911137fd9..66bd2ad76b34 100644 --- a/packages/state-transition/src/util/epoch.ts +++ b/packages/state-transition/src/util/epoch.ts @@ -1,5 +1,5 @@ import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, GENESIS_EPOCH, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {BeaconState, Epoch, Slot, SyncPeriod} from "@lodestar/types"; +import {BeaconState, Epoch, Slot, SyncPeriod, Gwei} from "@lodestar/types"; import {CachedBeaconStateElectra} from "../types.js"; import {getActivationExitChurnLimit, getConsolidationChurnLimit} from "./validator.js"; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index f02a1a27aefe..0df4f4893350 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -17,7 +17,7 @@ import { MAX_ATTESTATIONS_ELECTRA, MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, - MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, + MAX_CONSOLIDATIONS, PENDING_BALANCE_DEPOSITS_LIMIT, PENDING_PARTIAL_WITHDRAWALS_LIMIT, PENDING_CONSOLIDATIONS_LIMIT, @@ -142,10 +142,7 @@ export const ConsolidationRequest = new ContainerType( }, {typeName: "ConsolidationRequest", jsonCase: "eth2"} ); -export const ConsolidationRequests = new ListCompositeType( - ConsolidationRequest, - MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD -); +export const ConsolidationRequests = new ListCompositeType(ConsolidationRequest, MAX_CONSOLIDATIONS); export const ExecutionPayload = new ContainerType( { diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index 73cb84f5eab6..5f2cc7b72b15 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -37,6 +37,11 @@ type TypesByFork = { SignedBeaconBlock: phase0.SignedBeaconBlock; Metadata: phase0.Metadata; Attestation: phase0.Attestation; + IndexedAttestation: phase0.IndexedAttestation; + IndexedAttestationBigint: phase0.IndexedAttestationBigint; + AttesterSlashing: phase0.AttesterSlashing; + AggregateAndProof: phase0.AggregateAndProof; + SignedAggregateAndProof: phase0.SignedAggregateAndProof; }; [ForkName.altair]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -55,6 +60,11 @@ type TypesByFork = { SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; Attestation: phase0.Attestation; + IndexedAttestation: phase0.IndexedAttestation; + IndexedAttestationBigint: phase0.IndexedAttestationBigint; + AttesterSlashing: phase0.AttesterSlashing; + AggregateAndProof: phase0.AggregateAndProof; + SignedAggregateAndProof: phase0.SignedAggregateAndProof; }; [ForkName.bellatrix]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -81,6 +91,11 @@ type TypesByFork = { SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; Attestation: phase0.Attestation; + IndexedAttestation: phase0.IndexedAttestation; + IndexedAttestationBigint: phase0.IndexedAttestationBigint; + AttesterSlashing: phase0.AttesterSlashing; + AggregateAndProof: phase0.AggregateAndProof; + SignedAggregateAndProof: phase0.SignedAggregateAndProof; }; [ForkName.capella]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -107,6 +122,11 @@ type TypesByFork = { SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; Attestation: phase0.Attestation; + IndexedAttestation: phase0.IndexedAttestation; + IndexedAttestationBigint: phase0.IndexedAttestationBigint; + AttesterSlashing: phase0.AttesterSlashing; + AggregateAndProof: phase0.AggregateAndProof; + SignedAggregateAndProof: phase0.SignedAggregateAndProof; }; [ForkName.deneb]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -138,6 +158,11 @@ type TypesByFork = { SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; Attestation: phase0.Attestation; + IndexedAttestation: phase0.IndexedAttestation; + IndexedAttestationBigint: phase0.IndexedAttestationBigint; + AttesterSlashing: phase0.AttesterSlashing; + AggregateAndProof: phase0.AggregateAndProof; + SignedAggregateAndProof: phase0.SignedAggregateAndProof; }; [ForkName.electra]: { BeaconBlockHeader: phase0.BeaconBlockHeader; @@ -173,6 +198,11 @@ type TypesByFork = { SyncCommittee: altair.SyncCommittee; SyncAggregate: altair.SyncAggregate; Attestation: electra.Attestation; + IndexedAttestation: electra.IndexedAttestation; + IndexedAttestationBigint: electra.IndexedAttestationBigint; + AttesterSlashing: electra.AttesterSlashing; + AggregateAndProof: electra.AggregateAndProof; + SignedAggregateAndProof: electra.SignedAggregateAndProof; }; }; @@ -233,3 +263,8 @@ export type SignedBuilderBid = TypesByF export type SSEPayloadAttributes = TypesByFork[F]["SSEPayloadAttributes"]; export type Attestation = TypesByFork[F]["Attestation"]; +export type IndexedAttestation = TypesByFork[F]["IndexedAttestation"]; +export type IndexedAttestationBigint = TypesByFork[F]["IndexedAttestationBigint"]; +export type AttesterSlashing = TypesByFork[F]["AttesterSlashing"]; +export type AggregateAndProof = TypesByFork[F]["AggregateAndProof"]; +export type SignedAggregateAndProof = TypesByFork[F]["SignedAggregateAndProof"]; diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 61bd169afeda..4f33368e6581 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,5 +1,5 @@ import {toHexString} from "@chainsafe/ssz"; -import {allForks, BLSSignature, phase0, Slot, ssz} from "@lodestar/types"; +import {BLSSignature, phase0, Slot, ssz, Attestation, SignedAggregateAndProof} from "@lodestar/types"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; @@ -193,7 +193,7 @@ export class AttestationService { attestationNoCommittee: phase0.AttestationData, duties: AttDutyAndProof[] ): Promise { - const signedAttestations: allForks.Attestation[] = []; + const signedAttestations: Attestation[] = []; const headRootHex = toHexString(attestationNoCommittee.beaconBlockRoot); const currentEpoch = computeEpochAtSlot(slot); const isAfterElectra = currentEpoch >= this.config.ELECTRA_FORK_EPOCH; @@ -277,7 +277,7 @@ export class AttestationService { const aggregate = res.value(); this.metrics?.numParticipantsInAggregate.observe(aggregate.aggregationBits.getTrueBitIndexes().length); - const signedAggregateAndProofs: allForks.SignedAggregateAndProof[] = []; + const signedAggregateAndProofs: SignedAggregateAndProof[] = []; await Promise.all( duties.map(async ({duty, selectionProof}) => { diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index a85dd52a7c0d..cea281f0d144 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -37,6 +37,9 @@ import { Slot, ssz, ValidatorIndex, + Attestation, + AggregateAndProof, + SignedAggregateAndProof, } from "@lodestar/types"; import {routes} from "@lodestar/api"; import {ISlashingProtection} from "../slashingProtection/index.js"; @@ -495,7 +498,7 @@ export class ValidatorStore { duty: routes.validator.AttesterDuty, attestationData: phase0.AttestationData, currentEpoch: Epoch - ): Promise { + ): Promise { // Make sure the target epoch is not higher than the current epoch to avoid potential attacks. if (attestationData.target.epoch > currentEpoch) { throw Error( @@ -546,11 +549,11 @@ export class ValidatorStore { async signAggregateAndProof( duty: routes.validator.AttesterDuty, selectionProof: BLSSignature, - aggregate: allForks.Attestation - ): Promise { + aggregate: Attestation + ): Promise { this.validateAttestationDuty(duty, aggregate.data); - const aggregateAndProof: allForks.AggregateAndProof = { + const aggregateAndProof: AggregateAndProof = { aggregate, aggregatorIndex: duty.validatorIndex, selectionProof, diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index d5369515d47b..3a497555361a 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -234,7 +234,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Mon, 1 Jul 2024 15:05:37 +0530 Subject: [PATCH 076/145] feat: add and parse consolidation requests from engine api get the signing log --- .../src/execution/engine/payloadIdCache.ts | 6 +++ .../beacon-node/src/execution/engine/types.ts | 46 +++++++++++++++++-- packages/validator/src/services/block.ts | 2 +- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index 8cb5b4e4f8e9..005a1ef14322 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -32,6 +32,12 @@ export type WithdrawalRequestV1 = { amount: QUANTITY; }; +export type ConsolidationRequestV1 = { + sourceAddress: DATA; + sourcePubkey: DATA; + targetPubkey: DATA; +}; + type FcuAttributes = {headBlockHash: DATA; finalizedBlockHash: DATA} & Omit; export class PayloadIdCache { diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 6b37f3854b8f..63cb4da88b6c 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1, DepositRequestV1, WithdrawalRequestV1} from "./payloadIdCache.js"; +import {WithdrawalV1, DepositRequestV1, WithdrawalRequestV1, ConsolidationRequestV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -133,6 +133,7 @@ export type ExecutionPayloadBodyRpc = { // its likely CL receipt will be renamed to requests depositRequests: DepositRequestV1[] | null | undefined; withdrawalRequests: WithdrawalRequestV1[] | null | undefined; + consolidationRequests: ConsolidationRequestV1[] | null | undefined; }; export type ExecutionPayloadBody = { @@ -140,6 +141,7 @@ export type ExecutionPayloadBody = { withdrawals: capella.Withdrawals | null; depositRequests: electra.DepositRequests | null; withdrawalRequests: electra.WithdrawalRequests | null; + consolidationRequests: electra.ConsolidationRequests | null; }; export type ExecutionPayloadRpc = { @@ -163,6 +165,7 @@ export type ExecutionPayloadRpc = { parentBeaconBlockRoot?: QUANTITY; // DENEB depositRequests?: DepositRequestRpc[]; // ELECTRA withdrawalRequests?: WithdrawalRequestRpc[]; // ELECTRA + consolidationRequests?: ConsolidationRequestRpc[]; // ELECTRA }; export type WithdrawalRpc = { @@ -174,6 +177,7 @@ export type WithdrawalRpc = { export type DepositRequestRpc = DepositRequestV1; export type WithdrawalRequestRpc = WithdrawalRequestV1; +export type ConsolidationRequestRpc = ConsolidationRequestV1; export type VersionedHashesRpc = DATA[]; @@ -239,9 +243,10 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload // ELECTRA adds depositRequests/depositRequests to the ExecutionPayload if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositRequests, withdrawalRequests} = data as electra.ExecutionPayload; + const {depositRequests, withdrawalRequests, consolidationRequests} = data as electra.ExecutionPayload; payload.depositRequests = depositRequests.map(serializeDepositRequest); payload.withdrawalRequests = withdrawalRequests.map(serializeWithdrawalRequest); + payload.consolidationRequests = consolidationRequests.map(serializeConsolidationRequest); } return payload; @@ -331,7 +336,7 @@ export function parseExecutionPayload( if (ForkSeq[fork] >= ForkSeq.electra) { // electra adds depositRequests/depositRequests - const {depositRequests, withdrawalRequests} = data; + const {depositRequests, withdrawalRequests, consolidationRequests} = data; // Geth can also reply with null if (depositRequests == null) { throw Error( @@ -347,6 +352,15 @@ export function parseExecutionPayload( } (executionPayload as electra.ExecutionPayload).withdrawalRequests = withdrawalRequests.map(deserializeWithdrawalRequest); + + if (consolidationRequests == null) { + throw Error( + `consolidationRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` + ); + } + (executionPayload as electra.ExecutionPayload).consolidationRequests = consolidationRequests.map( + deserializeConsolidationRequest + ); } return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; @@ -451,6 +465,26 @@ export function deserializeWithdrawalRequest(withdrawalRequest: WithdrawalReques }; } +export function serializeConsolidationRequest( + consolidationRequest: electra.ConsolidationRequest +): ConsolidationRequestRpc { + return { + sourceAddress: bytesToData(consolidationRequest.sourceAddress), + sourcePubkey: bytesToData(consolidationRequest.sourcePubkey), + targetPubkey: bytesToData(consolidationRequest.targetPubkey), + }; +} + +export function deserializeConsolidationRequest( + consolidationRequest: ConsolidationRequestRpc +): electra.ConsolidationRequest { + return { + sourceAddress: dataToBytes(consolidationRequest.sourceAddress, 20), + sourcePubkey: dataToBytes(consolidationRequest.sourcePubkey, 48), + targetPubkey: dataToBytes(consolidationRequest.targetPubkey, 48), + }; +} + export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | null): ExecutionPayloadBody | null { return data ? { @@ -458,6 +492,9 @@ export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, depositRequests: data.depositRequests ? data.depositRequests.map(deserializeDepositRequest) : null, withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(deserializeWithdrawalRequest) : null, + consolidationRequests: data.consolidationRequests + ? data.consolidationRequests.map(deserializeConsolidationRequest) + : null, } : null; } @@ -469,6 +506,9 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, depositRequests: data.depositRequests ? data.depositRequests.map(serializeDepositRequest) : null, withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(serializeWithdrawalRequest) : null, + consolidationRequests: data.consolidationRequests + ? data.consolidationRequests.map(serializeConsolidationRequest) + : null, } : null; } diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index a9dd7654a8fc..7792ef85be95 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -160,7 +160,7 @@ export class BlockProposingService { this.logger.debug("Produced block", {...debugLogCtx, ...blockContents.debugLogCtx}); this.metrics?.blocksProduced.inc(); - const signedBlock = await this.validatorStore.signBlock(pubkey, blockContents.block, slot); + const signedBlock = await this.validatorStore.signBlock(pubkey, blockContents.block, slot, this.logger); const {broadcastValidation} = this.opts; const publishOpts = {broadcastValidation}; From 08445fd9406b52374f37b76cf6b580be747dfa64 Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 1 Jul 2024 20:01:27 +0530 Subject: [PATCH 077/145] feat: make produce block/signed block contents api multifork --- packages/api/src/beacon/routes/validator.ts | 16 +++++----------- packages/types/src/electra/sszTypes.ts | 18 ++++++++++++++++++ packages/types/src/electra/types.ts | 3 +++ packages/types/src/types.ts | 8 ++------ 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 154397107d47..86d4c6930951 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -83,15 +83,6 @@ export type ProduceBlockV3Meta = ValueOf & { executionPayloadSource: ProducedBlockSource; }; -export const BlockContentsType = new ContainerType( - { - block: ssz.deneb.BeaconBlock, - kzgProofs: ssz.deneb.KZGProofs, - blobs: ssz.deneb.Blobs, - }, - {jsonCase: "eth2"} -); - export const AttesterDutyType = new ContainerType( { /** The validator's public key, uniquely identifying them */ @@ -630,7 +621,10 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions (isForkBlobs(fork) ? BlockContentsType : ssz[fork].BeaconBlock) as Type + (fork) => + (isForkBlobs(fork) + ? ssz.allForksBlobs[fork].BlockContents + : ssz[fork].BeaconBlock) as Type ), meta: VersionCodec, }, @@ -693,7 +687,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ), meta: { diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 0df4f4893350..7ccbd16d8e9e 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -417,3 +417,21 @@ export const SSEPayloadAttributes = new ContainerType( }, {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} ); + +export const BlockContents = new ContainerType( + { + block: BeaconBlock, + kzgProofs: denebSsz.KZGProofs, + blobs: denebSsz.Blobs, + }, + {typeName: "BlockContents", jsonCase: "eth2"} +); + +export const SignedBlockContents = new ContainerType( + { + signedBlock: SignedBeaconBlock, + kzgProofs: denebSsz.KZGProofs, + blobs: denebSsz.Blobs, + }, + {typeName: "BlockContents", jsonCase: "eth2"} +); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 728ffc56ac83..9a81aec43b53 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -49,3 +49,6 @@ export type LightClientStore = ValueOf; export type PendingBalanceDeposit = ValueOf; export type PendingPartialWithdrawal = ValueOf; export type PendingConsolidation = ValueOf; + +export type BlockContents = ValueOf; +export type SignedBlockContents = ValueOf; diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index 5f2cc7b72b15..1071eed79a10 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -186,12 +186,8 @@ type TypesByFork = { BuilderBid: electra.BuilderBid; SignedBuilderBid: electra.SignedBuilderBid; SSEPayloadAttributes: electra.SSEPayloadAttributes; - BlockContents: {block: BeaconBlock; kzgProofs: deneb.KZGProofs; blobs: deneb.Blobs}; - SignedBlockContents: { - signedBlock: SignedBeaconBlock; - kzgProofs: deneb.KZGProofs; - blobs: deneb.Blobs; - }; + BlockContents: electra.BlockContents; + SignedBlockContents: electra.SignedBlockContents; ExecutionPayloadAndBlobsBundle: deneb.ExecutionPayloadAndBlobsBundle; BlobsBundle: deneb.BlobsBundle; Contents: deneb.Contents; From 3a98b2f026f6a3fa657cae39ec5edbe719b757b2 Mon Sep 17 00:00:00 2001 From: harkamal Date: Mon, 1 Jul 2024 22:40:56 +0530 Subject: [PATCH 078/145] fix: electra rebase fixes on the allforks type refactor tests fixes --- packages/api/src/beacon/routes/validator.ts | 7 +++---- packages/beacon-node/src/network/gossip/topic.ts | 6 +++--- .../src/network/processor/gossipHandlers.ts | 4 ++-- packages/beacon-node/src/util/sszBytes.ts | 10 +++++++--- .../beacon-node/test/spec/presets/fork_choice.test.ts | 4 ++-- .../beacon-node/test/spec/presets/operations.test.ts | 4 ++-- .../beacon-node/test/spec/presets/ssz_static.test.ts | 5 ----- .../test/unit/executionEngine/http.test.ts | 4 ++++ .../network/beaconBlocksMaybeBlobsByRange.test.ts | 2 +- packages/light-client/src/spec/utils.ts | 2 ++ packages/params/src/index.ts | 4 ++-- packages/params/src/presets/mainnet.ts | 4 ++-- packages/params/src/presets/minimal.ts | 4 ++-- packages/params/src/types.ts | 8 ++++---- .../params/test/e2e/ensure-config-is-synced.test.ts | 2 +- packages/types/src/electra/sszTypes.ts | 11 +++++++---- packages/types/test/unit/blinded.test.ts | 2 +- packages/validator/src/util/params.ts | 4 ++-- 18 files changed, 47 insertions(+), 40 deletions(-) diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 86d4c6930951..c0078c97d5f8 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -18,6 +18,7 @@ import { BeaconBlockOrContents, BlindedBeaconBlock, Attestation, + sszTypesFor, } from "@lodestar/types"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; import {fromGraffitiHex, toBoolean, toGraffitiHex} from "../../utils/serdes.js"; @@ -622,9 +623,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions - (isForkBlobs(fork) - ? ssz.allForksBlobs[fork].BlockContents - : ssz[fork].BeaconBlock) as Type + (isForkBlobs(fork) ? sszTypesFor(fork).BlockContents : ssz[fork].BeaconBlock) as Type ), meta: VersionCodec, }, @@ -687,7 +686,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ), meta: { diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index 2ccde0044037..dfca0eaf3dcd 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -87,9 +87,9 @@ export function getGossipSSZType(topic: GossipTopic) { case GossipType.blob_sidecar: return ssz.deneb.BlobSidecar; case GossipType.beacon_aggregate_and_proof: - return ssz.allForks[topic.fork].SignedAggregateAndProof; + return sszTypesFor(topic.fork).SignedAggregateAndProof; case GossipType.beacon_attestation: - return ssz.allForks[topic.fork].Attestation; + return sszTypesFor(topic.fork).Attestation; case GossipType.proposer_slashing: return ssz.phase0.ProposerSlashing; case GossipType.attester_slashing: @@ -129,7 +129,7 @@ export function sszDeserialize(topic: T, serializedData: * Deserialize a gossip serialized data into an Attestation object. */ export function sszDeserializeAttestation(fork: ForkName, serializedData: Uint8Array): Attestation { - const sszType = ssz.allForks[fork].Attestation; + const sszType = sszTypesFor(fork).Attestation; try { return sszType.deserialize(serializedData); } catch (e) { diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 3f2d6368928d..51fa7bd376c8 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -1,6 +1,6 @@ import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {LogLevel, Logger, prettyBytes, toRootHex} from "@lodestar/utils"; -import {Root, Slot, ssz, deneb, UintNum64, SignedBeaconBlock} from "@lodestar/types"; +import {Root, Slot, ssz, deneb, UintNum64, SignedBeaconBlock, sszTypesFor} from "@lodestar/types"; import {ForkName, ForkSeq} from "@lodestar/params"; import {routes} from "@lodestar/api"; import {computeTimeAtSlot} from "@lodestar/state-transition"; @@ -422,7 +422,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { chain.persistInvalidSszValue( - ssz.allForks[fork].SignedAggregateAndProof, + sszTypesFor(fork).SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject" ); diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index bac66ddf3a0d..89f1dd20a473 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -230,14 +230,18 @@ export function getSeenAttDataKeyFromSignedAggregateAndProof( */ export function getSeenAttDataKeyFromSignedAggregateAndProofElectra(data: Uint8Array): SeenAttDataKey | null { const startIndex = SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET; - const endIndex = startIndex + ATTESTATION_DATA_SIZE + COMMITTEE_BITS_SIZE; + const endIndex = startIndex + ATTESTATION_DATA_SIZE; - if (data.length < endIndex) { + if (data.length < endIndex + SIGNATURE_SIZE + COMMITTEE_BITS_SIZE) { return null; } // base64 is a bit efficient than hex - return toBase64(data.subarray(startIndex, endIndex)); + + return Buffer.concat([ + data.subarray(startIndex, endIndex), + data.subarray(endIndex + SIGNATURE_SIZE, endIndex + SIGNATURE_SIZE + COMMITTEE_BITS_SIZE), + ]).toString("base64"); } /** diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index 9f8d5c727879..01adcc77500b 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -343,8 +343,8 @@ const forkChoiceTest = [BLOCK_FILE_NAME]: ssz[fork].SignedBeaconBlock, [BLOBS_FILE_NAME]: ssz.deneb.Blobs, [POW_BLOCK_FILE_NAME]: ssz.bellatrix.PowBlock, - [ATTESTATION_FILE_NAME]: ssz.allForks[fork].Attestation, - [ATTESTER_SLASHING_FILE_NAME]: ssz.allForks[fork].AttesterSlashing, + [ATTESTATION_FILE_NAME]: ssz.phase0.Attestation, + [ATTESTER_SLASHING_FILE_NAME]: ssz.phase0.AttesterSlashing, }, mapToTestCase: (t: Record) => { // t has input file name as key diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 1128cdd11ba9..24f51b05c4f6 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -135,8 +135,8 @@ const operations: TestRunnerFn = (fork, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, - attestation: ssz.allForks[fork].Attestation, - attester_slashing: ssz.allForks[fork].AttesterSlashing, + attestation: ssz.phase0.Attestation, + attester_slashing: ssz.phase0.AttesterSlashing, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index bf7a3b5aad06..6e43d851ef66 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -2,13 +2,8 @@ import fs from "node:fs"; import path from "node:path"; import {expect, it, vi} from "vitest"; import {Type} from "@chainsafe/ssz"; -<<<<<<< HEAD import {ssz, sszTypesFor} from "@lodestar/types"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; -======= -import {ssz} from "@lodestar/types"; -import {ACTIVE_PRESET, ForkName, ForkLightClient, ForkExecution} from "@lodestar/params"; ->>>>>>> 82d0692d63d (test: fix ssz_static spec tests for all forks (#6771)) import {replaceUintTypeWithUintBigintType} from "../utils/replaceUintTypeWithUintBigintType.js"; import {parseSszStaticTestcase} from "../utils/sszTestCaseParser.js"; import {runValidSszTest} from "../utils/runValidSszTest.js"; diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index 90ca6595d8a2..5ac4fd4ca670 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -195,6 +195,7 @@ describe("ExecutionEngine / http", () => { ], depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, + consolidationRequests: null, }, null, // null returned for missing blocks { @@ -205,6 +206,7 @@ describe("ExecutionEngine / http", () => { withdrawals: null, // withdrawals is null pre-capella depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, + consolidationRequests: null, }, ], }; @@ -254,6 +256,7 @@ describe("ExecutionEngine / http", () => { ], depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, + consolidationRequests: null, }, null, // null returned for missing blocks { @@ -264,6 +267,7 @@ describe("ExecutionEngine / http", () => { withdrawals: null, // withdrawals is null pre-capella depositRequests: null, // depositRequests is null pre-electra withdrawalRequests: null, + consolidationRequests: null, }, ], }; diff --git a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts index c7de35d468df..d1dc7ba57fa9 100644 --- a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts +++ b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts @@ -102,7 +102,7 @@ describe("beaconBlocksMaybeBlobsByRange", () => { const expectedResponse = blocksWithBlobs.map(([block, blobSidecars]) => { const blobs = blobSidecars !== undefined ? blobSidecars : []; return getBlockInput.availableData(config, block, BlockSource.byRange, null, { - fork: ForkName.deneb, + fork: ForkName.electra, blobs, blobsSource: BlobsSource.byRange, blobsBytes: blobs.map(() => null), diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 679371a54d6c..3e59cb14cfa0 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -119,6 +119,8 @@ export function upgradeLightClientHeader( ssz.electra.LightClientHeader.fields.execution.fields.depositRequestsRoot.defaultValue(); (upgradedHeader as LightClientHeader).execution.withdrawalRequestsRoot = ssz.electra.LightClientHeader.fields.execution.fields.withdrawalRequestsRoot.defaultValue(); + (upgradedHeader as LightClientHeader).execution.consolidationRequestsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.consolidationRequestsRoot.defaultValue(); // Break if no further upgrades is required else fall through if (ForkSeq[targetFork] <= ForkSeq.electra) break; diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 0c2e3b34794b..00e9d48346c1 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -100,9 +100,9 @@ export const { PENDING_PARTIAL_WITHDRAWALS_LIMIT, PENDING_CONSOLIDATIONS_LIMIT, MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA, - MAX_CONSOLIDATIONS, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, - MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, + MAX_DEPOSIT_REQUESTS_PER_PAYLOAD, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_ATTESTATIONS_ELECTRA, diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 2495f7ef97a1..a4a88aac1f58 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -120,7 +120,7 @@ export const mainnetPreset: BeaconPreset = { KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17, // ELECTRA - MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192, + MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 8192, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16, MAX_ATTESTER_SLASHINGS_ELECTRA: 1, MAX_ATTESTATIONS_ELECTRA: 8, @@ -133,6 +133,6 @@ export const mainnetPreset: BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728, PENDING_CONSOLIDATIONS_LIMIT: 262144, - MAX_CONSOLIDATIONS: 1, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1, WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 8e71407965d7..97eff53cf013 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -121,7 +121,7 @@ export const minimalPreset: BeaconPreset = { KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9, // ELECTRA - MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4, + MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 4, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2, MAX_ATTESTER_SLASHINGS_ELECTRA: 1, MAX_ATTESTATIONS_ELECTRA: 8, @@ -134,6 +134,6 @@ export const minimalPreset: BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 64, PENDING_CONSOLIDATIONS_LIMIT: 64, - MAX_CONSOLIDATIONS: 1, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1, WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index dffd98518006..e867b4a3cf71 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -84,7 +84,7 @@ export type BeaconPreset = { KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: number; // ELECTRA - MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: number; + MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: number; MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: number; MAX_ATTESTER_SLASHINGS_ELECTRA: number; MAX_ATTESTATIONS_ELECTRA: number; @@ -95,7 +95,7 @@ export type BeaconPreset = { PENDING_BALANCE_DEPOSITS_LIMIT: number; PENDING_PARTIAL_WITHDRAWALS_LIMIT: number; PENDING_CONSOLIDATIONS_LIMIT: number; - MAX_CONSOLIDATIONS: number; + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: number; WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: number; }; @@ -184,7 +184,7 @@ export const beaconPresetTypes: BeaconPresetTypes = { KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: "number", // ELECTRA - MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: "number", + MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: "number", MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: "number", MAX_ATTESTER_SLASHINGS_ELECTRA: "number", MAX_ATTESTATIONS_ELECTRA: "number", @@ -195,7 +195,7 @@ export const beaconPresetTypes: BeaconPresetTypes = { PENDING_BALANCE_DEPOSITS_LIMIT: "number", PENDING_PARTIAL_WITHDRAWALS_LIMIT: "number", PENDING_CONSOLIDATIONS_LIMIT: "number", - MAX_CONSOLIDATIONS: "number", + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: "number", WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: "number", }; diff --git a/packages/params/test/e2e/ensure-config-is-synced.test.ts b/packages/params/test/e2e/ensure-config-is-synced.test.ts index 515d8f44a710..38168fa02bae 100644 --- a/packages/params/test/e2e/ensure-config-is-synced.test.ts +++ b/packages/params/test/e2e/ensure-config-is-synced.test.ts @@ -8,7 +8,7 @@ import {loadConfigYaml} from "../yaml.js"; // Not e2e, but slow. Run with e2e tests /** https://github.com/ethereum/consensus-specs/releases */ -const specConfigCommit = "v1.5.0-alpha.2"; +const specConfigCommit = "v1.5.0-alpha.3"; describe("Ensure config is synced", function () { vi.setConfig({testTimeout: 60 * 1000}); diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 7ccbd16d8e9e..a6874c7894e3 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -11,13 +11,13 @@ import { BLOCK_BODY_EXECUTION_PAYLOAD_DEPTH as EXECUTION_PAYLOAD_DEPTH, EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH, - MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, + MAX_DEPOSIT_REQUESTS_PER_PAYLOAD, MAX_VALIDATORS_PER_COMMITTEE, MAX_COMMITTEES_PER_SLOT, MAX_ATTESTATIONS_ELECTRA, MAX_ATTESTER_SLASHINGS_ELECTRA, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, - MAX_CONSOLIDATIONS, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, PENDING_BALANCE_DEPOSITS_LIMIT, PENDING_PARTIAL_WITHDRAWALS_LIMIT, PENDING_CONSOLIDATIONS_LIMIT, @@ -123,7 +123,7 @@ export const DepositRequest = new ContainerType( {typeName: "DepositRequest", jsonCase: "eth2"} ); -export const DepositRequests = new ListCompositeType(DepositRequest, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); +export const DepositRequests = new ListCompositeType(DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD); export const WithdrawalRequest = new ContainerType( { @@ -142,7 +142,10 @@ export const ConsolidationRequest = new ContainerType( }, {typeName: "ConsolidationRequest", jsonCase: "eth2"} ); -export const ConsolidationRequests = new ListCompositeType(ConsolidationRequest, MAX_CONSOLIDATIONS); +export const ConsolidationRequests = new ListCompositeType( + ConsolidationRequest, + MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD +); export const ExecutionPayload = new ContainerType( { diff --git a/packages/types/test/unit/blinded.test.ts b/packages/types/test/unit/blinded.test.ts index 77005f1defc2..3a4b346d29bf 100644 --- a/packages/types/test/unit/blinded.test.ts +++ b/packages/types/test/unit/blinded.test.ts @@ -10,7 +10,7 @@ describe("blinded data structures", function () { ]; for (const {a, b} of blindedTypes) { - for (const fork of Object.keys(ssz.allForks) as ForkName[]) { + for (const fork of Object.keys(ssz.sszTypesFor) as ForkName[]) { if (!isForkExecution(fork)) { continue; } diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 3a497555361a..6d6705f512df 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -223,7 +223,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Tue, 30 Jul 2024 17:04:58 +0530 Subject: [PATCH 079/145] chore: rebase fixes lint lint and type fix update spec version fix tests fix spec test Update packages/beacon-node/src/chain/stateCache/types.ts Co-authored-by: Cayman Update packages/beacon-node/src/chain/stateCache/types.ts Co-authored-by: Cayman Update packages/beacon-node/src/chain/blocks/types.ts Co-authored-by: Cayman Update packages/types/src/utils/typeguards.ts Co-authored-by: Cayman Variable naming fix the ForkElectra fix build fix spec test Co-authored-by: Cayman --- .../api/test/unit/beacon/oapiSpec.test.ts | 5 ++++ .../src/api/impl/config/constants.ts | 4 +-- .../beacon-node/src/chain/blocks/types.ts | 4 +-- .../opPools/aggregatedAttestationPool.ts | 3 +-- .../beacon-node/src/chain/opPools/opPool.ts | 4 +-- .../beacon-node/src/chain/stateCache/types.ts | 6 +++-- .../signatureSets/aggregateAndProof.ts | 2 +- .../src/execution/engine/interface.ts | 2 -- .../updateUnfinalizedPubkeys.test.ts | 6 ++--- .../test/sim/electra-interop.test.ts | 4 +-- .../test/spec/presets/fork_choice.test.ts | 26 +++++++++++++------ .../test/spec/presets/operations.test.ts | 4 +-- .../opPools/aggregatedAttestationPool.test.ts | 10 +++---- packages/params/src/forkName.ts | 13 ++++++++++ packages/params/src/index.ts | 2 +- .../unit/__snapshots__/forkName.test.ts.snap | 5 ++++ .../src/block/processDeposit.ts | 2 +- .../src/block/processDepositRequest.ts | 4 +-- .../src/slot/upgradeStateToElectra.ts | 10 +++---- packages/state-transition/src/util/electra.ts | 14 +++++----- packages/state-transition/src/util/genesis.ts | 4 +-- packages/types/src/utils/typeguards.ts | 6 ++--- yarn.lock | 14 ++++++++++ 23 files changed, 99 insertions(+), 55 deletions(-) diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index e5d473ab6a55..00f1a3d8731d 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -86,6 +86,11 @@ const ignoredTopics = [ topic block_gossip not implemented */ "block_gossip", + + // Modified in electra to include version + // should be removed from the ignore list after spec update + "attestation", + "attester_slashing", ]; // eventstream types are defined as comments in the description of "examples". diff --git a/packages/beacon-node/src/api/impl/config/constants.ts b/packages/beacon-node/src/api/impl/config/constants.ts index b5f810157581..78262ce15237 100644 --- a/packages/beacon-node/src/api/impl/config/constants.ts +++ b/packages/beacon-node/src/api/impl/config/constants.ts @@ -38,7 +38,7 @@ import { VERSIONED_HASH_VERSION_KZG, COMPOUNDING_WITHDRAWAL_PREFIX, DOMAIN_CONSOLIDATION, - UNSET_DEPOSIT_RECEIPTS_START_INDEX, + UNSET_DEPOSIT_REQUESTS_START_INDEX, FULL_EXIT_REQUEST_AMOUNT, } from "@lodestar/params"; @@ -108,6 +108,6 @@ export const specConstants = { VERSIONED_HASH_VERSION_KZG, // electra - UNSET_DEPOSIT_RECEIPTS_START_INDEX, + UNSET_DEPOSIT_REQUESTS_START_INDEX, FULL_EXIT_REQUEST_AMOUNT, }; diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 5eb5eebc7840..80c94e2d6fde 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -1,7 +1,7 @@ import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; import {MaybeValidExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {deneb, Slot, RootHex, SignedBeaconBlock} from "@lodestar/types"; -import {ForkSeq, ForkName} from "@lodestar/params"; +import {ForkSeq, ForkName, ForkBlobs} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; export enum BlockInputType { @@ -36,7 +36,7 @@ export enum GossipedInputType { type BlobsCacheMap = Map; -type ForkBlobsInfo = {fork: ForkName.deneb | ForkName.electra}; +type ForkBlobsInfo = {fork: ForkBlobs}; type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]; blobsSource: BlobsSource}; export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData; export type BlockInputData = BlockInputDataBlobs; diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 491b5fd6f72a..b3e82c0c6366 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -1,5 +1,4 @@ -import {aggregateSignatures} from "@chainsafe/blst"; -import {Signature} from "@chainsafe/bls/types"; +import {aggregateSignatures, Signature} from "@chainsafe/blst"; import {BitArray, toHexString} from "@chainsafe/ssz"; import { ForkName, diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index 54fec380de86..fc4558d9d995 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -210,7 +210,7 @@ export class OpPool { const endAttesterSlashings = stepsMetrics?.startTimer(); const attesterSlashings: AttesterSlashing[] = []; - const maxAttesterSlashing = stateFork >= ForkSeq.electra ? MAX_ATTESTER_SLASHINGS_ELECTRA : MAX_ATTESTER_SLASHINGS; + const maxAttesterSlashings = stateFork >= ForkSeq.electra ? MAX_ATTESTER_SLASHINGS_ELECTRA : MAX_ATTESTER_SLASHINGS; attesterSlashing: for (const attesterSlashing of this.attesterSlashings.values()) { /** Indices slashable in this attester slashing */ const slashableIndices = new Set(); @@ -225,7 +225,7 @@ export class OpPool { if (isSlashableAtEpoch(validator, stateEpoch)) { slashableIndices.add(index); } - if (attesterSlashings.length >= maxAttesterSlashing) { + if (attesterSlashings.length >= maxAttesterSlashings) { break attesterSlashing; } } diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index ad93def481e7..1e8d6bd1bd62 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -33,7 +33,8 @@ export interface BlockStateCache { prune(headStateRootHex: RootHex): void; deleteAllBeforeEpoch(finalizedEpoch: Epoch): void; dumpSummary(): routes.lodestar.StateCacheItem[]; - getStates(): IterableIterator; // Expose beacon states stored in cache. Use with caution + /** Expose beacon states stored in cache. Use with caution */ + getStates(): IterableIterator; } /** @@ -75,7 +76,8 @@ export interface CheckpointStateCache { processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise; clear(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; - getStates(): IterableIterator; // Expose beacon states stored in cache. Use with caution + /** Expose beacon states stored in cache. Use with caution */ + getStates(): IterableIterator; } export enum CacheItemType { diff --git a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts index 2945b4d445d0..31d931818595 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -1,4 +1,4 @@ -import type {PublicKey} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {DOMAIN_AGGREGATE_AND_PROOF, ForkSeq} from "@lodestar/params"; import {ssz, SignedAggregateAndProof} from "@lodestar/types"; import {Epoch} from "@lodestar/types"; diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index a835fa76f509..5226a46ac720 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -177,6 +177,4 @@ export interface IExecutionEngine { getPayloadBodiesByHash(fork: ForkName, blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>; getPayloadBodiesByRange(fork: ForkName, start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>; - - getState(): ExecutionEngineState; } diff --git a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts index b8f5c30a70ea..46659ab3d287 100644 --- a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts +++ b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts @@ -2,8 +2,7 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {Map} from "immutable"; import {toBufferBE} from "bigint-buffer"; import {digest} from "@chainsafe/as-sha256"; -import type {SecretKey} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; +import {SecretKey} from "@chainsafe/blst"; import {ssz} from "@lodestar/types"; import {type CachedBeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; import {bytesToBigInt, intToBytes} from "@lodestar/utils"; @@ -106,6 +105,7 @@ describe("updateUnfinalizedPubkeys perf tests", function () { function generatePrivateKey(index: number): SecretKey { const secretKeyBytes = toBufferBE(bytesToBigInt(digest(intToBytes(index, 32))) % BigInt("38581184513"), 32); - return bls.SecretKey.fromBytes(secretKeyBytes); + const secret: SecretKey = SecretKey.fromBytes(secretKeyBytes); + return secret; } }); diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 2dfa5ee0f77c..02fc99f9cfae 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert"; import {describe, it, vi, afterAll, afterEach} from "vitest"; import {LogLevel, sleep} from "@lodestar/utils"; -import {ForkName, SLOTS_PER_EPOCH, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {ForkName, SLOTS_PER_EPOCH, UNSET_DEPOSIT_REQUESTS_START_INDEX} from "@lodestar/params"; import {electra, Epoch, Slot} from "@lodestar/types"; import {ValidatorProposerConfig} from "@lodestar/validator"; @@ -431,7 +431,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { throw Error("Historical validator length for epoch 1 or 2 is not dropped properly"); } - if (headState.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + if (headState.depositRequestsStartIndex === UNSET_DEPOSIT_REQUESTS_START_INDEX) { throw Error("state.depositRequestsStartIndex is not set upon processing new deposit receipt"); } diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts index 01adcc77500b..7cb6e3c3d692 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.test.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.test.ts @@ -4,7 +4,17 @@ import {toHexString} from "@chainsafe/ssz"; import {BeaconStateAllForks, isExecutionStateType, signedBlockToSignedHeader} from "@lodestar/state-transition"; import {InputType} from "@lodestar/spec-test-util"; import {CheckpointWithHex, ForkChoice} from "@lodestar/fork-choice"; -import {phase0, bellatrix, ssz, RootHex, deneb, BeaconBlock, SignedBeaconBlock} from "@lodestar/types"; +import { + bellatrix, + ssz, + RootHex, + deneb, + BeaconBlock, + SignedBeaconBlock, + sszTypesFor, + Attestation, + AttesterSlashing, +} from "@lodestar/types"; import {bnToNum, fromHex} from "@lodestar/utils"; import {createBeaconConfig} from "@lodestar/config"; import {ACTIVE_PRESET, ForkSeq, isForkBlobs, ForkName} from "@lodestar/params"; @@ -136,7 +146,7 @@ const forkChoiceTest = const attestation = testcase.attestations.get(step.attestation); if (!attestation) throw Error(`No attestation ${step.attestation}`); const headState = chain.getHeadState(); - const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(attestation.data)); + const attDataRootHex = toHexString(sszTypesFor(fork).AttestationData.hashTreeRoot(attestation.data)); chain.forkChoice.onAttestation( headState.epochCtx.getIndexedAttestation(ForkSeq[fork], attestation), attDataRootHex @@ -343,16 +353,16 @@ const forkChoiceTest = [BLOCK_FILE_NAME]: ssz[fork].SignedBeaconBlock, [BLOBS_FILE_NAME]: ssz.deneb.Blobs, [POW_BLOCK_FILE_NAME]: ssz.bellatrix.PowBlock, - [ATTESTATION_FILE_NAME]: ssz.phase0.Attestation, - [ATTESTER_SLASHING_FILE_NAME]: ssz.phase0.AttesterSlashing, + [ATTESTATION_FILE_NAME]: sszTypesFor(fork).Attestation, + [ATTESTER_SLASHING_FILE_NAME]: sszTypesFor(fork).AttesterSlashing, }, mapToTestCase: (t: Record) => { // t has input file name as key const blocks = new Map(); const blobs = new Map(); const powBlocks = new Map(); - const attestations = new Map(); - const attesterSlashings = new Map(); + const attestations = new Map(); + const attesterSlashings = new Map(); for (const key in t) { const blockMatch = key.match(BLOCK_FILE_NAME); if (blockMatch) { @@ -495,8 +505,8 @@ type ForkChoiceTestCase = { blocks: Map; blobs: Map; powBlocks: Map; - attestations: Map; - attesterSlashings: Map; + attestations: Map; + attesterSlashings: Map; }; function isTick(step: Step): step is OnTick { diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 24f51b05c4f6..c978cac3c8a2 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -135,8 +135,8 @@ const operations: TestRunnerFn = (fork, sszTypes: { pre: ssz[fork].BeaconState, post: ssz[fork].BeaconState, - attestation: ssz.phase0.Attestation, - attester_slashing: ssz.phase0.AttesterSlashing, + attestation: sszTypesFor(fork).Attestation, + attester_slashing: sszTypesFor(fork).AttesterSlashing, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 57b5d6fad00f..c9c128e5485e 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,6 +1,6 @@ import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {describe, it, expect, beforeEach, beforeAll, afterEach, vi} from "vitest"; -import {SecretKey, Signature, fastAggregateVerify} from "@chainsafe/blst"; +import {SecretKey, Signature, fastAggregateVerify, aggregateSignatures} from "@chainsafe/blst"; import {CachedBeaconStateAllForks, newFilledArray} from "@lodestar/state-transition"; import { FAR_FUTURE_EPOCH, @@ -331,9 +331,9 @@ describe("MatchingDataAttestationGroup aggregateInto", function () { }); describe("aggregateConsolidation", function () { - const sk0 = bls.SecretKey.fromBytes(Buffer.alloc(32, 1)); - const sk1 = bls.SecretKey.fromBytes(Buffer.alloc(32, 2)); - const sk2 = bls.SecretKey.fromBytes(Buffer.alloc(32, 3)); + const sk0 = SecretKey.fromBytes(Buffer.alloc(32, 1)); + const sk1 = SecretKey.fromBytes(Buffer.alloc(32, 2)); + const sk2 = SecretKey.fromBytes(Buffer.alloc(32, 3)); const skArr = [sk0, sk1, sk2]; const testCases: { name: string; @@ -397,7 +397,7 @@ describe("aggregateConsolidation", function () { expect(finalAttestation.aggregationBits.uint8Array).toEqual(new Uint8Array(expectedAggregationBits)); expect(finalAttestation.committeeBits.toBoolArray()).toEqual(expectedCommitteeBits); expect(finalAttestation.data).toEqual(attData); - expect(finalAttestation.signature).toEqual(bls.Signature.aggregate(sigArr).toBytes()); + expect(finalAttestation.signature).toEqual(aggregateSignatures(sigArr).toBytes()); }); } }); diff --git a/packages/params/src/forkName.ts b/packages/params/src/forkName.ts index e1f1b2eda73a..4a41f0637340 100644 --- a/packages/params/src/forkName.ts +++ b/packages/params/src/forkName.ts @@ -80,3 +80,16 @@ export const forkBlobs = exclude(forkAll, [ForkName.phase0, ForkName.altair, For export function isForkBlobs(fork: ForkName): fork is ForkBlobs { return isForkWithdrawals(fork) && fork !== ForkName.capella; } + +export type ForkPreElectra = ForkPreBlobs | ForkName.deneb; +export type ForkElectra = Exclude; +export const forkElectra = exclude(forkAll, [ + ForkName.phase0, + ForkName.altair, + ForkName.bellatrix, + ForkName.capella, + ForkName.deneb, +]); +export function isForkElectra(fork: ForkName): fork is ForkElectra { + return isForkBlobs(fork) && fork !== ForkName.deneb; +} diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 00e9d48346c1..e7fd5b976336 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -262,7 +262,7 @@ export const KZG_COMMITMENT_SUBTREE_INDEX0 = KZG_COMMITMENT_GINDEX0 - 2 ** KZG_C export const BLOBSIDECAR_FIXED_SIZE = ACTIVE_PRESET === PresetName.minimal ? 131672 : 131928; // Electra Misc -export const UNSET_DEPOSIT_RECEIPTS_START_INDEX = 2n ** 64n - 1n; +export const UNSET_DEPOSIT_REQUESTS_START_INDEX = 2n ** 64n - 1n; export const FULL_EXIT_REQUEST_AMOUNT = 0; export const NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA = 87; export const NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA = 6; diff --git a/packages/params/test/unit/__snapshots__/forkName.test.ts.snap b/packages/params/test/unit/__snapshots__/forkName.test.ts.snap index 3d7009a7c2ec..a54f9dd6913b 100644 --- a/packages/params/test/unit/__snapshots__/forkName.test.ts.snap +++ b/packages/params/test/unit/__snapshots__/forkName.test.ts.snap @@ -7,12 +7,14 @@ exports[`forkName > should have valid allForks 1`] = ` "bellatrix", "capella", "deneb", + "electra", ] `; exports[`forkName > should have valid blobs forks 1`] = ` [ "deneb", + "electra", ] `; @@ -21,6 +23,7 @@ exports[`forkName > should have valid execution forks 1`] = ` "bellatrix", "capella", "deneb", + "electra", ] `; @@ -30,6 +33,7 @@ exports[`forkName > should have valid lightclient forks 1`] = ` "bellatrix", "capella", "deneb", + "electra", ] `; @@ -37,5 +41,6 @@ exports[`forkName > should have valid withdrawal forks 1`] = ` [ "capella", "deneb", + "electra", ] `; diff --git a/packages/state-transition/src/block/processDeposit.ts b/packages/state-transition/src/block/processDeposit.ts index bc140b864ad9..7e343d7fc33d 100644 --- a/packages/state-transition/src/block/processDeposit.ts +++ b/packages/state-transition/src/block/processDeposit.ts @@ -172,7 +172,7 @@ function isValidDepositSignature( const publicKey = PublicKey.fromBytes(pubkey, true); const signature = Signature.fromBytes(depositSignature, true); - return verify(signingRoot, publicKey, signature) + return verify(signingRoot, publicKey, signature); } catch (e) { return false; // Catch all BLS errors: failed key validation, failed signature validation, invalid signature } diff --git a/packages/state-transition/src/block/processDepositRequest.ts b/packages/state-transition/src/block/processDepositRequest.ts index ca6fe4206188..e5dd99a40c4e 100644 --- a/packages/state-transition/src/block/processDepositRequest.ts +++ b/packages/state-transition/src/block/processDepositRequest.ts @@ -1,5 +1,5 @@ import {electra} from "@lodestar/types"; -import {ForkSeq, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {ForkSeq, UNSET_DEPOSIT_REQUESTS_START_INDEX} from "@lodestar/params"; import {CachedBeaconStateElectra} from "../types.js"; import {applyDeposit} from "./processDeposit.js"; @@ -9,7 +9,7 @@ export function processDepositRequest( state: CachedBeaconStateElectra, depositRequest: electra.DepositRequest ): void { - if (state.depositRequestsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + if (state.depositRequestsStartIndex === UNSET_DEPOSIT_REQUESTS_START_INDEX) { state.depositRequestsStartIndex = BigInt(depositRequest.index); } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 7031aaa76fce..974f0d6a608b 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -1,5 +1,5 @@ import {Epoch, ValidatorIndex, ssz} from "@lodestar/types"; -import {FAR_FUTURE_EPOCH, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {FAR_FUTURE_EPOCH, UNSET_DEPOSIT_REQUESTS_START_INDEX} from "@lodestar/params"; import {CachedBeaconStateDeneb} from "../types.js"; import {CachedBeaconStateElectra, getCachedBeaconState} from "../cache/stateCache.js"; import { @@ -59,8 +59,8 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache stateElectraView.historicalSummaries = stateElectraCloned.historicalSummaries; // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default - // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX - stateElectraView.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + // default value of depositRequestsStartIndex is UNSET_DEPOSIT_REQUESTS_START_INDEX + stateElectraView.depositRequestsStartIndex = UNSET_DEPOSIT_REQUESTS_START_INDEX; stateElectraView.depositBalanceToConsume = BigInt(0); stateElectraView.exitBalanceToConsume = BigInt(0); @@ -138,8 +138,8 @@ export function upgradeStateToElectraOriginal(stateDeneb: CachedBeaconStateDeneb }); // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default - // default value of depositRequestsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX - stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + // default value of depositRequestsStartIndex is UNSET_DEPOSIT_REQUESTS_START_INDEX + stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_REQUESTS_START_INDEX; const validatorsArr = stateElectra.validators.getAllReadonly(); diff --git a/packages/state-transition/src/util/electra.ts b/packages/state-transition/src/util/electra.ts index 53f53aaf8d61..666f45435eb1 100644 --- a/packages/state-transition/src/util/electra.ts +++ b/packages/state-transition/src/util/electra.ts @@ -28,15 +28,15 @@ export function isFullyWithdrawableValidator( balance: number, epoch: number ): boolean { - const {withdrawableEpoch, withdrawalCredentials: withdrawalCredential} = validatorCredential; + const {withdrawableEpoch, withdrawalCredentials} = validatorCredential; if (fork < ForkSeq.capella) { throw new Error(`isFullyWithdrawableValidator not supported at forkSeq=${fork} < ForkSeq.capella`); } const hasWithdrawableCredentials = fork >= ForkSeq.electra - ? hasExecutionWithdrawalCredential(withdrawalCredential) - : hasEth1WithdrawalCredential(withdrawalCredential); + ? hasExecutionWithdrawalCredential(withdrawalCredentials) + : hasEth1WithdrawalCredential(withdrawalCredentials); return hasWithdrawableCredentials && withdrawableEpoch <= epoch && balance > 0; } @@ -46,18 +46,18 @@ export function isPartiallyWithdrawableValidator( validatorCredential: ValidatorInfo, balance: number ): boolean { - const {effectiveBalance, withdrawalCredentials: withdrawalCredential} = validatorCredential; + const {effectiveBalance, withdrawalCredentials} = validatorCredential; if (fork < ForkSeq.capella) { throw new Error(`isPartiallyWithdrawableValidator not supported at forkSeq=${fork} < ForkSeq.capella`); } const hasWithdrawableCredentials = fork >= ForkSeq.electra - ? hasExecutionWithdrawalCredential(withdrawalCredential) - : hasEth1WithdrawalCredential(withdrawalCredential); + ? hasExecutionWithdrawalCredential(withdrawalCredentials) + : hasEth1WithdrawalCredential(withdrawalCredentials); const validatorMaxEffectiveBalance = - fork >= ForkSeq.electra ? getValidatorMaxEffectiveBalance(withdrawalCredential) : MAX_EFFECTIVE_BALANCE; + fork >= ForkSeq.electra ? getValidatorMaxEffectiveBalance(withdrawalCredentials) : MAX_EFFECTIVE_BALANCE; const hasMaxEffectiveBalance = effectiveBalance === validatorMaxEffectiveBalance; const hasExcessBalance = balance > validatorMaxEffectiveBalance; diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index d1f09a9796e1..7bdc420d6458 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -8,7 +8,7 @@ import { GENESIS_EPOCH, GENESIS_SLOT, MAX_EFFECTIVE_BALANCE, - UNSET_DEPOSIT_RECEIPTS_START_INDEX, + UNSET_DEPOSIT_REQUESTS_START_INDEX, } from "@lodestar/params"; import {Bytes32, phase0, Root, ssz, TimeSeconds} from "@lodestar/types"; @@ -312,7 +312,7 @@ export function initializeBeaconStateFromEth1( stateElectra.latestExecutionPayloadHeader = (executionPayloadHeader as CompositeViewDU) ?? ssz.electra.ExecutionPayloadHeader.defaultViewDU(); - stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_REQUESTS_START_INDEX; } state.commit(); diff --git a/packages/types/src/utils/typeguards.ts b/packages/types/src/utils/typeguards.ts index a41b45eec546..2af11159878c 100644 --- a/packages/types/src/utils/typeguards.ts +++ b/packages/types/src/utils/typeguards.ts @@ -1,4 +1,4 @@ -import {ForkBlobs, ForkExecution, ForkName} from "@lodestar/params"; +import {ForkBlobs, ForkExecution, ForkElectra} from "@lodestar/params"; import { BlockContents, SignedBeaconBlock, @@ -68,8 +68,6 @@ export function isSignedBlockContents( return (data as SignedBlockContents).kzgProofs !== undefined; } -export function isElectraAttestation( - attestation: Attestation -): attestation is Attestation { +export function isElectraAttestation(attestation: Attestation): attestation is Attestation { return (attestation as Attestation).committeeBits !== undefined; } diff --git a/yarn.lock b/yarn.lock index 50e79fc267f3..ecfcb2c5ee6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3050,6 +3050,13 @@ resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708" integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw== +"@types/buffer-xor@^2.0.0": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@types/buffer-xor/-/buffer-xor-2.0.2.tgz#d8c463583b8fbb322ea824562dc78a0c3cea2ca6" + integrity sha512-OqdCua7QCTupPnJgmyGJUpxWgbuOi0IMIVslXTSePS2o+qDrDB6f2Pg44zRyqhUA5GbFAf39U8z0+mH4WG0fLQ== + dependencies: + "@types/node" "*" + "@types/cacheable-request@^6.0.1": version "6.0.3" resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.3.tgz#a430b3260466ca7b5ca5bfd735693b36e7a9d183" @@ -4626,6 +4633,13 @@ buffer-xor@^1.0.3: resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= +buffer-xor@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-2.0.2.tgz#34f7c64f04c777a1f8aac5e661273bb9dd320289" + integrity sha512-eHslX0bin3GB+Lx2p7lEYRShRewuNZL3fUl4qlVJGGiwoPGftmt8JQgk2Y9Ji5/01TnVDo33E5b5O3vUB1HdqQ== + dependencies: + safe-buffer "^5.1.1" + buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz" From 5f9f223fb2716fd35beca18c4ec511e2c5c65986 Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Tue, 6 Aug 2024 03:53:04 +0800 Subject: [PATCH 080/145] feat: add Electra attestation V2 endpoints (#6951) * Initial commit * getAggregatedAttestationV2 * Lint * Fix minor flaw * Add publishAggregateAndProofsV2 * Fix spelling * Fix CI * Fix spec test * Clean up events api * Run against latest beacon api spec * Revert changes to emitted events * Update packages/api/src/beacon/routes/beacon/pool.ts Co-authored-by: Nico Flaig * Update packages/api/src/beacon/routes/beacon/pool.ts Co-authored-by: Nico Flaig * Update packages/api/src/beacon/routes/beacon/pool.ts Co-authored-by: Nico Flaig * Update packages/api/src/beacon/routes/beacon/pool.ts Co-authored-by: Nico Flaig * Address comment * Add api stub back * Add todos * Review PR * Fix rebase * Lint --------- Co-authored-by: Nico Flaig --- .../api/src/beacon/routes/beacon/block.ts | 27 ++- packages/api/src/beacon/routes/beacon/pool.ts | 201 +++++++++++++++--- packages/api/src/beacon/routes/events.ts | 11 +- packages/api/src/beacon/routes/validator.ts | 148 +++++++++---- .../api/test/unit/beacon/oapiSpec.test.ts | 4 +- .../api/test/unit/beacon/testData/beacon.ts | 25 ++- .../api/test/unit/beacon/testData/events.ts | 98 ++++----- .../test/unit/beacon/testData/validator.ts | 10 +- .../src/api/impl/beacon/blocks/index.ts | 8 + .../src/api/impl/beacon/pool/index.ts | 29 ++- .../src/api/impl/validator/index.ts | 13 +- .../beacon-node/src/chain/blocks/types.ts | 2 +- .../src/chain/opPools/attestationPool.ts | 3 +- .../beacon-node/src/chain/opPools/opPool.ts | 1 + .../src/chain/validation/aggregateAndProof.ts | 2 +- .../src/network/processor/gossipHandlers.ts | 11 +- .../test/unit/api/impl/events/events.test.ts | 6 +- packages/flare/src/cmds/selfSlashAttester.ts | 2 +- .../validator/src/services/attestation.ts | 7 +- .../test/unit/services/attestation.test.ts | 16 +- packages/validator/test/utils/apiStub.ts | 3 + 21 files changed, 448 insertions(+), 179 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index 99fbf7d45645..2c141e3bdefd 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -8,13 +8,13 @@ import { deneb, isSignedBlockContents, SignedBeaconBlock, - BeaconBlockBody, SignedBeaconBlockOrContents, SignedBlindedBeaconBlock, SignedBlockContents, sszTypesFor, + BeaconBlockBody, } from "@lodestar/types"; -import {ForkName, ForkPreExecution, isForkBlobs, isForkExecution} from "@lodestar/params"; +import {ForkName, ForkPreElectra, ForkPreExecution, isForkBlobs, isForkExecution} from "@lodestar/params"; import {Endpoint, RequestCodec, RouteDefinitions, Schema} from "../../../utils/index.js"; import {EmptyMeta, EmptyResponseCodec, EmptyResponseData, WithVersion} from "../../../utils/codecs.js"; import { @@ -101,10 +101,22 @@ export type Endpoints = { "GET", BlockArgs, {params: {block_id: string}}, - BeaconBlockBody["attestations"], + BeaconBlockBody["attestations"], ExecutionOptimisticAndFinalizedMeta >; + /** + * Get block attestations + * Retrieves attestation included in requested block. + */ + getBlockAttestationsV2: Endpoint< + "GET", + BlockArgs, + {params: {block_id: string}}, + BeaconBlockBody["attestations"], + ExecutionOptimisticFinalizedAndVersionMeta + >; + /** * Get block header * Retrieves block header for given block id. @@ -251,6 +263,15 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ssz[fork].BeaconBlockBody.fields.attestations), + meta: ExecutionOptimisticFinalizedAndVersionCodec, + }, + }, getBlockHeader: { url: "/eth/v1/beacon/headers/{block_id}", method: "GET", diff --git a/packages/api/src/beacon/routes/beacon/pool.ts b/packages/api/src/beacon/routes/beacon/pool.ts index 499e8af432d7..aa2c36d3bc92 100644 --- a/packages/api/src/beacon/routes/beacon/pool.ts +++ b/packages/api/src/beacon/routes/beacon/pool.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {ForkSeq} from "@lodestar/params"; -import {phase0, capella, CommitteeIndex, Slot, ssz} from "@lodestar/types"; +import {isForkElectra} from "@lodestar/params"; +import {phase0, capella, CommitteeIndex, Slot, ssz, electra, AttesterSlashing} from "@lodestar/types"; import {Schema, Endpoint, RouteDefinitions} from "../../../utils/index.js"; import { ArrayOf, @@ -23,7 +23,8 @@ import {fromHeaders} from "../../../utils/headers.js"; const AttestationListTypePhase0 = ArrayOf(ssz.phase0.Attestation); const AttestationListTypeElectra = ArrayOf(ssz.electra.Attestation); -const AttesterSlashingListType = ArrayOf(ssz.phase0.AttesterSlashing); +const AttesterSlashingListTypePhase0 = ArrayOf(ssz.phase0.AttesterSlashing); +const AttesterSlashingListTypeElectra = ArrayOf(ssz.electra.AttesterSlashing); const ProposerSlashingListType = ArrayOf(ssz.phase0.ProposerSlashing); const SignedVoluntaryExitListType = ArrayOf(ssz.phase0.SignedVoluntaryExit); const SignedBLSToExecutionChangeListType = ArrayOf(ssz.capella.SignedBLSToExecutionChange); @@ -32,7 +33,11 @@ const SyncCommitteeMessageListType = ArrayOf(ssz.altair.SyncCommitteeMessage); type AttestationListPhase0 = ValueOf; type AttestationListElectra = ValueOf; type AttestationList = AttestationListPhase0 | AttestationListElectra; -type AttesterSlashingList = ValueOf; + +type AttesterSlashingListPhase0 = ValueOf; +type AttesterSlashingListElectra = ValueOf; +type AttesterSlashingList = AttesterSlashingListPhase0 | AttesterSlashingListElectra; + type ProposerSlashingList = ValueOf; type SignedVoluntaryExitList = ValueOf; type SignedBLSToExecutionChangeList = ValueOf; @@ -44,6 +49,18 @@ export type Endpoints = { * Retrieves attestations known by the node but not necessarily incorporated into any block */ getPoolAttestations: Endpoint< + "GET", + {slot?: Slot; committeeIndex?: CommitteeIndex}, + {query: {slot?: number; committee_index?: number}}, + AttestationListPhase0, + EmptyMeta + >; + + /** + * Get Attestations from operations pool + * Retrieves attestations known by the node but not necessarily incorporated into any block + */ + getPoolAttestationsV2: Endpoint< "GET", {slot?: Slot; committeeIndex?: CommitteeIndex}, {query: {slot?: number; committee_index?: number}}, @@ -60,10 +77,23 @@ export type Endpoints = { "GET", EmptyArgs, EmptyRequest, - AttesterSlashingList, + AttesterSlashingListPhase0, EmptyMeta >; + /** + * Get AttesterSlashings from operations pool + * Retrieves attester slashings known by the node but not necessarily incorporated into any block + */ + getPoolAttesterSlashingsV2: Endpoint< + // ⏎ + "GET", + EmptyArgs, + EmptyRequest, + AttesterSlashingList, + VersionMeta + >; + /** * Get ProposerSlashings from operations pool * Retrieves proposer slashings known by the node but not necessarily incorporated into any block @@ -112,6 +142,22 @@ export type Endpoints = { * If one or more attestations fail validation the node MUST return a 400 error with details of which attestations have failed, and why. */ submitPoolAttestations: Endpoint< + "POST", + {signedAttestations: AttestationListPhase0}, + {body: unknown}, + EmptyResponseData, + EmptyMeta + >; + + /** + * Submit Attestation objects to node + * Submits Attestation objects to the node. Each attestation in the request body is processed individually. + * + * If an attestation is validated successfully the node MUST publish that attestation on the appropriate subnet. + * + * If one or more attestations fail validation the node MUST return a 400 error with details of which attestations have failed, and why. + */ + submitPoolAttestationsV2: Endpoint< "POST", {signedAttestations: AttestationList}, {body: unknown; headers: {[MetaHeader.Version]: string}}, @@ -131,6 +177,18 @@ export type Endpoints = { EmptyMeta >; + /** + * Submit AttesterSlashing object to node's pool + * Submits AttesterSlashing object to node's pool and if passes validation node MUST broadcast it to network. + */ + submitPoolAttesterSlashingsV2: Endpoint< + "POST", + {attesterSlashing: AttesterSlashing}, + {body: unknown; headers: {[MetaHeader.Version]: string}}, + EmptyResponseData, + EmptyMeta + >; + /** * Submit ProposerSlashing object to node's pool * Submits ProposerSlashing object to node's pool and if passes validation node MUST broadcast it to network. @@ -191,9 +249,20 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions - ForkSeq[fork] >= ForkSeq.electra ? AttestationListTypeElectra : AttestationListTypePhase0 - ), + data: AttestationListTypePhase0, + meta: EmptyMetaCodec, + }, + }, + getPoolAttestationsV2: { + url: "/eth/v2/beacon/pool/attestations", + method: "GET", + req: { + writeReq: ({slot, committeeIndex}) => ({query: {slot, committee_index: committeeIndex}}), + parseReq: ({query}) => ({slot: query.slot, committeeIndex: query.committee_index}), + schema: {query: {slot: Schema.Uint, committee_index: Schema.Uint}}, + }, + resp: { + data: WithVersion((fork) => (isForkElectra(fork) ? AttestationListTypeElectra : AttestationListTypePhase0)), meta: VersionCodec, }, }, @@ -202,10 +271,21 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions + isForkElectra(fork) ? AttesterSlashingListTypeElectra : AttesterSlashingListTypePhase0 + ), + meta: VersionCodec, + }, + }, getPoolProposerSlashings: { url: "/eth/v1/beacon/pool/proposer_slashings", method: "GET", @@ -236,49 +316,55 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ + body: AttestationListTypePhase0.toJson(signedAttestations), + }), + parseReqJson: ({body}) => ({signedAttestations: AttestationListTypePhase0.fromJson(body)}), + writeReqSsz: ({signedAttestations}) => ({body: AttestationListTypePhase0.serialize(signedAttestations)}), + parseReqSsz: ({body}) => ({signedAttestations: AttestationListTypePhase0.deserialize(body)}), + schema: { + body: Schema.ObjectArray, + }, + }, + resp: EmptyResponseCodec, + }, + submitPoolAttestationsV2: { + url: "/eth/v2/beacon/pool/attestations", + method: "POST", req: { writeReqJson: ({signedAttestations}) => { - const fork = config.getForkName(signedAttestations[0].data.slot); + const fork = config.getForkName(signedAttestations[0]?.data.slot ?? 0); return { - body: - ForkSeq[fork] >= ForkSeq.electra - ? AttestationListTypeElectra.toJson(signedAttestations as AttestationListElectra) - : AttestationListTypePhase0.toJson(signedAttestations as AttestationListPhase0), + body: isForkElectra(fork) + ? AttestationListTypeElectra.toJson(signedAttestations as AttestationListElectra) + : AttestationListTypePhase0.toJson(signedAttestations as AttestationListPhase0), headers: {[MetaHeader.Version]: fork}, }; }, parseReqJson: ({body, headers}) => { - const versionHeader = fromHeaders(headers, MetaHeader.Version, false); - const fork = - versionHeader !== undefined - ? toForkName(versionHeader) - : config.getForkName(Number((body as {data: {slot: string}}[])[0]?.data.slot ?? 0)); - + const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAttestations: - ForkSeq[fork] >= ForkSeq.electra - ? AttestationListTypeElectra.fromJson(body) - : AttestationListTypePhase0.fromJson(body), + signedAttestations: isForkElectra(fork) + ? AttestationListTypeElectra.fromJson(body) + : AttestationListTypePhase0.fromJson(body), }; }, writeReqSsz: ({signedAttestations}) => { - const fork = config.getForkName(signedAttestations[0].data.slot); + const fork = config.getForkName(signedAttestations[0]?.data.slot ?? 0); return { - body: - ForkSeq[fork] >= ForkSeq.electra - ? AttestationListTypeElectra.serialize(signedAttestations as AttestationListElectra) - : AttestationListTypePhase0.serialize(signedAttestations as AttestationListPhase0), + body: isForkElectra(fork) + ? AttestationListTypeElectra.serialize(signedAttestations as AttestationListElectra) + : AttestationListTypePhase0.serialize(signedAttestations as AttestationListPhase0), headers: {[MetaHeader.Version]: fork}, }; }, parseReqSsz: ({body, headers}) => { - const versionHeader = fromHeaders(headers, MetaHeader.Version, true); - const fork = toForkName(versionHeader); + const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAttestations: - ForkSeq[fork] >= ForkSeq.electra - ? AttestationListTypeElectra.deserialize(body) - : AttestationListTypePhase0.deserialize(body), + signedAttestations: isForkElectra(fork) + ? AttestationListTypeElectra.deserialize(body) + : AttestationListTypePhase0.deserialize(body), }; }, schema: { @@ -302,6 +388,51 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { + const fork = config.getForkName(Number(attesterSlashing.attestation1.data.slot)); + return { + body: isForkElectra(fork) + ? ssz.electra.AttesterSlashing.toJson(attesterSlashing) + : ssz.phase0.AttesterSlashing.toJson(attesterSlashing), + headers: {[MetaHeader.Version]: fork}, + }; + }, + parseReqJson: ({body, headers}) => { + const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); + return { + attesterSlashing: isForkElectra(fork) + ? ssz.electra.AttesterSlashing.fromJson(body) + : ssz.phase0.AttesterSlashing.fromJson(body), + }; + }, + writeReqSsz: ({attesterSlashing}) => { + const fork = config.getForkName(Number(attesterSlashing.attestation1.data.slot)); + return { + body: isForkElectra(fork) + ? ssz.electra.AttesterSlashing.serialize(attesterSlashing as electra.AttesterSlashing) + : ssz.electra.AttesterSlashing.serialize(attesterSlashing as phase0.AttesterSlashing), + headers: {[MetaHeader.Version]: fork}, + }; + }, + parseReqSsz: ({body, headers}) => { + const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); + return { + attesterSlashing: isForkElectra(fork) + ? ssz.electra.AttesterSlashing.deserialize(body) + : ssz.phase0.AttesterSlashing.deserialize(body), + }; + }, + schema: { + body: Schema.Object, + headers: {[MetaHeader.Version]: Schema.String}, + }, + }, + resp: EmptyResponseCodec, + }, submitPoolProposerSlashings: { url: "/eth/v1/beacon/pool/proposer_slashings", method: "POST", diff --git a/packages/api/src/beacon/routes/events.ts b/packages/api/src/beacon/routes/events.ts index e15790fd3bea..23be5e7c2288 100644 --- a/packages/api/src/beacon/routes/events.ts +++ b/packages/api/src/beacon/routes/events.ts @@ -13,9 +13,6 @@ import { LightClientOptimisticUpdate, LightClientFinalityUpdate, SSEPayloadAttributes, - Attestation, - AttesterSlashing, - sszTypesFor, } from "@lodestar/types"; import {ForkName} from "@lodestar/params"; @@ -107,10 +104,10 @@ export type EventData = { block: RootHex; executionOptimistic: boolean; }; - [EventType.attestation]: {version: ForkName; data: Attestation}; + [EventType.attestation]: phase0.Attestation; [EventType.voluntaryExit]: phase0.SignedVoluntaryExit; [EventType.proposerSlashing]: phase0.ProposerSlashing; - [EventType.attesterSlashing]: {version: ForkName; data: AttesterSlashing}; + [EventType.attesterSlashing]: phase0.AttesterSlashing; [EventType.blsToExecutionChange]: capella.SignedBLSToExecutionChange; [EventType.finalizedCheckpoint]: { block: RootHex; @@ -228,10 +225,10 @@ export function getTypeByEvent(): {[K in EventType]: TypeJson} { {jsonCase: "eth2"} ), - [EventType.attestation]: WithVersion((fork) => sszTypesFor(fork).Attestation), + [EventType.attestation]: ssz.phase0.Attestation, [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit, [EventType.proposerSlashing]: ssz.phase0.ProposerSlashing, - [EventType.attesterSlashing]: WithVersion((fork) => sszTypesFor(fork).AttesterSlashing), + [EventType.attesterSlashing]: ssz.phase0.AttesterSlashing, [EventType.blsToExecutionChange]: ssz.capella.SignedBLSToExecutionChange, [EventType.finalizedCheckpoint]: new ContainerType( diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index c0078c97d5f8..ed7c0fd9ba0a 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ContainerType, fromHexString, toHexString, Type, ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {isForkBlobs, ForkSeq} from "@lodestar/params"; +import {isForkBlobs, isForkElectra} from "@lodestar/params"; import { altair, BLSSignature, @@ -203,7 +203,7 @@ export const AttesterDutyListType = ArrayOf(AttesterDutyType); export const ProposerDutyListType = ArrayOf(ProposerDutyType); export const SyncDutyListType = ArrayOf(SyncDutyType); export const SignedAggregateAndProofListPhase0Type = ArrayOf(ssz.phase0.SignedAggregateAndProof); -export const SignedAggregateAndProofListElectaType = ArrayOf(ssz.electra.SignedAggregateAndProof); +export const SignedAggregateAndProofListElectraType = ArrayOf(ssz.electra.SignedAggregateAndProof); export const SignedContributionAndProofListType = ArrayOf(ssz.altair.SignedContributionAndProof); export const BeaconCommitteeSubscriptionListType = ArrayOf(BeaconCommitteeSubscriptionType); export const SyncCommitteeSubscriptionListType = ArrayOf(SyncCommitteeSubscriptionType); @@ -221,8 +221,8 @@ export type ProposerDutyList = ValueOf; export type SyncDuty = ValueOf; export type SyncDutyList = ValueOf; export type SignedAggregateAndProofListPhase0 = ValueOf; -export type SignedAggregateAndProofListElecta = ValueOf; -export type SignedAggregateAndProofList = SignedAggregateAndProofListPhase0 | SignedAggregateAndProofListElecta; +export type SignedAggregateAndProofListElectra = ValueOf; +export type SignedAggregateAndProofList = SignedAggregateAndProofListPhase0 | SignedAggregateAndProofListElectra; export type SignedContributionAndProofList = ValueOf; export type BeaconCommitteeSubscription = ValueOf; export type BeaconCommitteeSubscriptionList = ValueOf; @@ -398,6 +398,23 @@ export type Endpoints = { * Returns an aggregated `Attestation` object with same `AttestationData` root. */ getAggregatedAttestation: Endpoint< + "GET", + { + /** HashTreeRoot of AttestationData that validator want's aggregated */ + attestationDataRoot: Root; + slot: Slot; + }, + {query: {attestation_data_root: string; slot: number}}, + phase0.Attestation, + EmptyMeta + >; + + /** + * Get aggregated attestation + * Aggregates all attestations matching given attestation data root, slot and committee index + * Returns an aggregated `Attestation` object with same `AttestationData` root. + */ + getAggregatedAttestationV2: Endpoint< "GET", { /** HashTreeRoot of AttestationData that validator want's aggregated */ @@ -405,7 +422,7 @@ export type Endpoints = { slot: Slot; committeeIndex: number; }, - {query: {attestation_data_root: string; slot: number; committeeIndex: number}}, + {query: {attestation_data_root: string; slot: number; committee_index: number}}, Attestation, VersionMeta >; @@ -415,6 +432,18 @@ export type Endpoints = { * Verifies given aggregate and proofs and publishes them on appropriate gossipsub topic. */ publishAggregateAndProofs: Endpoint< + "POST", + {signedAggregateAndProofs: SignedAggregateAndProofListPhase0}, + {body: unknown}, + EmptyResponseData, + EmptyMeta + >; + + /** + * Publish multiple aggregate and proofs + * Verifies given aggregate and proofs and publishes them on appropriate gossipsub topic. + */ + publishAggregateAndProofsV2: Endpoint< "POST", {signedAggregateAndProofs: SignedAggregateAndProofList}, {body: unknown; headers: {[MetaHeader.Version]: string}}, @@ -799,88 +828,113 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ + query: {attestation_data_root: toHexString(attestationDataRoot), slot}, + }), + parseReq: ({query}) => ({ + attestationDataRoot: fromHexString(query.attestation_data_root), + slot: query.slot, + }), + schema: { + query: { + attestation_data_root: Schema.StringRequired, + slot: Schema.UintRequired, + }, + }, + }, + resp: { + data: ssz.phase0.Attestation, + meta: EmptyMetaCodec, + }, + }, + getAggregatedAttestationV2: { + url: "/eth/v2/validator/aggregate_attestation", + method: "GET", req: { writeReq: ({attestationDataRoot, slot, committeeIndex}) => ({ - query: {attestation_data_root: toHexString(attestationDataRoot), slot, committeeIndex}, + query: {attestation_data_root: toHexString(attestationDataRoot), slot, committee_index: committeeIndex}, }), parseReq: ({query}) => ({ attestationDataRoot: fromHexString(query.attestation_data_root), slot: query.slot, - committeeIndex: query.committeeIndex, + committeeIndex: query.committee_index, }), schema: { query: { attestation_data_root: Schema.StringRequired, slot: Schema.UintRequired, - committeeIndex: Schema.UintRequired, + committee_index: Schema.UintRequired, }, }, }, resp: { - data: WithVersion((fork) => - ForkSeq[fork] >= ForkSeq.electra ? ssz.electra.Attestation : ssz.phase0.Attestation - ), + data: WithVersion((fork) => (isForkElectra(fork) ? ssz.electra.Attestation : ssz.phase0.Attestation)), meta: VersionCodec, }, }, publishAggregateAndProofs: { url: "/eth/v1/validator/aggregate_and_proofs", method: "POST", + req: { + writeReqJson: ({signedAggregateAndProofs}) => ({ + body: SignedAggregateAndProofListPhase0Type.toJson(signedAggregateAndProofs), + }), + parseReqJson: ({body}) => ({signedAggregateAndProofs: SignedAggregateAndProofListPhase0Type.fromJson(body)}), + writeReqSsz: ({signedAggregateAndProofs}) => ({ + body: SignedAggregateAndProofListPhase0Type.serialize(signedAggregateAndProofs), + }), + parseReqSsz: ({body}) => ({signedAggregateAndProofs: SignedAggregateAndProofListPhase0Type.deserialize(body)}), + schema: { + body: Schema.ObjectArray, + }, + }, + resp: EmptyResponseCodec, + }, + publishAggregateAndProofsV2: { + url: "/eth/v2/validator/aggregate_and_proofs", + method: "POST", req: { writeReqJson: ({signedAggregateAndProofs}) => { const fork = config.getForkName(signedAggregateAndProofs[0]?.message.aggregate.data.slot ?? 0); return { - body: - ForkSeq[fork] >= ForkSeq.electra - ? SignedAggregateAndProofListElectaType.toJson( - signedAggregateAndProofs as SignedAggregateAndProofListElecta - ) - : SignedAggregateAndProofListPhase0Type.toJson( - signedAggregateAndProofs as SignedAggregateAndProofListPhase0 - ), + body: isForkElectra(fork) + ? SignedAggregateAndProofListElectraType.toJson( + signedAggregateAndProofs as SignedAggregateAndProofListElectra + ) + : SignedAggregateAndProofListPhase0Type.toJson( + signedAggregateAndProofs as SignedAggregateAndProofListPhase0 + ), headers: {[MetaHeader.Version]: fork}, }; }, parseReqJson: ({body, headers}) => { - const versionHeader = fromHeaders(headers, MetaHeader.Version, false); - const fork = - versionHeader !== undefined - ? toForkName(versionHeader) - : config.getForkName( - Number( - (body as {message: {aggregate: {data: {slot: string}}}}[])[0]?.message.aggregate.data.slot ?? 0 - ) - ); - + const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAggregateAndProofs: - ForkSeq[fork] >= ForkSeq.electra - ? SignedAggregateAndProofListElectaType.fromJson(body) - : SignedAggregateAndProofListPhase0Type.fromJson(body), + signedAggregateAndProofs: isForkElectra(fork) + ? SignedAggregateAndProofListElectraType.fromJson(body) + : SignedAggregateAndProofListPhase0Type.fromJson(body), }; }, writeReqSsz: ({signedAggregateAndProofs}) => { const fork = config.getForkName(signedAggregateAndProofs[0]?.message.aggregate.data.slot ?? 0); return { - body: - ForkSeq[fork] >= ForkSeq.electra - ? SignedAggregateAndProofListElectaType.serialize( - signedAggregateAndProofs as SignedAggregateAndProofListElecta - ) - : SignedAggregateAndProofListPhase0Type.serialize( - signedAggregateAndProofs as SignedAggregateAndProofListPhase0 - ), + body: isForkElectra(fork) + ? SignedAggregateAndProofListElectraType.serialize( + signedAggregateAndProofs as SignedAggregateAndProofListElectra + ) + : SignedAggregateAndProofListPhase0Type.serialize( + signedAggregateAndProofs as SignedAggregateAndProofListPhase0 + ), headers: {[MetaHeader.Version]: fork}, }; }, parseReqSsz: ({body, headers}) => { - const versionHeader = fromHeaders(headers, MetaHeader.Version, true); - const fork = toForkName(versionHeader); + const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAggregateAndProofs: - ForkSeq[fork] >= ForkSeq.electra - ? SignedAggregateAndProofListElectaType.deserialize(body) - : SignedAggregateAndProofListPhase0Type.deserialize(body), + signedAggregateAndProofs: isForkElectra(fork) + ? SignedAggregateAndProofListElectraType.deserialize(body) + : SignedAggregateAndProofListPhase0Type.deserialize(body), }; }, schema: { diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index 00f1a3d8731d..2ccf4720db27 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -21,9 +21,9 @@ import {testData as validatorTestData} from "./testData/validator.js"; // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const version = "v2.5.0"; +const version = "v2.6.0-alpha.1"; const openApiFile: OpenApiFile = { - url: `https://github.com/ethereum/beacon-APIs/releases/download/${version}/beacon-node-oapi.json`, + url: `https://raw.githubusercontent.com/nflaig/beacon-api-spec/main/${version}/beacon-node-oapi.json`, filepath: path.join(__dirname, "../../../oapi-schemas/beacon-node-oapi.json"), version: RegExp(version), }; diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index da13d4c7783c..ac47a58777eb 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -49,6 +49,13 @@ export const testData: GenericServerTestCases = { args: {blockId: "head"}, res: {data: [ssz.phase0.Attestation.defaultValue()], meta: {executionOptimistic: true, finalized: false}}, }, + getBlockAttestationsV2: { + args: {blockId: "head"}, + res: { + data: [ssz.electra.Attestation.defaultValue()], + meta: {executionOptimistic: true, finalized: false, version: ForkName.electra}, + }, + }, getBlockHeader: { args: {blockId: "head"}, res: {data: blockHeaderResponse, meta: {executionOptimistic: true, finalized: false}}, @@ -92,12 +99,20 @@ export const testData: GenericServerTestCases = { getPoolAttestations: { args: {slot: 1, committeeIndex: 2}, - res: {data: [ssz.phase0.Attestation.defaultValue()], meta: {version: ForkName.deneb}}, + res: {data: [ssz.phase0.Attestation.defaultValue()]}, + }, + getPoolAttestationsV2: { + args: {slot: 1, committeeIndex: 2}, + res: {data: [ssz.electra.Attestation.defaultValue()], meta: {version: ForkName.electra}}, }, getPoolAttesterSlashings: { args: undefined, res: {data: [ssz.phase0.AttesterSlashing.defaultValue()]}, }, + getPoolAttesterSlashingsV2: { + args: undefined, + res: {data: [ssz.electra.AttesterSlashing.defaultValue()], meta: {version: ForkName.electra}}, + }, getPoolProposerSlashings: { args: undefined, res: {data: [ssz.phase0.ProposerSlashing.defaultValue()]}, @@ -114,10 +129,18 @@ export const testData: GenericServerTestCases = { args: {signedAttestations: [ssz.phase0.Attestation.defaultValue()]}, res: undefined, }, + submitPoolAttestationsV2: { + args: {signedAttestations: [ssz.phase0.Attestation.defaultValue()]}, + res: undefined, + }, submitPoolAttesterSlashings: { args: {attesterSlashing: ssz.phase0.AttesterSlashing.defaultValue()}, res: undefined, }, + submitPoolAttesterSlashingsV2: { + args: {attesterSlashing: ssz.electra.AttesterSlashing.defaultValue()}, + res: undefined, + }, submitPoolProposerSlashings: { args: {proposerSlashing: ssz.phase0.ProposerSlashing.defaultValue()}, res: undefined, diff --git a/packages/api/test/unit/beacon/testData/events.ts b/packages/api/test/unit/beacon/testData/events.ts index 7966d0ea3b8a..8a7610a26836 100644 --- a/packages/api/test/unit/beacon/testData/events.ts +++ b/packages/api/test/unit/beacon/testData/events.ts @@ -31,21 +31,18 @@ export const eventTestData: EventData = { block: "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf", executionOptimistic: false, }, - [EventType.attestation]: { - version: ForkName.altair, - data: ssz.phase0.Attestation.fromJson({ - aggregation_bits: "0x01", - signature: - "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", - data: { - slot: "1", - index: "1", - beacon_block_root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", - source: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, - target: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, - }, - }), - }, + [EventType.attestation]: ssz.phase0.Attestation.fromJson({ + aggregation_bits: "0x01", + signature: + "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", + data: { + slot: "1", + index: "1", + beacon_block_root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", + source: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, + target: {epoch: "1", root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, + }, + }), [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit.fromJson({ message: {epoch: "1", validator_index: "1"}, signature: @@ -75,47 +72,44 @@ export const eventTestData: EventData = { "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", }, }), - [EventType.attesterSlashing]: { - version: ForkName.altair, - data: ssz.phase0.AttesterSlashing.fromJson({ - attestation_1: { - attesting_indices: ["0", "1"], - data: { - slot: "0", - index: "0", - beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", - source: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", - }, - target: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", - }, + [EventType.attesterSlashing]: ssz.phase0.AttesterSlashing.fromJson({ + attestation_1: { + attesting_indices: ["0", "1"], + data: { + slot: "0", + index: "0", + beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", + source: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + target: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", }, - signature: - "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", }, - attestation_2: { - attesting_indices: ["0", "1"], - data: { - slot: "0", - index: "0", - beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", - source: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", - }, - target: { - epoch: "0", - root: "0x0000000000000000000000000000000000000000000000000000000000000000", - }, + signature: + "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + }, + attestation_2: { + attesting_indices: ["0", "1"], + data: { + slot: "0", + index: "0", + beacon_block_root: "0x0000000000000000000000000000000000000000000000000000000000000000", + source: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + target: { + epoch: "0", + root: "0x0000000000000000000000000000000000000000000000000000000000000000", }, - signature: - "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", }, - }), - }, + signature: + "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + }, + }), [EventType.blsToExecutionChange]: ssz.capella.SignedBLSToExecutionChange.fromJson({ message: { validator_index: "1", diff --git a/packages/api/test/unit/beacon/testData/validator.ts b/packages/api/test/unit/beacon/testData/validator.ts index b27fc9c38733..d4fae4bfe290 100644 --- a/packages/api/test/unit/beacon/testData/validator.ts +++ b/packages/api/test/unit/beacon/testData/validator.ts @@ -99,13 +99,21 @@ export const testData: GenericServerTestCases = { res: {data: ssz.altair.SyncCommitteeContribution.defaultValue()}, }, getAggregatedAttestation: { + args: {attestationDataRoot: ZERO_HASH, slot: 32000}, + res: {data: ssz.phase0.Attestation.defaultValue()}, + }, + getAggregatedAttestationV2: { args: {attestationDataRoot: ZERO_HASH, slot: 32000, committeeIndex: 2}, - res: {data: ssz.phase0.Attestation.defaultValue(), meta: {version: ForkName.phase0}}, + res: {data: ssz.electra.Attestation.defaultValue(), meta: {version: ForkName.electra}}, }, publishAggregateAndProofs: { args: {signedAggregateAndProofs: [ssz.phase0.SignedAggregateAndProof.defaultValue()]}, res: undefined, }, + publishAggregateAndProofsV2: { + args: {signedAggregateAndProofs: [ssz.phase0.SignedAggregateAndProof.defaultValue()]}, + res: undefined, + }, publishContributionAndProofs: { args: {contributionAndProofs: [ssz.altair.SignedContributionAndProof.defaultValue()]}, res: undefined, diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 4d0f96d1ea08..251218f510ec 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -413,6 +413,14 @@ export function getBeaconBlockApi({ }; }, + async getBlockAttestationsV2({blockId}) { + const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); + return { + data: Array.from(block.message.body.attestations), + meta: {executionOptimistic, finalized, version: config.getForkName(block.message.slot)}, + }; + }, + async getBlockRoot({blockId}) { // Fast path: From head state already available in memory get historical blockRoot const slot = typeof blockId === "string" ? parseInt(blockId) : blockId; diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 19c11362c910..bc8c838a5401 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -27,7 +27,18 @@ export function getBeaconPoolApi({ async getPoolAttestations({slot, committeeIndex}) { // Already filtered by slot let attestations = chain.aggregatedAttestationPool.getAll(slot); - const fork = chain.config.getForkName(slot ?? attestations[0].data.slot) ?? ForkName.phase0; + + if (committeeIndex !== undefined) { + attestations = attestations.filter((attestation) => committeeIndex === attestation.data.index); + } + + return {data: attestations}; + }, + + async getPoolAttestationsV2({slot, committeeIndex}) { + // Already filtered by slot + let attestations = chain.aggregatedAttestationPool.getAll(slot); + const fork = chain.config.getForkName(slot ?? attestations[0]?.data.slot ?? chain.clock.currentSlot); if (committeeIndex !== undefined) { attestations = attestations.filter((attestation) => committeeIndex === attestation.data.index); @@ -40,6 +51,11 @@ export function getBeaconPoolApi({ return {data: chain.opPool.getAllAttesterSlashings()}; }, + async getPoolAttesterSlashingsV2() { + // TODO Electra: Determine fork based on data returned by api + return {data: chain.opPool.getAllAttesterSlashings(), meta: {version: ForkName.phase0}}; + }, + async getPoolProposerSlashings() { return {data: chain.opPool.getAllProposerSlashings()}; }, @@ -53,6 +69,10 @@ export function getBeaconPoolApi({ }, async submitPoolAttestations({signedAttestations}) { + await this.submitPoolAttestationsV2({signedAttestations}); + }, + + async submitPoolAttestationsV2({signedAttestations}) { const seenTimestampSec = Date.now() / 1000; const errors: Error[] = []; @@ -79,7 +99,7 @@ export function getBeaconPoolApi({ metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); } - chain.emitter.emit(routes.events.EventType.attestation, {data: attestation, version: ForkName.phase0}); + chain.emitter.emit(routes.events.EventType.attestation, attestation); const sentPeers = await network.publishBeaconAttestation(attestation, subnet); metrics?.onPoolSubmitUnaggregatedAttestation(seenTimestampSec, indexedAttestation, subnet, sentPeers); @@ -115,6 +135,11 @@ export function getBeaconPoolApi({ await network.publishAttesterSlashing(attesterSlashing); }, + async submitPoolAttesterSlashingsV2({attesterSlashing}) { + // TODO Electra: Refactor submitPoolAttesterSlashings and submitPoolAttesterSlashingsV2 + await this.submitPoolAttesterSlashings({attesterSlashing}); + }, + async submitPoolProposerSlashings({proposerSlashing}) { await validateApiProposerSlashing(chain, proposerSlashing); chain.opPool.insertProposerSlashing(proposerSlashing); diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 856b301d7a21..06502146ce2f 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1072,7 +1072,12 @@ export function getValidatorApi( }; }, - async getAggregatedAttestation({attestationDataRoot, slot, committeeIndex}) { + // TODO Electra: Implement getAggregatedAttestation to properly handle pre-electra + async getAggregatedAttestation() { + throw new Error("Not implemented. Use getAggregatedAttestationV2 for now."); + }, + + async getAggregatedAttestationV2({attestationDataRoot, slot, committeeIndex}) { notWhileSyncing(); await waitForSlot(slot); // Must never request for a future slot > currentSlot @@ -1083,7 +1088,7 @@ export function getValidatorApi( if (!aggregate) { throw new ApiError( 404, - `No aggregated attestation for slot=${slot} committeeIndex=${committeeIndex}, dataRoot=${dataRootHex}` + `No aggregated attestation for slot=${slot}, committeeIndex=${committeeIndex}, dataRoot=${dataRootHex}` ); } @@ -1096,6 +1101,10 @@ export function getValidatorApi( }, async publishAggregateAndProofs({signedAggregateAndProofs}) { + await this.publishAggregateAndProofsV2({signedAggregateAndProofs}); + }, + + async publishAggregateAndProofsV2({signedAggregateAndProofs}) { notWhileSyncing(); const seenTimestampSec = Date.now() / 1000; diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 80c94e2d6fde..8b793932e951 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -1,7 +1,7 @@ import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; import {MaybeValidExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {deneb, Slot, RootHex, SignedBeaconBlock} from "@lodestar/types"; -import {ForkSeq, ForkName, ForkBlobs} from "@lodestar/params"; +import {ForkSeq, ForkBlobs} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; export enum BlockInputType { diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index 8daae6b663c7..f125b8c941db 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -144,6 +144,7 @@ export class AttestationPool { /** * For validator API to get an aggregate */ + // TODO Electra: Change attestation pool to accomodate pre-electra request getAggregate(slot: Slot, committeeIndex: CommitteeIndex, dataRootHex: RootHex): Attestation | null { const aggregate = this.aggregateByIndexByRootBySlot.get(slot)?.get(dataRootHex)?.get(committeeIndex); if (!aggregate) { @@ -233,7 +234,7 @@ function attestationToAggregate(attestation: Attestation): AggregateFast { } /** - * Unwrap AggregateFast to phase0.Attestation + * Unwrap AggregateFast to Attestation */ function fastToAttestation(aggFast: AggregateFast): Attestation { return {...aggFast, signature: aggFast.signature.toBytes()}; diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index fc4558d9d995..a2180a718fee 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -285,6 +285,7 @@ export class OpPool { } /** For beacon pool API */ + // TODO Electra: Update to adapt electra.AttesterSlashing getAllAttesterSlashings(): phase0.AttesterSlashing[] { return Array.from(this.attesterSlashings.values()).map((attesterSlashings) => attesterSlashings.attesterSlashing); } diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 839f7850f75d..8034b998fdce 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -28,7 +28,7 @@ export type AggregateAndProofValidationResult = { export async function validateApiAggregateAndProof( fork: ForkName, chain: IBeaconChain, - signedAggregateAndProof: phase0.SignedAggregateAndProof + signedAggregateAndProof: SignedAggregateAndProof ): Promise { const skipValidationKnownAttesters = true; const prioritizeBls = true; diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 51fa7bd376c8..9e6f08a803c1 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -454,10 +454,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } } - chain.emitter.emit(routes.events.EventType.attestation, { - version: fork, - data: signedAggregateAndProof.message.aggregate, - }); + chain.emitter.emit(routes.events.EventType.attestation, signedAggregateAndProof.message.aggregate); }, [GossipType.beacon_attestation]: async ({ gossipData, @@ -509,7 +506,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } } - chain.emitter.emit(routes.events.EventType.attestation, {version: fork, data: attestation}); + chain.emitter.emit(routes.events.EventType.attestation, attestation); }, [GossipType.attester_slashing]: async ({ @@ -529,7 +526,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler logger.error("Error adding attesterSlashing to pool", {}, e as Error); } - chain.emitter.emit(routes.events.EventType.attesterSlashing, {version: topic.fork, data: attesterSlashing}); + chain.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing); }, [GossipType.proposer_slashing]: async ({ @@ -717,7 +714,7 @@ function getBatchHandlers(modules: ValidatorFnsModules, options: GossipHandlerOp } } - chain.emitter.emit(routes.events.EventType.attestation, {version: fork, data: attestation}); + chain.emitter.emit(routes.events.EventType.attestation, attestation); } if (batchableBls) { diff --git a/packages/beacon-node/test/unit/api/impl/events/events.test.ts b/packages/beacon-node/test/unit/api/impl/events/events.test.ts index b1f85b5e6e44..e031c3ac9958 100644 --- a/packages/beacon-node/test/unit/api/impl/events/events.test.ts +++ b/packages/beacon-node/test/unit/api/impl/events/events.test.ts @@ -2,7 +2,6 @@ import {describe, it, expect, beforeEach, afterEach, vi, MockedObject} from "vit import {routes} from "@lodestar/api"; import {config} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; -import {ForkName} from "@lodestar/params"; import {BeaconChain, ChainEventEmitter, HeadEventData} from "../../../../../src/chain/index.js"; import {getEventsApi} from "../../../../../src/api/impl/events/index.js"; import {ZERO_HASH_HEX} from "../../../../../src/constants/constants.js"; @@ -67,10 +66,7 @@ describe("Events api impl", function () { it("should ignore not sent topics", async function () { const events = getEvents([routes.events.EventType.head]); - chainEventEmmitter.emit(routes.events.EventType.attestation, { - version: ForkName.phase0, - data: ssz.phase0.Attestation.defaultValue(), - }); + chainEventEmmitter.emit(routes.events.EventType.attestation, ssz.phase0.Attestation.defaultValue()); chainEventEmmitter.emit(routes.events.EventType.head, headEventData); expect(events).toHaveLength(1); diff --git a/packages/flare/src/cmds/selfSlashAttester.ts b/packages/flare/src/cmds/selfSlashAttester.ts index 8b43e6a92cb0..cb29ed94bb74 100644 --- a/packages/flare/src/cmds/selfSlashAttester.ts +++ b/packages/flare/src/cmds/selfSlashAttester.ts @@ -130,7 +130,7 @@ export async function selfSlashAttesterHandler(args: SelfSlashArgs): Promise 0) { try { - (await this.api.validator.publishAggregateAndProofs({signedAggregateAndProofs})).assertOk(); + (await this.api.validator.publishAggregateAndProofsV2({signedAggregateAndProofs})).assertOk(); this.logger.info("Published aggregateAndProofs", {...logCtx, count: signedAggregateAndProofs.length}); this.metrics?.publishedAggregates.inc(signedAggregateAndProofs.length); } catch (e) { diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index fdfc34c857b9..591ac51fa05e 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -106,12 +106,12 @@ describe("AttestationService", function () { // Mock beacon's attestation and aggregates endpoints api.validator.produceAttestationData.mockResolvedValue(mockApiResponse({data: attestation.data})); - api.validator.getAggregatedAttestation.mockResolvedValue( + api.validator.getAggregatedAttestationV2.mockResolvedValue( mockApiResponse({data: attestation, meta: {version: ForkName.phase0}}) ); - api.beacon.submitPoolAttestations.mockResolvedValue(mockApiResponse({})); - api.validator.publishAggregateAndProofs.mockResolvedValue(mockApiResponse({})); + api.beacon.submitPoolAttestationsV2.mockResolvedValue(mockApiResponse({})); + api.validator.publishAggregateAndProofsV2.mockResolvedValue(mockApiResponse({})); if (opts.distributedAggregationSelection) { // Mock distributed validator middleware client selections endpoint @@ -153,12 +153,12 @@ describe("AttestationService", function () { } // Must submit the attestation received through produceAttestationData() - expect(api.beacon.submitPoolAttestations).toHaveBeenCalledOnce(); - expect(api.beacon.submitPoolAttestations).toHaveBeenCalledWith({signedAttestations: [attestation]}); + expect(api.beacon.submitPoolAttestationsV2).toHaveBeenCalledOnce(); + expect(api.beacon.submitPoolAttestationsV2).toHaveBeenCalledWith({signedAttestations: [attestation]}); - // Must submit the aggregate received through getAggregatedAttestation() then createAndSignAggregateAndProof() - expect(api.validator.publishAggregateAndProofs).toHaveBeenCalledOnce(); - expect(api.validator.publishAggregateAndProofs).toHaveBeenCalledWith({signedAggregateAndProofs: [aggregate]}); + // Must submit the aggregate received through getAggregatedAttestationV2() then createAndSignAggregateAndProof() + expect(api.validator.publishAggregateAndProofsV2).toHaveBeenCalledOnce(); + expect(api.validator.publishAggregateAndProofsV2).toHaveBeenCalledWith({signedAggregateAndProofs: [aggregate]}); }); }); } diff --git a/packages/validator/test/utils/apiStub.ts b/packages/validator/test/utils/apiStub.ts index 3c1d80fff75d..ef615af03369 100644 --- a/packages/validator/test/utils/apiStub.ts +++ b/packages/validator/test/utils/apiStub.ts @@ -20,6 +20,7 @@ export function getApiClientStub(): ApiClientStub { publishBlockV2: vi.fn(), submitPoolSyncCommitteeSignatures: vi.fn(), submitPoolAttestations: vi.fn(), + submitPoolAttestationsV2: vi.fn(), }, node: { getSyncingStatus: vi.fn(), @@ -36,7 +37,9 @@ export function getApiClientStub(): ApiClientStub { submitSyncCommitteeSelections: vi.fn(), produceAttestationData: vi.fn(), getAggregatedAttestation: vi.fn(), + getAggregatedAttestationV2: vi.fn(), publishAggregateAndProofs: vi.fn(), + publishAggregateAndProofsV2: vi.fn(), submitBeaconCommitteeSelections: vi.fn(), }, httpClient: httpClientStub, From e5cda9caf12168f9364be36afbdec653f1018082 Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:13:21 +0800 Subject: [PATCH 081/145] Update packages/beacon-node/src/chain/validation/aggregateAndProof.ts Co-authored-by: Cayman Update packages/types/src/utils/typeguards.ts Co-authored-by: Cayman Partial address comment Partial address comment Co-authored-by: Cayman --- packages/api/src/beacon/routes/beacon/pool.ts | 22 +++++++++---------- packages/api/src/beacon/routes/validator.ts | 12 +++++----- packages/beacon-node/src/chain/chain.ts | 3 +-- .../beacon-node/src/chain/genesis/genesis.ts | 8 +------ .../chain/stateCache/fifoBlockStateCache.ts | 2 +- .../stateCache/persistentCheckpointsCache.ts | 6 ++++- .../src/chain/validation/aggregateAndProof.ts | 2 +- .../src/eth1/eth1DepositDataTracker.ts | 2 +- .../src/metrics/metrics/lodestar.ts | 2 +- .../beacon-node/src/network/gossip/topic.ts | 3 +-- .../src/options/beaconNodeOptions/network.ts | 8 +++---- packages/cli/src/util/gitData/index.ts | 4 ++-- packages/config/src/forkConfig/index.ts | 8 +++---- packages/config/src/forkConfig/types.ts | 4 ++-- packages/params/src/forkName.ts | 6 ++--- packages/prover/src/utils/gitData/index.ts | 4 ++-- packages/state-transition/package.json | 4 ---- .../src/block/processConsolidationRequest.ts | 2 +- .../state-transition/src/cache/epochCache.ts | 16 +++++++------- .../epoch/processPendingBalanceDeposits.ts | 2 +- .../src/slot/upgradeStateToElectra.ts | 4 ++-- packages/state-transition/src/util/epoch.ts | 4 ++-- packages/state-transition/src/util/genesis.ts | 4 ++-- .../state-transition/src/util/validator.ts | 16 +++++++------- .../test/unit/util/deposit.test.ts | 6 ++--- packages/types/src/utils/typeguards.ts | 6 ++--- packages/utils/src/promise.ts | 2 +- .../validator/src/services/attestation.ts | 8 +++---- .../validator/src/services/validatorStore.ts | 6 ++--- 29 files changed, 84 insertions(+), 92 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/pool.ts b/packages/api/src/beacon/routes/beacon/pool.ts index aa2c36d3bc92..42880b276732 100644 --- a/packages/api/src/beacon/routes/beacon/pool.ts +++ b/packages/api/src/beacon/routes/beacon/pool.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {isForkElectra} from "@lodestar/params"; +import {isForkPostElectra} from "@lodestar/params"; import {phase0, capella, CommitteeIndex, Slot, ssz, electra, AttesterSlashing} from "@lodestar/types"; import {Schema, Endpoint, RouteDefinitions} from "../../../utils/index.js"; import { @@ -262,7 +262,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions (isForkElectra(fork) ? AttestationListTypeElectra : AttestationListTypePhase0)), + data: WithVersion((fork) => (isForkPostElectra(fork) ? AttestationListTypeElectra : AttestationListTypePhase0)), meta: VersionCodec, }, }, @@ -281,7 +281,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions - isForkElectra(fork) ? AttesterSlashingListTypeElectra : AttesterSlashingListTypePhase0 + isForkPostElectra(fork) ? AttesterSlashingListTypeElectra : AttesterSlashingListTypePhase0 ), meta: VersionCodec, }, @@ -336,7 +336,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = config.getForkName(signedAttestations[0]?.data.slot ?? 0); return { - body: isForkElectra(fork) + body: isForkPostElectra(fork) ? AttestationListTypeElectra.toJson(signedAttestations as AttestationListElectra) : AttestationListTypePhase0.toJson(signedAttestations as AttestationListPhase0), headers: {[MetaHeader.Version]: fork}, @@ -345,7 +345,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAttestations: isForkElectra(fork) + signedAttestations: isForkPostElectra(fork) ? AttestationListTypeElectra.fromJson(body) : AttestationListTypePhase0.fromJson(body), }; @@ -353,7 +353,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = config.getForkName(signedAttestations[0]?.data.slot ?? 0); return { - body: isForkElectra(fork) + body: isForkPostElectra(fork) ? AttestationListTypeElectra.serialize(signedAttestations as AttestationListElectra) : AttestationListTypePhase0.serialize(signedAttestations as AttestationListPhase0), headers: {[MetaHeader.Version]: fork}, @@ -362,7 +362,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAttestations: isForkElectra(fork) + signedAttestations: isForkPostElectra(fork) ? AttestationListTypeElectra.deserialize(body) : AttestationListTypePhase0.deserialize(body), }; @@ -395,7 +395,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = config.getForkName(Number(attesterSlashing.attestation1.data.slot)); return { - body: isForkElectra(fork) + body: isForkPostElectra(fork) ? ssz.electra.AttesterSlashing.toJson(attesterSlashing) : ssz.phase0.AttesterSlashing.toJson(attesterSlashing), headers: {[MetaHeader.Version]: fork}, @@ -404,7 +404,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - attesterSlashing: isForkElectra(fork) + attesterSlashing: isForkPostElectra(fork) ? ssz.electra.AttesterSlashing.fromJson(body) : ssz.phase0.AttesterSlashing.fromJson(body), }; @@ -412,7 +412,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = config.getForkName(Number(attesterSlashing.attestation1.data.slot)); return { - body: isForkElectra(fork) + body: isForkPostElectra(fork) ? ssz.electra.AttesterSlashing.serialize(attesterSlashing as electra.AttesterSlashing) : ssz.electra.AttesterSlashing.serialize(attesterSlashing as phase0.AttesterSlashing), headers: {[MetaHeader.Version]: fork}, @@ -421,7 +421,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - attesterSlashing: isForkElectra(fork) + attesterSlashing: isForkPostElectra(fork) ? ssz.electra.AttesterSlashing.deserialize(body) : ssz.phase0.AttesterSlashing.deserialize(body), }; diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index ed7c0fd9ba0a..5bc5c17c09b3 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ContainerType, fromHexString, toHexString, Type, ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {isForkBlobs, isForkElectra} from "@lodestar/params"; +import {isForkBlobs, isForkPostElectra} from "@lodestar/params"; import { altair, BLSSignature, @@ -869,7 +869,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions (isForkElectra(fork) ? ssz.electra.Attestation : ssz.phase0.Attestation)), + data: WithVersion((fork) => (isForkPostElectra(fork) ? ssz.electra.Attestation : ssz.phase0.Attestation)), meta: VersionCodec, }, }, @@ -898,7 +898,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = config.getForkName(signedAggregateAndProofs[0]?.message.aggregate.data.slot ?? 0); return { - body: isForkElectra(fork) + body: isForkPostElectra(fork) ? SignedAggregateAndProofListElectraType.toJson( signedAggregateAndProofs as SignedAggregateAndProofListElectra ) @@ -911,7 +911,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAggregateAndProofs: isForkElectra(fork) + signedAggregateAndProofs: isForkPostElectra(fork) ? SignedAggregateAndProofListElectraType.fromJson(body) : SignedAggregateAndProofListPhase0Type.fromJson(body), }; @@ -919,7 +919,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = config.getForkName(signedAggregateAndProofs[0]?.message.aggregate.data.slot ?? 0); return { - body: isForkElectra(fork) + body: isForkPostElectra(fork) ? SignedAggregateAndProofListElectraType.serialize( signedAggregateAndProofs as SignedAggregateAndProofListElectra ) @@ -932,7 +932,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); return { - signedAggregateAndProofs: isForkElectra(fork) + signedAggregateAndProofs: isForkPostElectra(fork) ? SignedAggregateAndProofListElectraType.deserialize(body) : SignedAggregateAndProofListPhase0Type.deserialize(body), }; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 9f30936305f0..598b26d7061f 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1120,11 +1120,10 @@ export class BeaconChain implements IBeaconChain { } const cpEpoch = cp.epoch; - const electraEpoch = headState?.config.ELECTRA_FORK_EPOCH ?? Infinity; if (headState === null) { this.logger.verbose("Head state is null"); - } else if (cpEpoch >= electraEpoch) { + } else if (cpEpoch >= this.config.ELECTRA_FORK_EPOCH) { // Get the validator.length from the state at cpEpoch // We are confident the last element in the list is from headEpoch // Thus we query from the end of the list. (cpEpoch - headEpoch - 1) is negative number diff --git a/packages/beacon-node/src/chain/genesis/genesis.ts b/packages/beacon-node/src/chain/genesis/genesis.ts index 5efd352357eb..979476c69530 100644 --- a/packages/beacon-node/src/chain/genesis/genesis.ts +++ b/packages/beacon-node/src/chain/genesis/genesis.ts @@ -174,13 +174,7 @@ export class GenesisBuilder implements IGenesisBuilder { }; }); - const {activatedValidatorCount} = applyDeposits( - this.config.getForkSeq(this.state.slot), - this.config, - this.state, - newDeposits, - this.depositTree - ); + const {activatedValidatorCount} = applyDeposits(this.config, this.state, newDeposits, this.depositTree); this.activatedValidatorCount += activatedValidatorCount; // TODO: If necessary persist deposits here to this.db.depositData, this.db.depositDataRoot diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index f115806874f8..d38fc323174c 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -191,7 +191,7 @@ export class FIFOBlockStateCache implements BlockStateCache { } getStates(): IterableIterator { - throw new Error("Method not implemented."); + return this.cache.values(); } /** diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index c59c15de616d..b315beba46d9 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -595,7 +595,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } getStates(): IterableIterator { - throw new Error("Method not implemented."); + const items = Array.from(this.cache.values()) + .filter(isInMemoryCacheItem) + .map((item) => item.state); + + return items.values(); } /** ONLY FOR DEBUGGING PURPOSES. For spec tests on error */ diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 8034b998fdce..02c6613b170f 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -180,7 +180,7 @@ async function validateAggregateAndProof( } const attestingIndices = aggregate.aggregationBits.intersectValues(committeeIndices); - const indexedAttestationContent = { + const indexedAttestationContent: IndexedAttestation = { attestingIndices, data: attData, signature: aggregate.signature, diff --git a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts index d0578718f29f..a38b3f9987d9 100644 --- a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts +++ b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts @@ -128,7 +128,7 @@ export class Eth1DepositDataTracker { */ async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { if ( - state.epochCtx.isAfterElectra() && + state.epochCtx.isPostElectra() && state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositRequestsStartIndex ) { // No need to poll eth1Data since Electra deprecates the mechanism after depositRequestsStartIndex is reached diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 595e4a7a45fc..6b4bdb111f41 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1397,7 +1397,7 @@ export function createLodestarMetrics( regenFnDeletePubkeyTime: register.histogram({ name: "lodestar_regen_fn_delete_pubkey_time_seconds", help: "Histrogram of time spent on deleting pubkeys from all state cache items in seconds", - buckets: [0.01, 0.1, 0.5, 1, 2, 5], + buckets: [0.01, 0.1, 0.5, 1], }), regenFnNumStatesUpdated: register.histogram({ name: "lodestar_regen_state_cache_state_updated_count", diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index dfca0eaf3dcd..f3968b8e65b6 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -129,9 +129,8 @@ export function sszDeserialize(topic: T, serializedData: * Deserialize a gossip serialized data into an Attestation object. */ export function sszDeserializeAttestation(fork: ForkName, serializedData: Uint8Array): Attestation { - const sszType = sszTypesFor(fork).Attestation; try { - return sszType.deserialize(serializedData); + return sszTypesFor(fork).Attestation.deserialize(serializedData); } catch (e) { throw new GossipActionError(GossipAction.REJECT, {code: GossipErrorCode.INVALID_SERIALIZED_BYTES_ERROR_CODE}); } diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 25ba036a5dbf..6c6f1ed70cc6 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -63,13 +63,13 @@ export function parseListenArgs(args: NetworkArgs) { // If listenAddress is explicitly set, use it // If listenAddress6 is not set, use defaultListenAddress const listenAddress = args.listenAddress ?? (args.listenAddress6 ? undefined : defaultListenAddress); - const port = listenAddress ? args.port ?? defaultP2pPort : undefined; - const discoveryPort = listenAddress ? args.discoveryPort ?? args.port ?? defaultP2pPort : undefined; + const port = listenAddress ? (args.port ?? defaultP2pPort) : undefined; + const discoveryPort = listenAddress ? (args.discoveryPort ?? args.port ?? defaultP2pPort) : undefined; // Only use listenAddress6 if it is explicitly set const listenAddress6 = args.listenAddress6; - const port6 = listenAddress6 ? args.port6 ?? defaultP2pPort6 : undefined; - const discoveryPort6 = listenAddress6 ? args.discoveryPort6 ?? args.port6 ?? defaultP2pPort6 : undefined; + const port6 = listenAddress6 ? (args.port6 ?? defaultP2pPort6) : undefined; + const discoveryPort6 = listenAddress6 ? (args.discoveryPort6 ?? args.port6 ?? defaultP2pPort6) : undefined; return {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6}; } diff --git a/packages/cli/src/util/gitData/index.ts b/packages/cli/src/util/gitData/index.ts index 96c5e4bbaef2..93b748d42071 100644 --- a/packages/cli/src/util/gitData/index.ts +++ b/packages/cli/src/util/gitData/index.ts @@ -23,11 +23,11 @@ export function readAndGetGitData(): GitData { branch: currentGitData.branch && currentGitData.branch.length > 0 ? currentGitData.branch - : persistedGitData.branch ?? "", + : (persistedGitData.branch ?? ""), commit: currentGitData.commit && currentGitData.commit.length > 0 ? currentGitData.commit - : persistedGitData.commit ?? "", + : (persistedGitData.commit ?? ""), }; } catch (e) { return { diff --git a/packages/config/src/forkConfig/index.ts b/packages/config/src/forkConfig/index.ts index dd2a1773c390..513cd7559ee3 100644 --- a/packages/config/src/forkConfig/index.ts +++ b/packages/config/src/forkConfig/index.ts @@ -84,9 +84,9 @@ export function createForkConfig(config: ChainConfig): ForkConfig { // Fork convenience methods getForkInfo(slot: Slot): ForkInfo { const epoch = Math.floor(Math.max(slot, 0) / SLOTS_PER_EPOCH); - return this.getForkInfoFromEpoch(epoch); + return this.getForkInfoAtEpoch(epoch); }, - getForkInfoFromEpoch(epoch: Epoch): ForkInfo { + getForkInfoAtEpoch(epoch: Epoch): ForkInfo { // NOTE: forks must be sorted by descending epoch, latest fork first for (const fork of forksDescendingEpochOrder) { if (epoch >= fork.epoch) return fork; @@ -99,8 +99,8 @@ export function createForkConfig(config: ChainConfig): ForkConfig { getForkSeq(slot: Slot): ForkSeq { return this.getForkInfo(slot).seq; }, - getForkSeqFromEpoch(epoch: Epoch): ForkSeq { - return this.getForkInfoFromEpoch(epoch).seq; + getForkSeqAtEpoch(epoch: Epoch): ForkSeq { + return this.getForkInfoAtEpoch(epoch).seq; }, getForkVersion(slot: Slot): Version { return this.getForkInfo(slot).version; diff --git a/packages/config/src/forkConfig/types.ts b/packages/config/src/forkConfig/types.ts index dd0381f4e9bf..ebb2899a2a21 100644 --- a/packages/config/src/forkConfig/types.ts +++ b/packages/config/src/forkConfig/types.ts @@ -22,13 +22,13 @@ export type ForkConfig = { /** Get the hard-fork info for the active fork at `slot` */ getForkInfo(slot: Slot): ForkInfo; /** Get the hard-fork info for the active fork at `epoch` */ - getForkInfoFromEpoch(epoch: Epoch): ForkInfo; + getForkInfoAtEpoch(epoch: Epoch): ForkInfo; /** Get the hard-fork name at a given slot */ getForkName(slot: Slot): ForkName; /** Get the hard-fork sequence number at a given slot */ getForkSeq(slot: Slot): ForkSeq; /** Get the hard-fork sequence number at a given epoch */ - getForkSeqFromEpoch(epoch: Epoch): ForkSeq; + getForkSeqAtEpoch(epoch: Epoch): ForkSeq; /** Get the hard-fork version at a given slot */ getForkVersion(slot: Slot): Version; /** Get SSZ types by hard-fork */ diff --git a/packages/params/src/forkName.ts b/packages/params/src/forkName.ts index 4a41f0637340..42e8917942d2 100644 --- a/packages/params/src/forkName.ts +++ b/packages/params/src/forkName.ts @@ -82,14 +82,14 @@ export function isForkBlobs(fork: ForkName): fork is ForkBlobs { } export type ForkPreElectra = ForkPreBlobs | ForkName.deneb; -export type ForkElectra = Exclude; -export const forkElectra = exclude(forkAll, [ +export type ForkPostElectra = Exclude; +export const forkPostElectra = exclude(forkAll, [ ForkName.phase0, ForkName.altair, ForkName.bellatrix, ForkName.capella, ForkName.deneb, ]); -export function isForkElectra(fork: ForkName): fork is ForkElectra { +export function isForkPostElectra(fork: ForkName): fork is ForkPostElectra { return isForkBlobs(fork) && fork !== ForkName.deneb; } diff --git a/packages/prover/src/utils/gitData/index.ts b/packages/prover/src/utils/gitData/index.ts index 96c5e4bbaef2..93b748d42071 100644 --- a/packages/prover/src/utils/gitData/index.ts +++ b/packages/prover/src/utils/gitData/index.ts @@ -23,11 +23,11 @@ export function readAndGetGitData(): GitData { branch: currentGitData.branch && currentGitData.branch.length > 0 ? currentGitData.branch - : persistedGitData.branch ?? "", + : (persistedGitData.branch ?? ""), commit: currentGitData.commit && currentGitData.commit.length > 0 ? currentGitData.commit - : persistedGitData.commit ?? "", + : (persistedGitData.commit ?? ""), }; } catch (e) { return { diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index c9551195e87e..480f804785e8 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -69,12 +69,8 @@ "@lodestar/types": "^1.21.0", "@lodestar/utils": "^1.21.0", "bigint-buffer": "^1.1.5", - "buffer-xor": "^2.0.2", "immutable": "^4.3.2" }, - "devDependencies": { - "@types/buffer-xor": "^2.0.0" - }, "keywords": [ "ethereum", "eth-consensus", diff --git a/packages/state-transition/src/block/processConsolidationRequest.ts b/packages/state-transition/src/block/processConsolidationRequest.ts index 2f12065fde41..71b85e927336 100644 --- a/packages/state-transition/src/block/processConsolidationRequest.ts +++ b/packages/state-transition/src/block/processConsolidationRequest.ts @@ -16,7 +16,7 @@ export function processConsolidationRequest( } // If there is too little available consolidation churn limit, consolidation requests are ignored - if (getConsolidationChurnLimit(state) <= MIN_ACTIVATION_BALANCE) { + if (getConsolidationChurnLimit(state.epochCtx) <= MIN_ACTIVATION_BALANCE) { return; } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index cf463a93a92b..af6e976e9089 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -400,7 +400,7 @@ export class EpochCache { const proposers = currentShuffling.activeIndices.length > 0 ? computeProposers( - config.getForkSeqFromEpoch(currentEpoch), + config.getForkSeqAtEpoch(currentEpoch), currentProposerSeed, currentShuffling, effectiveBalanceIncrements @@ -584,7 +584,7 @@ export class EpochCache { const currentProposerSeed = getSeed(state, this.currentShuffling.epoch, DOMAIN_BEACON_PROPOSER); this.proposers = computeProposers( - this.config.getForkSeqFromEpoch(currEpoch), + this.config.getForkSeqAtEpoch(currEpoch), currentProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements @@ -785,7 +785,7 @@ export class EpochCache { getBeaconProposersNextEpoch(): ValidatorIndex[] { if (!this.proposersNextEpoch.computed) { const indexes = computeProposers( - this.config.getForkSeqFromEpoch(this.epoch + 1), + this.config.getForkSeqAtEpoch(this.epoch + 1), this.proposersNextEpoch.seed, this.nextShuffling, this.effectiveBalanceIncrements @@ -914,7 +914,7 @@ export class EpochCache { } getValidatorIndex(pubkey: Uint8Array | PubkeyHex): ValidatorIndex | undefined { - if (this.isAfterElectra()) { + if (this.isPostElectra()) { return this.pubkey2index.get(pubkey) ?? this.unfinalizedPubkey2index.get(toMemoryEfficientHexStr(pubkey)); } else { return this.pubkey2index.get(pubkey); @@ -927,7 +927,7 @@ export class EpochCache { * */ addPubkey(index: ValidatorIndex, pubkey: Uint8Array): void { - if (this.isAfterElectra()) { + if (this.isPostElectra()) { this.addUnFinalizedPubkey(index, pubkey); } else { // deposit mechanism pre ELECTRA follows a safe distance with assumption @@ -1049,7 +1049,7 @@ export class EpochCache { } effectiveBalanceIncrementsSet(index: number, effectiveBalance: number): void { - if (this.isAfterElectra()) { + if (this.isPostElectra()) { // TODO: electra // getting length and setting getEffectiveBalanceIncrementsByteLen is not fork safe // so each time we add an index, we should new the Uint8Array to keep it forksafe @@ -1074,8 +1074,8 @@ export class EpochCache { this.effectiveBalanceIncrements[index] = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); } - isAfterElectra(): boolean { - return this.epoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity); + isPostElectra(): boolean { + return this.epoch >= this.config.ELECTRA_FORK_EPOCH; } getValidatorCountAtEpoch(targetEpoch: Epoch): number | undefined { diff --git a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts index e6e43bbaa089..f89e938cc790 100644 --- a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts @@ -14,7 +14,7 @@ import {getCurrentEpoch} from "../util/epoch.js"; * TODO Electra: Update ssz library to support batch push to `pendingBalanceDeposits` */ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): void { - const availableForProcessing = state.depositBalanceToConsume + BigInt(getActivationExitChurnLimit(state)); + const availableForProcessing = state.depositBalanceToConsume + BigInt(getActivationExitChurnLimit(state.epochCtx)); const currentEpoch = getCurrentEpoch(state); let processedAmount = 0n; let nextDepositIndex = 0; diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 974f0d6a608b..ef7b0bd661b4 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -92,8 +92,8 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache // TODO-electra: can we improve this? stateElectraView.commit(); const tmpElectraState = getCachedBeaconState(stateElectraView, stateDeneb); - stateElectraView.exitBalanceToConsume = BigInt(getActivationExitChurnLimit(tmpElectraState)); - stateElectraView.consolidationBalanceToConsume = BigInt(getConsolidationChurnLimit(tmpElectraState)); + stateElectraView.exitBalanceToConsume = BigInt(getActivationExitChurnLimit(tmpElectraState.epochCtx)); + stateElectraView.consolidationBalanceToConsume = BigInt(getConsolidationChurnLimit(tmpElectraState.epochCtx)); preActivation.sort((i0, i1) => { const res = validatorsArr[i0].activationEligibilityEpoch - validatorsArr[i1].activationEligibilityEpoch; diff --git a/packages/state-transition/src/util/epoch.ts b/packages/state-transition/src/util/epoch.ts index 66bd2ad76b34..7fed5e53f1f3 100644 --- a/packages/state-transition/src/util/epoch.ts +++ b/packages/state-transition/src/util/epoch.ts @@ -43,7 +43,7 @@ export function computeActivationExitEpoch(epoch: Epoch): Epoch { export function computeExitEpochAndUpdateChurn(state: CachedBeaconStateElectra, exitBalance: Gwei): number { let earliestExitEpoch = Math.max(state.earliestExitEpoch, computeActivationExitEpoch(state.epochCtx.epoch)); - const perEpochChurn = getActivationExitChurnLimit(state); + const perEpochChurn = getActivationExitChurnLimit(state.epochCtx); // New epoch for exits. let exitBalanceToConsume = @@ -72,7 +72,7 @@ export function computeConsolidationEpochAndUpdateChurn( state.earliestConsolidationEpoch, computeActivationExitEpoch(state.epochCtx.epoch) ); - const perEpochConsolidationChurn = getConsolidationChurnLimit(state); + const perEpochConsolidationChurn = getConsolidationChurnLimit(state.epochCtx); // New epoch for consolidations let consolidationBalanceToConsume = diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 7bdc420d6458..02bcef00bb55 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -129,12 +129,12 @@ export function applyTimestamp(config: ChainForkConfig, state: CachedBeaconState * @returns active validator indices */ export function applyDeposits( - fork: ForkSeq, config: ChainForkConfig, state: CachedBeaconStateAllForks, newDeposits: phase0.Deposit[], fullDepositDataRootList?: DepositDataRootViewDU ): {activatedValidatorCount: number} { + const fork = config.getForkSeq(state.slot); const depositDataRootList: Root[] = []; const fullDepositDataRootArr = fullDepositDataRootList ? fullDepositDataRootList.getAllReadonlyValues() : null; @@ -258,7 +258,7 @@ export function initializeBeaconStateFromEth1( applyEth1BlockHash(state, eth1BlockHash); // Process deposits - applyDeposits(fork, config, state, deposits, fullDepositDataRootList); + applyDeposits(config, state, deposits, fullDepositDataRootList); // Commit before reading all validators in `getActiveValidatorIndices()` state.commit(); diff --git a/packages/state-transition/src/util/validator.ts b/packages/state-transition/src/util/validator.ts index 0c8c74b2cb03..728f14587fde 100644 --- a/packages/state-transition/src/util/validator.ts +++ b/packages/state-transition/src/util/validator.ts @@ -7,7 +7,7 @@ import { MAX_EFFECTIVE_BALANCE_ELECTRA, MIN_ACTIVATION_BALANCE, } from "@lodestar/params"; -import {BeaconStateAllForks, CachedBeaconStateElectra} from "../types.js"; +import {BeaconStateAllForks, CachedBeaconStateElectra, EpochCache} from "../types.js"; import {hasCompoundingWithdrawalCredential} from "./electra.js"; /** @@ -57,22 +57,22 @@ export function getChurnLimit(config: ChainForkConfig, activeValidatorCount: num /** * Get combined churn limit of activation-exit and consolidation */ -export function getBalanceChurnLimit(state: CachedBeaconStateElectra): number { +export function getBalanceChurnLimit(epochCtx: EpochCache): number { const churnLimitByTotalActiveBalance = Math.floor( - (state.epochCtx.totalActiveBalanceIncrements / state.config.CHURN_LIMIT_QUOTIENT) * EFFECTIVE_BALANCE_INCREMENT + (epochCtx.totalActiveBalanceIncrements / epochCtx.config.CHURN_LIMIT_QUOTIENT) * EFFECTIVE_BALANCE_INCREMENT ); // TODO Electra: verify calculation - const churn = Math.max(churnLimitByTotalActiveBalance, state.config.MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA); + const churn = Math.max(churnLimitByTotalActiveBalance, epochCtx.config.MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA); return churn - (churn % EFFECTIVE_BALANCE_INCREMENT); } -export function getActivationExitChurnLimit(state: CachedBeaconStateElectra): number { - return Math.min(state.config.MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT, getBalanceChurnLimit(state)); +export function getActivationExitChurnLimit(epochCtx: EpochCache): number { + return Math.min(epochCtx.config.MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT, getBalanceChurnLimit(epochCtx)); } -export function getConsolidationChurnLimit(state: CachedBeaconStateElectra): number { - return getBalanceChurnLimit(state) - getActivationExitChurnLimit(state); +export function getConsolidationChurnLimit(epochCtx: EpochCache): number { + return getBalanceChurnLimit(epochCtx) - getActivationExitChurnLimit(epochCtx); } export function getValidatorMaxEffectiveBalance(withdrawalCredentials: Uint8Array): number { diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts index 677a15724ece..b108de502a1f 100644 --- a/packages/state-transition/test/unit/util/deposit.test.ts +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -10,7 +10,7 @@ describe("getEth1DepositCount", () => { const stateView = ssz.altair.BeaconState.defaultViewDU(); const preElectraState = createCachedBeaconStateTest(stateView); - if (preElectraState.epochCtx.isAfterElectra()) { + if (preElectraState.epochCtx.isPostElectra()) { throw Error("Not a pre-Electra state"); } @@ -39,7 +39,7 @@ describe("getEth1DepositCount", () => { {skipSyncCommitteeCache: true, skipSyncPubkeys: true} ) as CachedBeaconStateElectra; - if (!postElectraState.epochCtx.isAfterElectra()) { + if (!postElectraState.epochCtx.isPostElectra()) { throw Error("Not a post-Electra state"); } @@ -73,7 +73,7 @@ describe("getEth1DepositCount", () => { {skipSyncCommitteeCache: true, skipSyncPubkeys: true} ) as CachedBeaconStateElectra; - if (!postElectraState.epochCtx.isAfterElectra()) { + if (!postElectraState.epochCtx.isPostElectra()) { throw Error("Not a post-Electra state"); } diff --git a/packages/types/src/utils/typeguards.ts b/packages/types/src/utils/typeguards.ts index 2af11159878c..72910645f6e1 100644 --- a/packages/types/src/utils/typeguards.ts +++ b/packages/types/src/utils/typeguards.ts @@ -1,4 +1,4 @@ -import {ForkBlobs, ForkExecution, ForkElectra} from "@lodestar/params"; +import {ForkBlobs, ForkExecution, ForkPostElectra} from "@lodestar/params"; import { BlockContents, SignedBeaconBlock, @@ -68,6 +68,6 @@ export function isSignedBlockContents( return (data as SignedBlockContents).kzgProofs !== undefined; } -export function isElectraAttestation(attestation: Attestation): attestation is Attestation { - return (attestation as Attestation).committeeBits !== undefined; +export function isElectraAttestation(attestation: Attestation): attestation is Attestation { + return (attestation as Attestation).committeeBits !== undefined; } diff --git a/packages/utils/src/promise.ts b/packages/utils/src/promise.ts index 2be7467bfff4..81fa72e7d587 100644 --- a/packages/utils/src/promise.ts +++ b/packages/utils/src/promise.ts @@ -109,7 +109,7 @@ export async function resolveOrRacePromises wrapPromise(p)) as ReturnPromiseWithTuple; // We intentionally want an array of promises here - // eslint-disable-next-line @typescript-eslint/no-floating-promises + promises = (promiseResults as PromiseResult[]).map((p) => p.promise) as unknown as T; try { diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 4226bd572bd0..d6f48aeb3d1f 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -159,13 +159,13 @@ export class AttestationService { this.metrics?.attesterStepCallProduceAggregate.observe(this.clock.secFromSlot(slot + 2 / 3)); const dutiesByCommitteeIndex = groupAttDutiesByCommitteeIndex(dutiesAll); - const isAfterElectra = computeEpochAtSlot(slot) >= this.config.ELECTRA_FORK_EPOCH; + const isPostElectra = computeEpochAtSlot(slot) >= this.config.ELECTRA_FORK_EPOCH; // Then download, sign and publish a `SignedAggregateAndProof` for each // validator that is elected to aggregate for this `slot` and `committeeIndex`. await Promise.all( Array.from(dutiesByCommitteeIndex.entries()).map(([index, dutiesSameCommittee]) => { - const attestationData: phase0.AttestationData = {...attestationNoCommittee, index: isAfterElectra ? 0 : index}; + const attestationData: phase0.AttestationData = {...attestationNoCommittee, index: isPostElectra ? 0 : index}; return this.produceAndPublishAggregates(attestationData, index, dutiesSameCommittee); }) ); @@ -196,11 +196,11 @@ export class AttestationService { const signedAttestations: Attestation[] = []; const headRootHex = toHexString(attestationNoCommittee.beaconBlockRoot); const currentEpoch = computeEpochAtSlot(slot); - const isAfterElectra = currentEpoch >= this.config.ELECTRA_FORK_EPOCH; + const isPostElectra = currentEpoch >= this.config.ELECTRA_FORK_EPOCH; await Promise.all( duties.map(async ({duty}) => { - const index = isAfterElectra ? 0 : duty.committeeIndex; + const index = isPostElectra ? 0 : duty.committeeIndex; const attestationData: phase0.AttestationData = {...attestationNoCommittee, index}; const logCtxValidator = {slot, index, head: headRootHex, validatorIndex: duty.validatorIndex}; diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index cea281f0d144..252d0886cc6e 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -801,13 +801,13 @@ export class ValidatorStore { throw Error(`Inconsistent duties during signing: duty.slot ${duty.slot} != att.slot ${data.slot}`); } - const isAfterElectra = computeEpochAtSlot(duty.slot) >= this.config.ELECTRA_FORK_EPOCH; - if (!isAfterElectra && duty.committeeIndex != data.index) { + const isPostElectra = computeEpochAtSlot(duty.slot) >= this.config.ELECTRA_FORK_EPOCH; + if (!isPostElectra && duty.committeeIndex != data.index) { throw Error( `Inconsistent duties during signing: duty.committeeIndex ${duty.committeeIndex} != att.committeeIndex ${data.index}` ); } - if (isAfterElectra && data.index !== 0) { + if (isPostElectra && data.index !== 0) { throw Error(`Non-zero committee index post-electra during signing: att.committeeIndex ${data.index}`); } if (this.config.getForkSeq(duty.slot) >= ForkSeq.electra && data.index !== 0) { From da0f40c43c8068892c2a5eae4d562ce330765428 Mon Sep 17 00:00:00 2001 From: harkamal Date: Wed, 7 Aug 2024 16:25:24 +0530 Subject: [PATCH 082/145] fix: fix the devnet2 bug of invalid stateroot calc post consolidation --- packages/state-transition/src/util/electra.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/state-transition/src/util/electra.ts b/packages/state-transition/src/util/electra.ts index 666f45435eb1..63f74bc96cc9 100644 --- a/packages/state-transition/src/util/electra.ts +++ b/packages/state-transition/src/util/electra.ts @@ -68,7 +68,12 @@ export function switchToCompoundingValidator(state: CachedBeaconStateElectra, in const validator = state.validators.get(index); if (hasEth1WithdrawalCredential(validator.withdrawalCredentials)) { - validator.withdrawalCredentials[0] = COMPOUNDING_WITHDRAWAL_PREFIX; + // directly modifying the byte leads to ssz missing the modification resulting into + // wrong root compute, although slicing can be avoided but anyway this is not going + // to be a hot path so its better to clean slice and avoid side effects + const newWithdrawalCredentials = validator.withdrawalCredentials.slice(); + newWithdrawalCredentials[0] = COMPOUNDING_WITHDRAWAL_PREFIX; + validator.withdrawalCredentials = newWithdrawalCredentials; queueExcessActiveBalance(state, index); } } From dc4fa9ec04c14e87c96e70871ae3d422f61203ed Mon Sep 17 00:00:00 2001 From: gajinder Date: Fri, 9 Aug 2024 14:19:39 +0530 Subject: [PATCH 083/145] chore: further rebase fixes to unstable lint fixes lint Fix browser test Fix unit test --------- Co-authored-by: NC <17676176+ensi321@users.noreply.github.com> --- .../src/chain/blocks/importBlock.ts | 10 ++---- .../src/options/beaconNodeOptions/network.ts | 8 ++--- packages/cli/src/util/gitData/index.ts | 4 +-- .../test/unit/webEsmBundle.browser.test.ts | 4 +-- packages/prover/src/utils/gitData/index.ts | 4 +-- .../state-transition/src/cache/pubkeyCache.ts | 1 - .../test/unit/cachedBeaconState.test.ts | 2 +- packages/utils/src/promise.ts | 2 +- yarn.lock | 36 ++----------------- 9 files changed, 16 insertions(+), 55 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index fa862a2fb31e..de5ecf607d95 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -409,18 +409,12 @@ export async function importBlock( } if (this.emitter.listenerCount(routes.events.EventType.attestation)) { for (const attestation of block.message.body.attestations) { - this.emitter.emit(routes.events.EventType.attestation, { - version: this.config.getForkName(blockSlot), - data: attestation, - }); + this.emitter.emit(routes.events.EventType.attestation, attestation); } } if (this.emitter.listenerCount(routes.events.EventType.attesterSlashing)) { for (const attesterSlashing of block.message.body.attesterSlashings) { - this.emitter.emit(routes.events.EventType.attesterSlashing, { - version: this.config.getForkName(blockSlot), - data: attesterSlashing, - }); + this.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing); } } if (this.emitter.listenerCount(routes.events.EventType.proposerSlashing)) { diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 6c6f1ed70cc6..25ba036a5dbf 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -63,13 +63,13 @@ export function parseListenArgs(args: NetworkArgs) { // If listenAddress is explicitly set, use it // If listenAddress6 is not set, use defaultListenAddress const listenAddress = args.listenAddress ?? (args.listenAddress6 ? undefined : defaultListenAddress); - const port = listenAddress ? (args.port ?? defaultP2pPort) : undefined; - const discoveryPort = listenAddress ? (args.discoveryPort ?? args.port ?? defaultP2pPort) : undefined; + const port = listenAddress ? args.port ?? defaultP2pPort : undefined; + const discoveryPort = listenAddress ? args.discoveryPort ?? args.port ?? defaultP2pPort : undefined; // Only use listenAddress6 if it is explicitly set const listenAddress6 = args.listenAddress6; - const port6 = listenAddress6 ? (args.port6 ?? defaultP2pPort6) : undefined; - const discoveryPort6 = listenAddress6 ? (args.discoveryPort6 ?? args.port6 ?? defaultP2pPort6) : undefined; + const port6 = listenAddress6 ? args.port6 ?? defaultP2pPort6 : undefined; + const discoveryPort6 = listenAddress6 ? args.discoveryPort6 ?? args.port6 ?? defaultP2pPort6 : undefined; return {listenAddress, port, discoveryPort, listenAddress6, port6, discoveryPort6}; } diff --git a/packages/cli/src/util/gitData/index.ts b/packages/cli/src/util/gitData/index.ts index 93b748d42071..96c5e4bbaef2 100644 --- a/packages/cli/src/util/gitData/index.ts +++ b/packages/cli/src/util/gitData/index.ts @@ -23,11 +23,11 @@ export function readAndGetGitData(): GitData { branch: currentGitData.branch && currentGitData.branch.length > 0 ? currentGitData.branch - : (persistedGitData.branch ?? ""), + : persistedGitData.branch ?? "", commit: currentGitData.commit && currentGitData.commit.length > 0 ? currentGitData.commit - : (persistedGitData.commit ?? ""), + : persistedGitData.commit ?? "", }; } catch (e) { return { diff --git a/packages/light-client/test/unit/webEsmBundle.browser.test.ts b/packages/light-client/test/unit/webEsmBundle.browser.test.ts index 49b0f877d46c..08ca86e52d75 100644 --- a/packages/light-client/test/unit/webEsmBundle.browser.test.ts +++ b/packages/light-client/test/unit/webEsmBundle.browser.test.ts @@ -1,7 +1,7 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ import {expect, describe, it, vi, beforeAll} from "vitest"; import {sleep} from "@lodestar/utils"; -import {Lightclient, LightclientEvent, utils, transport} from "../../src/index.js"; +import {Lightclient, LightclientEvent, utils, transport} from "../../dist/lightclient.min.mjs"; describe("web bundle for lightclient", () => { vi.setConfig({testTimeout: 20_000}); diff --git a/packages/prover/src/utils/gitData/index.ts b/packages/prover/src/utils/gitData/index.ts index 93b748d42071..96c5e4bbaef2 100644 --- a/packages/prover/src/utils/gitData/index.ts +++ b/packages/prover/src/utils/gitData/index.ts @@ -23,11 +23,11 @@ export function readAndGetGitData(): GitData { branch: currentGitData.branch && currentGitData.branch.length > 0 ? currentGitData.branch - : (persistedGitData.branch ?? ""), + : persistedGitData.branch ?? "", commit: currentGitData.commit && currentGitData.commit.length > 0 ? currentGitData.commit - : (persistedGitData.commit ?? ""), + : persistedGitData.commit ?? "", }; } catch (e) { return { diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index a7eacb7961fd..44fe6df45592 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,7 +1,6 @@ import {PublicKey} from "@chainsafe/blst"; import * as immutable from "immutable"; import {ValidatorIndex, phase0} from "@lodestar/types"; -import {BeaconStateAllForks} from "./types.js"; export type Index2PubkeyCache = PublicKey[]; /** diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 9b05b8947f73..77c5da7a5f4a 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -131,7 +131,7 @@ describe("CachedBeaconState", () => { const capellaStateType = ssz.capella.BeaconState; - for (let validatorCountDelta = -numValidator; validatorCountDelta <= numValidator; validatorCountDelta++) { + for (let validatorCountDelta = -numValidator + 1; validatorCountDelta <= numValidator; validatorCountDelta++) { const testName = `loadCachedBeaconState - ${validatorCountDelta > 0 ? "more" : "less"} ${Math.abs( validatorCountDelta )} validators`; diff --git a/packages/utils/src/promise.ts b/packages/utils/src/promise.ts index 81fa72e7d587..2be7467bfff4 100644 --- a/packages/utils/src/promise.ts +++ b/packages/utils/src/promise.ts @@ -109,7 +109,7 @@ export async function resolveOrRacePromises wrapPromise(p)) as ReturnPromiseWithTuple; // We intentionally want an array of promises here - + // eslint-disable-next-line @typescript-eslint/no-floating-promises promises = (promiseResults as PromiseResult[]).map((p) => p.promise) as unknown as T; try { diff --git a/yarn.lock b/yarn.lock index ecfcb2c5ee6f..7e2e3d8e6447 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3050,13 +3050,6 @@ resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708" integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw== -"@types/buffer-xor@^2.0.0": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@types/buffer-xor/-/buffer-xor-2.0.2.tgz#d8c463583b8fbb322ea824562dc78a0c3cea2ca6" - integrity sha512-OqdCua7QCTupPnJgmyGJUpxWgbuOi0IMIVslXTSePS2o+qDrDB6f2Pg44zRyqhUA5GbFAf39U8z0+mH4WG0fLQ== - dependencies: - "@types/node" "*" - "@types/cacheable-request@^6.0.1": version "6.0.3" resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.3.tgz#a430b3260466ca7b5ca5bfd735693b36e7a9d183" @@ -4633,13 +4626,6 @@ buffer-xor@^1.0.3: resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer-xor@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-2.0.2.tgz#34f7c64f04c777a1f8aac5e661273bb9dd320289" - integrity sha512-eHslX0bin3GB+Lx2p7lEYRShRewuNZL3fUl4qlVJGGiwoPGftmt8JQgk2Y9Ji5/01TnVDo33E5b5O3vUB1HdqQ== - dependencies: - safe-buffer "^5.1.1" - buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz" @@ -12094,16 +12080,7 @@ string-argv@~0.3.1: resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -13678,7 +13655,7 @@ workerpool@6.2.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343" integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -13696,15 +13673,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From ce6a0c719f3e6d80f932b0bd8321985ac69251a9 Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 13 Aug 2024 19:35:51 +0700 Subject: [PATCH 084/145] fix: cached balances in epoch transition (#7018) * fix: update cached balances in epoch transition * fix: more comments in processEffectiveBalanceUpdates() --- .../src/block/processAttestationsAltair.ts | 3 ++- packages/state-transition/src/epoch/index.ts | 4 ++-- .../epoch/processEffectiveBalanceUpdates.ts | 10 ++++------ .../src/epoch/processPendingBalanceDeposits.ts | 18 +++++++++++++----- .../src/epoch/processPendingConsolidations.ts | 12 +++++++++--- .../src/epoch/processSyncCommitteeUpdates.ts | 5 ++--- 6 files changed, 32 insertions(+), 20 deletions(-) diff --git a/packages/state-transition/src/block/processAttestationsAltair.ts b/packages/state-transition/src/block/processAttestationsAltair.ts index f3d4a3e16c21..046a23d7dc27 100644 --- a/packages/state-transition/src/block/processAttestationsAltair.ts +++ b/packages/state-transition/src/block/processAttestationsAltair.ts @@ -75,6 +75,7 @@ export function processAttestationsAltair( // For each participant, update their participation // In epoch processing, this participation info is used to calculate balance updates let totalBalanceIncrementsWithWeight = 0; + const validators = state.validators; for (const index of attestingIndices) { const flags = epochParticipation.get(index); @@ -104,7 +105,7 @@ export function processAttestationsAltair( // TODO: describe issue. Compute progressive target balances // When processing each attestation, increase the cummulative target balance. Only applies post-altair if ((flagsNewSet & TIMELY_TARGET) === TIMELY_TARGET) { - const validator = state.validators.getReadonly(index); + const validator = validators.getReadonly(index); if (!validator.slashed) { if (inCurrentEpoch) { epochCtx.currentTargetUnslashedBalanceIncrements += effectiveBalanceIncrements[index]; diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index 21455521897b..85e7c348dad3 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -133,7 +133,7 @@ export function processEpoch( const timer = metrics?.epochTransitionStepTime.startTimer({ step: EpochTransitionStep.processPendingBalanceDeposits, }); - processPendingBalanceDeposits(stateElectra); + processPendingBalanceDeposits(stateElectra, cache); timer?.(); } @@ -141,7 +141,7 @@ export function processEpoch( const timer = metrics?.epochTransitionStepTime.startTimer({ step: EpochTransitionStep.processPendingConsolidations, }); - processPendingConsolidations(stateElectra); + processPendingConsolidations(stateElectra, cache); timer?.(); } } diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index f4fae6e41d4f..fdbc99b9265e 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -39,12 +39,10 @@ export function processEffectiveBalanceUpdates( // update effective balances with hysteresis - // epochTransitionCache.balances is set in processRewardsAndPenalties(), so it's recycled here for performance. - // It defaults to `state.balances.getAll()` to make Typescript happy and for spec tests - const balances = state.balances.getAll(); - - // TODO: (@matthewkeil) This was causing additional failures but should not. Check the EpochTransitionCache for why - // const balances = cache.balances ?? state.balances.getAll(); + // epochTransitionCache.balances is initialized in processRewardsAndPenalties() + // and updated in processPendingBalanceDeposits() and processPendingConsolidations() + // so it's recycled here for performance. + const balances = cache.balances ?? state.balances.getAll(); for (let i = 0, len = balances.length; i < len; i++) { const balance = balances[i]; diff --git a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts index f89e938cc790..81da34349e53 100644 --- a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts @@ -1,5 +1,5 @@ import {FAR_FUTURE_EPOCH} from "@lodestar/params"; -import {CachedBeaconStateElectra} from "../types.js"; +import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js"; import {increaseBalance} from "../util/balance.js"; import {getActivationExitChurnLimit} from "../util/validator.js"; import {getCurrentEpoch} from "../util/epoch.js"; @@ -13,16 +13,18 @@ import {getCurrentEpoch} from "../util/epoch.js"; * * TODO Electra: Update ssz library to support batch push to `pendingBalanceDeposits` */ -export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): void { +export function processPendingBalanceDeposits(state: CachedBeaconStateElectra, cache: EpochTransitionCache): void { const availableForProcessing = state.depositBalanceToConsume + BigInt(getActivationExitChurnLimit(state.epochCtx)); const currentEpoch = getCurrentEpoch(state); let processedAmount = 0n; let nextDepositIndex = 0; const depositsToPostpone = []; + const validators = state.validators; + const cachedBalances = cache.balances; for (const deposit of state.pendingBalanceDeposits.getAllReadonly()) { const {amount, index: depositIndex} = deposit; - const validator = state.validators.get(depositIndex); + const validator = validators.getReadonly(depositIndex); // Validator is exiting, postpone the deposit until after withdrawable epoch if (validator.exitEpoch < FAR_FUTURE_EPOCH) { @@ -30,7 +32,10 @@ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): depositsToPostpone.push(deposit); } else { // Deposited balance will never become active. Increase balance but do not consume churn - increaseBalance(state, deposit.index, Number(amount)); + increaseBalance(state, depositIndex, Number(amount)); + if (cachedBalances) { + cachedBalances[depositIndex] += Number(amount); + } } } else { // Validator is not exiting, attempt to process deposit @@ -39,7 +44,10 @@ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra): break; } else { // Deposit fits in the churn, process it. Increase balance and consume churn. - increaseBalance(state, deposit.index, Number(amount)); + increaseBalance(state, depositIndex, Number(amount)); + if (cachedBalances) { + cachedBalances[depositIndex] += Number(amount); + } processedAmount = processedAmount + amount; } } diff --git a/packages/state-transition/src/epoch/processPendingConsolidations.ts b/packages/state-transition/src/epoch/processPendingConsolidations.ts index e025a454f64e..d6f760a6b27a 100644 --- a/packages/state-transition/src/epoch/processPendingConsolidations.ts +++ b/packages/state-transition/src/epoch/processPendingConsolidations.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStateElectra} from "../types.js"; +import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js"; import {decreaseBalance, increaseBalance} from "../util/balance.js"; import {getActiveBalance} from "../util/validator.js"; import {switchToCompoundingValidator} from "../util/electra.js"; @@ -16,12 +16,14 @@ import {switchToCompoundingValidator} from "../util/electra.js"; * Dequeue all processed consolidations from `state.pendingConsolidation` * */ -export function processPendingConsolidations(state: CachedBeaconStateElectra): void { +export function processPendingConsolidations(state: CachedBeaconStateElectra, cache: EpochTransitionCache): void { let nextPendingConsolidation = 0; + const validators = state.validators; + const cachedBalances = cache.balances; for (const pendingConsolidation of state.pendingConsolidations.getAllReadonly()) { const {sourceIndex, targetIndex} = pendingConsolidation; - const sourceValidator = state.validators.getReadonly(sourceIndex); + const sourceValidator = validators.getReadonly(sourceIndex); if (sourceValidator.slashed) { nextPendingConsolidation++; @@ -37,6 +39,10 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra): v const activeBalance = getActiveBalance(state, sourceIndex); decreaseBalance(state, sourceIndex, activeBalance); increaseBalance(state, targetIndex, activeBalance); + if (cachedBalances) { + cachedBalances[sourceIndex] -= activeBalance; + cachedBalances[targetIndex] += activeBalance; + } nextPendingConsolidation++; } diff --git a/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts b/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts index aa22bb4f7bcf..df4e0364be17 100644 --- a/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts +++ b/packages/state-transition/src/epoch/processSyncCommitteeUpdates.ts @@ -23,11 +23,10 @@ export function processSyncCommitteeUpdates(fork: ForkSeq, state: CachedBeaconSt activeValidatorIndices, effectiveBalanceIncrements ); + const validators = state.validators; // Using the index2pubkey cache is slower because it needs the serialized pubkey. - const nextSyncCommitteePubkeys = nextSyncCommitteeIndices.map( - (index) => state.validators.getReadonly(index).pubkey - ); + const nextSyncCommitteePubkeys = nextSyncCommitteeIndices.map((index) => validators.getReadonly(index).pubkey); // Rotate syncCommittee in state state.currentSyncCommittee = state.nextSyncCommittee; From 4b2f34b1945f4c3e0f4df83dc5fb5138194d63db Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 13 Aug 2024 21:16:28 +0100 Subject: [PATCH 085/145] chore: review electra branch - part 1 (#7015) --- packages/api/src/beacon/routes/beacon/block.ts | 2 +- packages/api/src/beacon/routes/beacon/pool.ts | 6 ++---- packages/api/src/beacon/routes/validator.ts | 8 ++++++-- packages/api/test/unit/beacon/oapiSpec.test.ts | 5 ----- .../api/test/unit/beacon/testData/beacon.ts | 2 +- .../src/api/impl/beacon/blocks/index.ts | 14 ++++++++++---- .../src/api/impl/config/constants.ts | 2 +- .../beacon-node/src/api/impl/validator/index.ts | 5 +++-- .../src/chain/errors/attestationError.ts | 2 +- packages/beacon-node/src/eth1/interface.ts | 2 +- .../beacon-node/src/network/gossip/topic.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../test/perf/chain/opPools/opPool.test.ts | 2 +- .../chain/produceBlock/produceBlockBody.test.ts | 2 +- .../scripts/el-interop/besu/common-setup.sh | 2 +- .../test/scripts/el-interop/besu/post-merge.sh | 2 +- .../scripts/el-interop/besudocker/post-merge.sh | 2 +- .../el-interop/ethereumjsdocker/electra.tmpl | 2 +- .../test/sim/electra-interop.test.ts | 2 +- .../test/spec/presets/genesis.test.ts | 4 +--- .../test/spec/presets/operations.test.ts | 17 +++++++---------- .../test/unit/chain/shufflingCache.test.ts | 8 ++++---- .../test/unit/eth1/utils/deposits.test.ts | 6 +++--- .../beacon-node/test/unit/util/sszBytes.test.ts | 2 +- .../test/utils/validationData/attestation.ts | 7 +++---- packages/flare/src/cmds/selfSlashAttester.ts | 4 ++-- .../test/unit/webEsmBundle.browser.test.ts | 2 +- packages/state-transition/package.json | 1 - 28 files changed, 57 insertions(+), 60 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index 2c141e3bdefd..be6789753e0f 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -8,11 +8,11 @@ import { deneb, isSignedBlockContents, SignedBeaconBlock, + BeaconBlockBody, SignedBeaconBlockOrContents, SignedBlindedBeaconBlock, SignedBlockContents, sszTypesFor, - BeaconBlockBody, } from "@lodestar/types"; import {ForkName, ForkPreElectra, ForkPreExecution, isForkBlobs, isForkExecution} from "@lodestar/params"; import {Endpoint, RequestCodec, RouteDefinitions, Schema} from "../../../utils/index.js"; diff --git a/packages/api/src/beacon/routes/beacon/pool.ts b/packages/api/src/beacon/routes/beacon/pool.ts index 42880b276732..4fe3efd4daf2 100644 --- a/packages/api/src/beacon/routes/beacon/pool.ts +++ b/packages/api/src/beacon/routes/beacon/pool.ts @@ -317,9 +317,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - body: AttestationListTypePhase0.toJson(signedAttestations), - }), + writeReqJson: ({signedAttestations}) => ({body: AttestationListTypePhase0.toJson(signedAttestations)}), parseReqJson: ({body}) => ({signedAttestations: AttestationListTypePhase0.fromJson(body)}), writeReqSsz: ({signedAttestations}) => ({body: AttestationListTypePhase0.serialize(signedAttestations)}), parseReqSsz: ({body}) => ({signedAttestations: AttestationListTypePhase0.deserialize(body)}), @@ -414,7 +412,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ body: SignedAggregateAndProofListPhase0Type.toJson(signedAggregateAndProofs), }), - parseReqJson: ({body}) => ({signedAggregateAndProofs: SignedAggregateAndProofListPhase0Type.fromJson(body)}), + parseReqJson: ({body}) => ({ + signedAggregateAndProofs: SignedAggregateAndProofListPhase0Type.fromJson(body), + }), writeReqSsz: ({signedAggregateAndProofs}) => ({ body: SignedAggregateAndProofListPhase0Type.serialize(signedAggregateAndProofs), }), - parseReqSsz: ({body}) => ({signedAggregateAndProofs: SignedAggregateAndProofListPhase0Type.deserialize(body)}), + parseReqSsz: ({body}) => ({ + signedAggregateAndProofs: SignedAggregateAndProofListPhase0Type.deserialize(body), + }), schema: { body: Schema.ObjectArray, }, diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index 2ccf4720db27..efce60a5338f 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -86,11 +86,6 @@ const ignoredTopics = [ topic block_gossip not implemented */ "block_gossip", - - // Modified in electra to include version - // should be removed from the ignore list after spec update - "attestation", - "attester_slashing", ]; // eventstream types are defined as comments in the description of "examples". diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index ac47a58777eb..b0e958f4bc58 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -138,7 +138,7 @@ export const testData: GenericServerTestCases = { res: undefined, }, submitPoolAttesterSlashingsV2: { - args: {attesterSlashing: ssz.electra.AttesterSlashing.defaultValue()}, + args: {attesterSlashing: ssz.phase0.AttesterSlashing.defaultValue()}, res: undefined, }, submitPoolProposerSlashings: { diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 251218f510ec..89e8a20d4f86 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -1,12 +1,12 @@ import {routes} from "@lodestar/api"; -import {ApplicationMethods} from "@lodestar/api/server"; +import {ApiError, ApplicationMethods} from "@lodestar/api/server"; import { computeEpochAtSlot, computeTimeAtSlot, reconstructFullBlockOrContents, signedBeaconBlockToBlinded, } from "@lodestar/state-transition"; -import {ForkExecution, SLOTS_PER_HISTORICAL_ROOT, isForkExecution} from "@lodestar/params"; +import {ForkExecution, SLOTS_PER_HISTORICAL_ROOT, isForkExecution, isForkPostElectra} from "@lodestar/params"; import {sleep, fromHex, toRootHex} from "@lodestar/utils"; import { deneb, @@ -407,8 +407,14 @@ export function getBeaconBlockApi({ async getBlockAttestations({blockId}) { const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); + const fork = config.getForkName(block.message.slot); + + if (isForkPostElectra(fork)) { + throw new ApiError(400, `Use getBlockAttestationsV2 to retrieve electra+ block attestations fork=${fork}`); + } + return { - data: Array.from(block.message.body.attestations), + data: block.message.body.attestations, meta: {executionOptimistic, finalized}, }; }, @@ -416,7 +422,7 @@ export function getBeaconBlockApi({ async getBlockAttestationsV2({blockId}) { const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId); return { - data: Array.from(block.message.body.attestations), + data: block.message.body.attestations, meta: {executionOptimistic, finalized, version: config.getForkName(block.message.slot)}, }; }, diff --git a/packages/beacon-node/src/api/impl/config/constants.ts b/packages/beacon-node/src/api/impl/config/constants.ts index 78262ce15237..4b239ee4cac6 100644 --- a/packages/beacon-node/src/api/impl/config/constants.ts +++ b/packages/beacon-node/src/api/impl/config/constants.ts @@ -46,7 +46,7 @@ import { /** * Hand-picked list of constants declared in consensus-spec .md files. - * This list is asserted to be up-to-date with the test `test/e2e/api/specConstants.test.ts` + * This list is asserted to be up-to-date with the test `test/e2e/api/impl/config.test.ts` */ export const specConstants = { // phase0/beacon-chain.md diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 06502146ce2f..2d453f0d24c7 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -21,6 +21,7 @@ import { ForkPreBlobs, ForkBlobs, ForkExecution, + isForkPostElectra, } from "@lodestar/params"; import {MAX_BUILDER_BOOST_FACTOR} from "@lodestar/validator"; import { @@ -814,7 +815,7 @@ export function getValidatorApi( const attEpoch = computeEpochAtSlot(slot); const headBlockRootHex = chain.forkChoice.getHead().blockRoot; const headBlockRoot = fromHex(headBlockRootHex); - const fork = config.getForkSeq(slot); + const fork = config.getForkName(slot); const beaconBlockRoot = slot >= headSlot @@ -846,7 +847,7 @@ export function getValidatorApi( return { data: { slot, - index: fork >= ForkSeq.electra ? 0 : committeeIndex, + index: isForkPostElectra(fork) ? 0 : committeeIndex, beaconBlockRoot, source: attEpochState.currentJustifiedCheckpoint, target: {epoch: attEpoch, root: targetRoot}, diff --git a/packages/beacon-node/src/chain/errors/attestationError.ts b/packages/beacon-node/src/chain/errors/attestationError.ts index d53a6f99682d..9f8e86cea1ab 100644 --- a/packages/beacon-node/src/chain/errors/attestationError.ts +++ b/packages/beacon-node/src/chain/errors/attestationError.ts @@ -128,7 +128,7 @@ export enum AttestationErrorCode { /** Too many skipped slots. */ TOO_MANY_SKIPPED_SLOTS = "ATTESTATION_ERROR_TOO_MANY_SKIPPED_SLOTS", /** - * Electra: The aggregated attestation doesn't have only one committee bit set. + * Electra: The aggregated attestation does not have exactly one committee bit set. */ NOT_EXACTLY_ONE_COMMITTEE_BIT_SET = "ATTESTATION_ERROR_NOT_EXACTLY_ONE_COMMITTEE_BIT_SET", /** diff --git a/packages/beacon-node/src/eth1/interface.ts b/packages/beacon-node/src/eth1/interface.ts index 898041ac8947..54fcdd12492f 100644 --- a/packages/beacon-node/src/eth1/interface.ts +++ b/packages/beacon-node/src/eth1/interface.ts @@ -64,7 +64,7 @@ export interface IEth1ForBlockProduction { startPollingMergeBlock(): void; /** - * Should stop polling eth1Data after a Electra block is finalized AND deposit_receipts_start_index is reached + * Should stop polling eth1Data after a Electra block is finalized AND deposit_requests_start_index is reached */ stopPollingEth1Data(): void; } diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index f3968b8e65b6..b7c7425584c5 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -93,7 +93,7 @@ export function getGossipSSZType(topic: GossipTopic) { case GossipType.proposer_slashing: return ssz.phase0.ProposerSlashing; case GossipType.attester_slashing: - return ssz.phase0.AttesterSlashing; + return sszTypesFor(topic.fork).AttesterSlashing; case GossipType.voluntary_exit: return ssz.phase0.SignedVoluntaryExit; case GossipType.sync_committee_contribution_and_proof: diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index ee14eb27b51d..60ff6ce48302 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -32,7 +32,7 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { before(function () { this.timeout(5 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}) as CachedBeaconStateAltair; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}); const {blockHeader, checkpoint} = computeAnchorCheckpoint(originalState.config, originalState); // TODO figure out why getBlockRootAtSlot(originalState, justifiedSlot) is not the same to justifiedCheckpoint.root diff --git a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts index 2632c593e78c..6e420f0e1011 100644 --- a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts @@ -24,7 +24,7 @@ describe("opPool", () => { before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}) as CachedBeaconStateAltair; + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}); }); itBench({ diff --git a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts index f90f148f3e01..7bf8c2f7252f 100644 --- a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts +++ b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts @@ -25,7 +25,7 @@ describe("produceBlockBody", () => { before(async () => { db = new BeaconDb(config, await LevelDbController.create({name: ".tmpdb"}, {logger})); - state = stateOg.clone() as CachedBeaconStateAltair; + state = stateOg.clone(); chain = new BeaconChain( { proposerBoost: true, diff --git a/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh b/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh index 1444f6d3a479..f211f5d0714b 100755 --- a/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh +++ b/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh @@ -16,4 +16,4 @@ echo "12345678" > $DATA_DIR/password.txt pubKey="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" # echo a hex encoded 256 bit secret into a file -echo $JWT_SECRET_HEX> $DATA_DIR/jwtsecret \ No newline at end of file +echo $JWT_SECRET_HEX> $DATA_DIR/jwtsecret diff --git a/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh b/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh index 47bec71cf8bb..d864814ece7a 100755 --- a/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh +++ b/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh @@ -5,4 +5,4 @@ currentDir=$(pwd) . $scriptDir/common-setup.sh -$EL_BINARY_DIR/besu --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret $currentDir/$DATA_DIR/jwtsecret --data-path $DATA_DIR --data-storage-format BONSAI --genesis-file $DATA_DIR/genesis.json \ No newline at end of file +$EL_BINARY_DIR/besu --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret $currentDir/$DATA_DIR/jwtsecret --data-path $DATA_DIR --data-storage-format BONSAI --genesis-file $DATA_DIR/genesis.json diff --git a/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh b/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh index fb091a7d838b..d26307ee3d24 100755 --- a/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh +++ b/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh @@ -5,4 +5,4 @@ currentDir=$(pwd) . $scriptDir/common-setup.sh -docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --name custom-execution -p $ETH_PORT:$ETH_PORT -p $ENGINE_PORT:$ENGINE_PORT -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret /data/jwtsecret --data-path /data/besu --data-storage-format BONSAI --genesis-file /data/genesis.json \ No newline at end of file +docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --name custom-execution -p $ETH_PORT:$ETH_PORT -p $ENGINE_PORT:$ENGINE_PORT -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret /data/jwtsecret --data-path /data/besu --data-storage-format BONSAI --genesis-file /data/genesis.json diff --git a/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl b/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl index d557d505c649..3a06b75cd000 100644 --- a/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl +++ b/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/electra.tmpl @@ -91,4 +91,4 @@ "gasUsed":"0x0", "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", "baseFeePerGas":"0x7" -} \ No newline at end of file +} diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index 02fc99f9cfae..d0c00e75fd59 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -252,7 +252,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { /** * Want to test two things: - * 1) Send two raw deposit transactions, and see if two new validators with corrent balances show up in the state.validators and unfinalized cache + * 1) Send two raw deposit transactions, and see if two new validators with correct balances show up in the state.validators and unfinalized cache * 2) Upon state-transition, see if the two new validators move from unfinalized cache to finalized cache */ async function runNodeWithEL({ diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index d8f85dd9b5b8..773debe3bb19 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -60,9 +60,7 @@ const genesisInitialization: TestRunnerFn - ) + executionPayloadHeaderType.toViewDU(testcase["execution_payload_header"]) ); }, // eth1.yaml diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index c978cac3c8a2..7e2e9c1e9c5d 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -11,7 +11,7 @@ import { import * as blockFns from "@lodestar/state-transition/block"; import {ssz, phase0, altair, bellatrix, capella, electra, sszTypesFor} from "@lodestar/types"; import {InputType} from "@lodestar/spec-test-util"; -import {ACTIVE_PRESET, ForkName, ForkSeq} from "@lodestar/params"; +import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js"; import {expectEqualBeaconState, inputTypeSszTreeViewDU} from "../utils/expectEqualBeaconState.js"; @@ -57,11 +57,6 @@ const operationFns: Record> = blockFns.processDeposit(fork, state, testCase.deposit); }, - deposit_receipt: (state, testCase: {deposit_receipt: electra.DepositRequest}) => { - const fork = state.config.getForkSeq(state.slot); - blockFns.processDepositRequest(fork, state as CachedBeaconStateElectra, testCase.deposit_receipt); - }, - proposer_slashing: (state, testCase: {proposer_slashing: phase0.ProposerSlashing}) => { const fork = state.config.getForkSeq(state.slot); blockFns.processProposerSlashing(fork, state, testCase.proposer_slashing); @@ -89,15 +84,18 @@ const operationFns: Record> = }, withdrawals: (state, testCase: {execution_payload: capella.ExecutionPayload}) => { - blockFns.processWithdrawals(ForkSeq.capella, state as CachedBeaconStateCapella, testCase.execution_payload); + const fork = state.config.getForkSeq(state.slot); + blockFns.processWithdrawals(fork, state as CachedBeaconStateCapella, testCase.execution_payload); }, withdrawal_request: (state, testCase: {withdrawal_request: electra.WithdrawalRequest}) => { - blockFns.processWithdrawalRequest(ForkSeq.electra, state as CachedBeaconStateElectra, testCase.withdrawal_request); + const fork = state.config.getForkSeq(state.slot); + blockFns.processWithdrawalRequest(fork, state as CachedBeaconStateElectra, testCase.withdrawal_request); }, deposit_request: (state, testCase: {deposit_request: electra.DepositRequest}) => { - blockFns.processDepositRequest(ForkSeq.electra, state as CachedBeaconStateElectra, testCase.deposit_request); + const fork = state.config.getForkSeq(state.slot); + blockFns.processDepositRequest(fork, state as CachedBeaconStateElectra, testCase.deposit_request); }, consolidation_request: (state, testCase: {consolidation_request: electra.ConsolidationRequest}) => { @@ -140,7 +138,6 @@ const operations: TestRunnerFn = (fork, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, - deposit_receipt: ssz.electra.DepositRequest, proposer_slashing: ssz.phase0.ProposerSlashing, voluntary_exit: ssz.phase0.SignedVoluntaryExit, // Altair diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 035746438563..6295a993c072 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect, beforeEach} from "vitest"; -import {getShufflingDecisionBlock, CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import {getShufflingDecisionBlock} from "@lodestar/state-transition"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; @@ -14,7 +14,7 @@ describe("ShufflingCache", function () { beforeEach(() => { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.processState(state as CachedBeaconStateAllForks, currentEpoch); + shufflingCache.processState(state, currentEpoch); }); it("should get shuffling from cache", async function () { @@ -29,7 +29,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shufflings at other epochs does prune the cache - shufflingCache.processState(state as CachedBeaconStateAllForks, currentEpoch + 1); + shufflingCache.processState(state, currentEpoch + 1); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); }); @@ -39,7 +39,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - shufflingCache.processState(state as CachedBeaconStateAllForks, currentEpoch + 1); + shufflingCache.processState(state, currentEpoch + 1); expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); }); diff --git a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts index 6cc3ae8f1ce0..316f75efc5ce 100644 --- a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts +++ b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts @@ -74,7 +74,7 @@ describe("eth1 / util / deposits", function () { expectedReturnedIndexes: [], }, { - id: "No deposits to be included post Electra after deposit_receipts_start_index", + id: "No deposits to be included post Electra after deposit_requests_start_index", depositCount: 2030, eth1DepositIndex: 2025, depositIndexes: Array.from({length: 2030}, (_, i) => i), @@ -82,7 +82,7 @@ describe("eth1 / util / deposits", function () { postElectra: true, }, { - id: "Should return deposits post Electra before deposit_receipts_start_index", + id: "Should return deposits post Electra before deposit_requests_start_index", depositCount: 2022, eth1DepositIndex: 2018, depositIndexes: Array.from({length: 2022}, (_, i) => i), @@ -90,7 +90,7 @@ describe("eth1 / util / deposits", function () { postElectra: true, }, { - id: "Should return deposits less than MAX_DEPOSITS post Electra before deposit_receipts_start_index", + id: "Should return deposits less than MAX_DEPOSITS post Electra before deposit_requests_start_index", depositCount: 10 * MAX_DEPOSITS, eth1DepositIndex: 0, depositIndexes: Array.from({length: 10 * MAX_DEPOSITS}, (_, i) => i), diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 643b6b1bac0f..6fe6ae15c448 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -88,7 +88,7 @@ describe("attestation SSZ serialized picking", () => { } }); - it("getAggregateionBitsFromAttestationSerialized - invalid data", () => { + it("getAggregationBitsFromAttestationSerialized - invalid data", () => { const invalidAggregationBitsDataSizes = [0, 4, 100, 128, 227]; for (const size of invalidAggregationBitsDataSizes) { expect(getAggregationBitsFromAttestationSerialized(ForkName.phase0, Buffer.alloc(size))).toBeNull(); diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index 82bdca901889..c33d942dabc5 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -4,7 +4,6 @@ import { computeSigningRoot, computeStartSlotAtEpoch, getShufflingDecisionBlock, - CachedBeaconStateAllForks, } from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; @@ -83,8 +82,8 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { }; const shufflingCache = new ShufflingCache(); - shufflingCache.processState(state as CachedBeaconStateAllForks, state.epochCtx.currentShuffling.epoch); - shufflingCache.processState(state as CachedBeaconStateAllForks, state.epochCtx.nextShuffling.epoch); + shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); + shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); const forkChoice = { @@ -134,7 +133,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { getState: async () => state, // TODO: remove this once we have a better way to get state getStateSync: () => state, - } as unknown as Partial as IStateRegenerator; + } as Partial as IStateRegenerator; const chain = { clock, diff --git a/packages/flare/src/cmds/selfSlashAttester.ts b/packages/flare/src/cmds/selfSlashAttester.ts index cb29ed94bb74..e29a956a9306 100644 --- a/packages/flare/src/cmds/selfSlashAttester.ts +++ b/packages/flare/src/cmds/selfSlashAttester.ts @@ -1,6 +1,6 @@ import {SecretKey, aggregateSignatures} from "@chainsafe/blst"; import {getClient} from "@lodestar/api"; -import {phase0, ssz} from "@lodestar/types"; +import {AttesterSlashing, phase0, ssz} from "@lodestar/types"; import {config as chainConfig} from "@lodestar/config/default"; import {createBeaconConfig, BeaconConfig} from "@lodestar/config"; import {DOMAIN_BEACON_ATTESTER, MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; @@ -117,7 +117,7 @@ export async function selfSlashAttesterHandler(args: SelfSlashArgs): Promise Date: Tue, 13 Aug 2024 21:18:16 +0100 Subject: [PATCH 086/145] chore: review electra branch - part 2 (#7019) --- .../src/block/processOperations.ts | 2 +- .../src/signatureSets/indexedAttestation.ts | 2 +- .../src/slot/upgradeStateToElectra.ts | 2 +- .../state-transition/src/stateTransition.ts | 12 ++++++------ .../test/unit/util/deposit.test.ts | 6 +++--- packages/types/src/electra/sszTypes.ts | 18 +++++++++--------- packages/validator/src/services/attestation.ts | 3 ++- .../validator/src/services/validatorStore.ts | 7 ++----- 8 files changed, 25 insertions(+), 27 deletions(-) diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 6d4fb25b9f5b..6f61e7c242fb 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -72,7 +72,7 @@ export function processOperations( } for (const elWithdrawalRequest of bodyElectra.executionPayload.withdrawalRequests) { - processWithdrawalRequest(fork, state as CachedBeaconStateElectra, elWithdrawalRequest); + processWithdrawalRequest(fork, stateElectra, elWithdrawalRequest); } for (const elConsolidationRequest of bodyElectra.executionPayload.consolidationRequests) { diff --git a/packages/state-transition/src/signatureSets/indexedAttestation.ts b/packages/state-transition/src/signatureSets/indexedAttestation.ts index d3decaf6406f..86535fece8b8 100644 --- a/packages/state-transition/src/signatureSets/indexedAttestation.ts +++ b/packages/state-transition/src/signatureSets/indexedAttestation.ts @@ -45,7 +45,7 @@ export function getAttestationsSignatureSets( return signedBlock.message.body.attestations.map((attestation) => getIndexedAttestationSignatureSet( state, - state.epochCtx.getIndexedAttestation(state.epochCtx.config.getForkSeq(signedBlock.message.slot), attestation) + state.epochCtx.getIndexedAttestation(state.config.getForkSeq(signedBlock.message.slot), attestation) ) ); } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index ef7b0bd661b4..6600ad98e80a 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -11,7 +11,7 @@ import {computeActivationExitEpoch} from "../util/epoch.js"; import {getActivationExitChurnLimit, getConsolidationChurnLimit} from "../util/validator.js"; /** - * Upgrade a state from Capella to Deneb. + * Upgrade a state from Deneb to Electra. */ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): CachedBeaconStateElectra { const {config} = stateDeneb; diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 5f433918415a..40e87c8d07d2 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -228,20 +228,20 @@ function processSlotsWithTransientCache( epochTransitionTimer?.(); // Upgrade state if exactly at epoch boundary - const stateSlot = computeEpochAtSlot(postState.slot); - if (stateSlot === config.ALTAIR_FORK_EPOCH) { + const stateEpoch = computeEpochAtSlot(postState.slot); + if (stateEpoch === config.ALTAIR_FORK_EPOCH) { postState = upgradeStateToAltair(postState as CachedBeaconStatePhase0) as CachedBeaconStateAllForks; } - if (stateSlot === config.BELLATRIX_FORK_EPOCH) { + if (stateEpoch === config.BELLATRIX_FORK_EPOCH) { postState = upgradeStateToBellatrix(postState as CachedBeaconStateAltair) as CachedBeaconStateAllForks; } - if (stateSlot === config.CAPELLA_FORK_EPOCH) { + if (stateEpoch === config.CAPELLA_FORK_EPOCH) { postState = upgradeStateToCapella(postState as CachedBeaconStateBellatrix) as CachedBeaconStateAllForks; } - if (stateSlot === config.DENEB_FORK_EPOCH) { + if (stateEpoch === config.DENEB_FORK_EPOCH) { postState = upgradeStateToDeneb(postState as CachedBeaconStateCapella) as CachedBeaconStateAllForks; } - if (stateSlot === config.ELECTRA_FORK_EPOCH) { + if (stateEpoch === config.ELECTRA_FORK_EPOCH) { postState = upgradeStateToElectra(postState as CachedBeaconStateDeneb) as CachedBeaconStateAllForks; } } else { diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts index b108de502a1f..3cfa4abb3409 100644 --- a/packages/state-transition/test/unit/util/deposit.test.ts +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -2,7 +2,7 @@ import {describe, it, expect} from "vitest"; import {ssz} from "@lodestar/types"; import {createChainForkConfig} from "@lodestar/config"; import {MAX_DEPOSITS} from "@lodestar/params"; -import {CachedBeaconStateElectra, getEth1DepositCount} from "../../../src/index.js"; +import {getEth1DepositCount} from "../../../src/index.js"; import {createCachedBeaconStateTest} from "../../utils/state.js"; describe("getEth1DepositCount", () => { @@ -37,7 +37,7 @@ describe("getEth1DepositCount", () => { ELECTRA_FORK_EPOCH: 0, }), {skipSyncCommitteeCache: true, skipSyncPubkeys: true} - ) as CachedBeaconStateElectra; + ); if (!postElectraState.epochCtx.isPostElectra()) { throw Error("Not a post-Electra state"); @@ -71,7 +71,7 @@ describe("getEth1DepositCount", () => { ELECTRA_FORK_EPOCH: 0, }), {skipSyncCommitteeCache: true, skipSyncPubkeys: true} - ) as CachedBeaconStateElectra; + ); if (!postElectraState.epochCtx.isPostElectra()) { throw Error("Not a post-Electra state"); diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index a6874c7894e3..31844aac86cc 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -152,7 +152,7 @@ export const ExecutionPayload = new ContainerType( ...denebSsz.ExecutionPayload.fields, depositRequests: DepositRequests, // New in ELECTRA withdrawalRequests: WithdrawalRequests, // New in ELECTRA - consolidationRequests: ConsolidationRequests, // [New in Electra] + consolidationRequests: ConsolidationRequests, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -333,14 +333,14 @@ export const BeaconState = new ContainerType( // Deep history valid from Capella onwards historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, depositRequestsStartIndex: UintBn64, // New in ELECTRA:EIP6110 - depositBalanceToConsume: Gwei, // New in Electra:EIP7251 - exitBalanceToConsume: Gwei, // New in Electra:EIP7251 - earliestExitEpoch: Epoch, // New in Electra:EIP7251 - consolidationBalanceToConsume: Gwei, // New in Electra:EIP7251 - earliestConsolidationEpoch: Epoch, // New in Electra:EIP7251 - pendingBalanceDeposits: PendingBalanceDeposits, // New in Electra:EIP7251 - pendingPartialWithdrawals: new ListCompositeType(PendingPartialWithdrawal, PENDING_PARTIAL_WITHDRAWALS_LIMIT), // New in Electra:EIP7251 - pendingConsolidations: new ListCompositeType(PendingConsolidation, PENDING_CONSOLIDATIONS_LIMIT), // New in Electra:EIP7251 + depositBalanceToConsume: Gwei, // New in ELECTRA:EIP7251 + exitBalanceToConsume: Gwei, // New in ELECTRA:EIP7251 + earliestExitEpoch: Epoch, // New in ELECTRA:EIP7251 + consolidationBalanceToConsume: Gwei, // New in ELECTRA:EIP7251 + earliestConsolidationEpoch: Epoch, // New in ELECTRA:EIP7251 + pendingBalanceDeposits: PendingBalanceDeposits, // New in ELECTRA:EIP7251 + pendingPartialWithdrawals: new ListCompositeType(PendingPartialWithdrawal, PENDING_PARTIAL_WITHDRAWALS_LIMIT), // New in ELECTRA:EIP7251 + pendingConsolidations: new ListCompositeType(PendingConsolidation, PENDING_CONSOLIDATIONS_LIMIT), // New in ELECTRA:EIP7251 }, {typeName: "BeaconState", jsonCase: "eth2"} ); diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index d6f48aeb3d1f..604f4adeb47e 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,5 +1,6 @@ import {toHexString} from "@chainsafe/ssz"; import {BLSSignature, phase0, Slot, ssz, Attestation, SignedAggregateAndProof} from "@lodestar/types"; +import {ForkSeq} from "@lodestar/params"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; @@ -159,7 +160,7 @@ export class AttestationService { this.metrics?.attesterStepCallProduceAggregate.observe(this.clock.secFromSlot(slot + 2 / 3)); const dutiesByCommitteeIndex = groupAttDutiesByCommitteeIndex(dutiesAll); - const isPostElectra = computeEpochAtSlot(slot) >= this.config.ELECTRA_FORK_EPOCH; + const isPostElectra = this.config.getForkSeq(slot) >= ForkSeq.electra; // Then download, sign and publish a `SignedAggregateAndProof` for each // validator that is elected to aggregate for this `slot` and `committeeIndex`. diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 252d0886cc6e..fa9d855aa24a 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -534,8 +534,8 @@ export class ValidatorStore { return { aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), data: attestationData, - committeeBits: BitArray.fromSingleBit(MAX_COMMITTEES_PER_SLOT, duty.committeeIndex), signature: await this.getSignature(duty.pubkey, signingRoot, signingSlot, signableMessage), + committeeBits: BitArray.fromSingleBit(MAX_COMMITTEES_PER_SLOT, duty.committeeIndex), }; } else { return { @@ -801,7 +801,7 @@ export class ValidatorStore { throw Error(`Inconsistent duties during signing: duty.slot ${duty.slot} != att.slot ${data.slot}`); } - const isPostElectra = computeEpochAtSlot(duty.slot) >= this.config.ELECTRA_FORK_EPOCH; + const isPostElectra = this.config.getForkSeq(duty.slot) >= ForkSeq.electra; if (!isPostElectra && duty.committeeIndex != data.index) { throw Error( `Inconsistent duties during signing: duty.committeeIndex ${duty.committeeIndex} != att.committeeIndex ${data.index}` @@ -810,9 +810,6 @@ export class ValidatorStore { if (isPostElectra && data.index !== 0) { throw Error(`Non-zero committee index post-electra during signing: att.committeeIndex ${data.index}`); } - if (this.config.getForkSeq(duty.slot) >= ForkSeq.electra && data.index !== 0) { - throw Error(`Attestataion data index must be 0 post electra: index ${data.index}`); - } } private assertDoppelgangerSafe(pubKey: PubkeyHex | BLSPubkey): void { From a157b55577546819d51fff4c7cdcfdfbe5e63c37 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 15 Aug 2024 11:36:13 +0100 Subject: [PATCH 087/145] feat: support post-electra attestation and attester slashing events (#7026) --- packages/api/src/beacon/client/events.ts | 2 +- packages/api/src/beacon/routes/events.ts | 35 +++++++++++++++---- packages/api/src/beacon/server/events.ts | 2 +- .../api/test/unit/beacon/oapiSpec.test.ts | 2 +- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/api/src/beacon/client/events.ts b/packages/api/src/beacon/client/events.ts index 2d63925a738a..34f14f2e8397 100644 --- a/packages/api/src/beacon/client/events.ts +++ b/packages/api/src/beacon/client/events.ts @@ -13,7 +13,7 @@ export type ApiClient = ApiClientMethods; */ export function getClient(config: ChainForkConfig, baseUrl: string): ApiClient { const definitions = getDefinitions(config); - const eventSerdes = getEventSerdes(); + const eventSerdes = getEventSerdes(config); return { eventstream: async ({topics, signal, onEvent, onError, onClose}) => { diff --git a/packages/api/src/beacon/routes/events.ts b/packages/api/src/beacon/routes/events.ts index 23be5e7c2288..1f041aa30194 100644 --- a/packages/api/src/beacon/routes/events.ts +++ b/packages/api/src/beacon/routes/events.ts @@ -13,6 +13,9 @@ import { LightClientOptimisticUpdate, LightClientFinalityUpdate, SSEPayloadAttributes, + Attestation, + AttesterSlashing, + sszTypesFor, } from "@lodestar/types"; import {ForkName} from "@lodestar/params"; @@ -104,10 +107,10 @@ export type EventData = { block: RootHex; executionOptimistic: boolean; }; - [EventType.attestation]: phase0.Attestation; + [EventType.attestation]: Attestation; [EventType.voluntaryExit]: phase0.SignedVoluntaryExit; [EventType.proposerSlashing]: phase0.ProposerSlashing; - [EventType.attesterSlashing]: phase0.AttesterSlashing; + [EventType.attesterSlashing]: AttesterSlashing; [EventType.blsToExecutionChange]: capella.SignedBLSToExecutionChange; [EventType.finalizedCheckpoint]: { block: RootHex; @@ -184,7 +187,7 @@ export type TypeJson = { fromJson: (data: unknown) => T; // client }; -export function getTypeByEvent(): {[K in EventType]: TypeJson} { +export function getTypeByEvent(config: ChainForkConfig): {[K in EventType]: TypeJson} { // eslint-disable-next-line @typescript-eslint/naming-convention const WithVersion = (getType: (fork: ForkName) => TypeJson): TypeJson<{data: T; version: ForkName}> => { return { @@ -225,10 +228,28 @@ export function getTypeByEvent(): {[K in EventType]: TypeJson} { {jsonCase: "eth2"} ), - [EventType.attestation]: ssz.phase0.Attestation, + [EventType.attestation]: { + toJson: (attestation) => { + const fork = config.getForkName(attestation.data.slot); + return sszTypesFor(fork).Attestation.toJson(attestation); + }, + fromJson: (attestation) => { + const fork = config.getForkName((attestation as Attestation).data.slot); + return sszTypesFor(fork).Attestation.fromJson(attestation); + }, + }, [EventType.voluntaryExit]: ssz.phase0.SignedVoluntaryExit, [EventType.proposerSlashing]: ssz.phase0.ProposerSlashing, - [EventType.attesterSlashing]: ssz.phase0.AttesterSlashing, + [EventType.attesterSlashing]: { + toJson: (attesterSlashing) => { + const fork = config.getForkName(Number(attesterSlashing.attestation1.data.slot)); + return sszTypesFor(fork).AttesterSlashing.toJson(attesterSlashing); + }, + fromJson: (attesterSlashing) => { + const fork = config.getForkName(Number((attesterSlashing as AttesterSlashing).attestation1.data.slot)); + return sszTypesFor(fork).AttesterSlashing.fromJson(attesterSlashing); + }, + }, [EventType.blsToExecutionChange]: ssz.capella.SignedBLSToExecutionChange, [EventType.finalizedCheckpoint]: new ContainerType( @@ -269,8 +290,8 @@ export function getTypeByEvent(): {[K in EventType]: TypeJson} { } // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export function getEventSerdes() { - const typeByEvent = getTypeByEvent(); +export function getEventSerdes(config: ChainForkConfig) { + const typeByEvent = getTypeByEvent(config); return { toJson: (event: BeaconEvent): unknown => { diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index cbeae24f6908..96212f006d8f 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -3,7 +3,7 @@ import {ApiError, ApplicationMethods, FastifyRoutes, createFastifyRoutes} from " import {Endpoints, getDefinitions, eventTypes, getEventSerdes} from "../routes/events.js"; export function getRoutes(config: ChainForkConfig, methods: ApplicationMethods): FastifyRoutes { - const eventSerdes = getEventSerdes(); + const eventSerdes = getEventSerdes(config); const serverRoutes = createFastifyRoutes(getDefinitions(config), methods); return { diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index efce60a5338f..a84b2cc36767 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -108,7 +108,7 @@ describe("eventstream event data", () => { } }); - const eventSerdes = routes.events.getEventSerdes(); + const eventSerdes = routes.events.getEventSerdes(config); const knownTopics = new Set(Object.values(routes.events.eventTypes)); for (const [topic, {value}] of Object.entries(eventstreamExamples ?? {}).filter( From 7d900c5b37577ac875a953d0250da945ddb286ab Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Fri, 16 Aug 2024 22:40:14 +0800 Subject: [PATCH 088/145] feat: pre-electra support from attestation pool (#6998) * Initial commit * Update packages/beacon-node/src/chain/chain.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/api/impl/validator/index.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts Co-authored-by: Nico Flaig * address comment * Add unit test for attestation pool * fix: getSeenAttDataKey apis (#7009) * fix: getSeenAttDataKey apis * chore: use ForkName instead of ForkSeq * Update packages/beacon-node/src/util/sszBytes.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts Co-authored-by: Nico Flaig * address comment * Update error message * Update packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts Co-authored-by: Nico Flaig * address comment * Move determining post-electra fork out of loops --------- Co-authored-by: Nico Flaig Co-authored-by: twoeths --- .../src/api/impl/beacon/pool/index.ts | 20 ++- .../src/api/impl/validator/index.ts | 28 +++- packages/beacon-node/src/chain/chain.ts | 10 +- .../opPools/aggregatedAttestationPool.ts | 40 ++++-- .../src/chain/opPools/attestationPool.ts | 34 ++++- .../beacon-node/src/chain/opPools/utils.ts | 5 + .../chain/seenCache/seenAttestationData.ts | 7 +- .../src/chain/validation/aggregateAndProof.ts | 6 +- .../src/chain/validation/attestation.ts | 68 ++++++++-- .../network/processor/gossipQueues/index.ts | 7 +- packages/beacon-node/src/util/sszBytes.ts | 88 +++++-------- packages/beacon-node/test/mocks/clock.ts | 1 + .../test/mocks/mockedBeaconChain.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 9 +- .../perf/chain/validation/attestation.test.ts | 6 +- .../opPools/aggregatedAttestationPool.test.ts | 6 +- .../chain/opPools/attestationPool.test.ts | 120 ++++++++++++++++++ .../attestation/validateAttestation.test.ts | 83 +++++++++--- .../test/unit/util/sszBytes.test.ts | 32 +++-- .../src/block/processAttestationPhase0.ts | 1 - 20 files changed, 437 insertions(+), 136 deletions(-) create mode 100644 packages/beacon-node/test/unit/chain/opPools/attestationPool.test.ts diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index bc8c838a5401..398238aa2508 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -1,7 +1,7 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; -import {Epoch, ssz} from "@lodestar/types"; -import {ForkName, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; +import {Attestation, Epoch, isElectraAttestation, ssz} from "@lodestar/types"; +import {ForkName, SYNC_COMMITTEE_SUBNET_SIZE, isForkPostElectra} from "@lodestar/params"; import {validateApiAttestation} from "../../../../chain/validation/index.js"; import {validateApiAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js"; import {validateApiProposerSlashing} from "../../../../chain/validation/proposerSlashing.js"; @@ -16,6 +16,7 @@ import { SyncCommitteeError, } from "../../../../chain/errors/index.js"; import {validateGossipFnRetryUnknownRoot} from "../../../../network/processor/gossipHandlers.js"; +import {ApiError} from "../../errors.js"; export function getBeaconPoolApi({ chain, @@ -26,7 +27,15 @@ export function getBeaconPoolApi({ return { async getPoolAttestations({slot, committeeIndex}) { // Already filtered by slot - let attestations = chain.aggregatedAttestationPool.getAll(slot); + let attestations: Attestation[] = chain.aggregatedAttestationPool.getAll(slot); + const fork = chain.config.getForkName(slot ?? chain.clock.currentSlot); + + if (isForkPostElectra(fork)) { + throw new ApiError( + 400, + `Use getPoolAttestationsV2 to retrieve pool attestations for post-electra fork=${fork}` + ); + } if (committeeIndex !== undefined) { attestations = attestations.filter((attestation) => committeeIndex === attestation.data.index); @@ -39,6 +48,11 @@ export function getBeaconPoolApi({ // Already filtered by slot let attestations = chain.aggregatedAttestationPool.getAll(slot); const fork = chain.config.getForkName(slot ?? attestations[0]?.data.slot ?? chain.clock.currentSlot); + const isPostElectra = isForkPostElectra(fork); + + attestations = attestations.filter((attestation) => + isPostElectra ? isElectraAttestation(attestation) : !isElectraAttestation(attestation) + ); if (committeeIndex !== undefined) { attestations = attestations.filter((attestation) => committeeIndex === attestation.data.index); diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 2d453f0d24c7..a17c1418809e 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1073,9 +1073,31 @@ export function getValidatorApi( }; }, - // TODO Electra: Implement getAggregatedAttestation to properly handle pre-electra - async getAggregatedAttestation() { - throw new Error("Not implemented. Use getAggregatedAttestationV2 for now."); + async getAggregatedAttestation({attestationDataRoot, slot}) { + notWhileSyncing(); + + await waitForSlot(slot); // Must never request for a future slot > currentSlot + + const dataRootHex = toHex(attestationDataRoot); + const aggregate = chain.attestationPool.getAggregate(slot, null, dataRootHex); + const fork = chain.config.getForkName(slot); + + if (isForkPostElectra(fork)) { + throw new ApiError( + 400, + `Use getAggregatedAttestationV2 to retrieve aggregated attestations for post-electra fork=${fork}` + ); + } + + if (!aggregate) { + throw new ApiError(404, `No aggregated attestation for slot=${slot}, dataRoot=${dataRootHex}`); + } + + metrics?.production.producedAggregateParticipants.observe(aggregate.aggregationBits.getTrueBitIndexes().length); + + return { + data: aggregate, + }; }, async getAggregatedAttestationV2({attestationDataRoot, slot, committeeIndex}) { diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 598b26d7061f..b410a1e84655 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -133,7 +133,7 @@ export class BeaconChain implements IBeaconChain { // Ops pool readonly attestationPool: AttestationPool; - readonly aggregatedAttestationPool = new AggregatedAttestationPool(); + readonly aggregatedAttestationPool: AggregatedAttestationPool; readonly syncCommitteeMessagePool: SyncCommitteeMessagePool; readonly syncContributionAndProofPool = new SyncContributionAndProofPool(); readonly opPool = new OpPool(); @@ -226,7 +226,13 @@ export class BeaconChain implements IBeaconChain { if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal}); const preAggregateCutOffTime = (2 / 3) * this.config.SECONDS_PER_SLOT; - this.attestationPool = new AttestationPool(clock, preAggregateCutOffTime, this.opts?.preaggregateSlotDistance); + this.attestationPool = new AttestationPool( + config, + clock, + preAggregateCutOffTime, + this.opts?.preaggregateSlotDistance + ); + this.aggregatedAttestationPool = new AggregatedAttestationPool(this.config); this.syncCommitteeMessagePool = new SyncCommitteeMessagePool( clock, preAggregateCutOffTime, diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index b3e82c0c6366..e656915552b4 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -3,6 +3,7 @@ import {BitArray, toHexString} from "@chainsafe/ssz"; import { ForkName, ForkSeq, + isForkPostElectra, MAX_ATTESTATIONS, MAX_ATTESTATIONS_ELECTRA, MAX_COMMITTEES_PER_SLOT, @@ -30,6 +31,7 @@ import { } from "@lodestar/state-transition"; import {IForkChoice, EpochDifference} from "@lodestar/fork-choice"; import {MapDef, toRootHex, assert} from "@lodestar/utils"; +import {ChainForkConfig} from "@lodestar/config"; import {intersectUint8Arrays, IntersectResult} from "../../util/bitArray.js"; import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; import {InsertOutcome} from "./types.js"; @@ -101,6 +103,8 @@ export class AggregatedAttestationPool { >(() => new Map>()); private lowestPermissibleSlot = 0; + constructor(private readonly config: ChainForkConfig) {} + /** For metrics to track size of the pool */ getAttestationCount(): {attestationCount: number; attestationDataCount: number} { let attestationCount = 0; @@ -136,10 +140,20 @@ export class AggregatedAttestationPool { attestationGroupByIndex = new Map(); attestationGroupByIndexByDataHash.set(dataRootHex, attestationGroupByIndex); } - const committeeIndex = isElectraAttestation(attestation) - ? // this attestation is added to pool after validation - attestation.committeeBits.getSingleTrueBit() - : attestation.data.index; + + let committeeIndex; + + if (isForkPostElectra(this.config.getForkName(slot))) { + if (!isElectraAttestation(attestation)) { + throw Error(`Attestation should be type electra.Attestation for slot ${slot}`); + } + committeeIndex = attestation.committeeBits.getSingleTrueBit(); + } else { + if (isElectraAttestation(attestation)) { + throw Error(`Attestation should be type phase0.Attestation for slot ${slot}`); + } + committeeIndex = attestation.data.index; + } // this should not happen because attestation should be validated before reaching this assert.notNull(committeeIndex, "Committee index should not be null in aggregated attestation pool"); let attestationGroup = attestationGroupByIndex.get(committeeIndex); @@ -390,6 +404,10 @@ export class AggregatedAttestationPool { /** * Get all attestations optionally filtered by `attestation.data.slot` + * Note this function is not fork aware and can potentially return a mix + * of phase0.Attestations and electra.Attestations. + * Caller of this function is expected to filtered result if they desire + * a homogenous array. * @param bySlot slot to filter, `bySlot === attestation.data.slot` */ getAll(bySlot?: Slot): Attestation[] { @@ -504,8 +522,15 @@ export class MatchingDataAttestationGroup { */ getAttestationsForBlock(fork: ForkName, notSeenAttestingIndices: Set): AttestationNonParticipant[] { const attestations: AttestationNonParticipant[] = []; - const forkSeq = ForkSeq[fork]; + const isPostElectra = isForkPostElectra(fork); for (const {attestation} of this.attestations) { + if ( + (isPostElectra && !isElectraAttestation(attestation)) || + (!isPostElectra && isElectraAttestation(attestation)) + ) { + continue; + } + let notSeenAttesterCount = 0; const {aggregationBits} = attestation; for (const notSeenIndex of notSeenAttestingIndices) { @@ -514,13 +539,12 @@ export class MatchingDataAttestationGroup { } } - // if fork >= electra, should return electra-only attestations - if (notSeenAttesterCount > 0 && (forkSeq < ForkSeq.electra || isElectraAttestation(attestation))) { + if (notSeenAttesterCount > 0) { attestations.push({attestation, notSeenAttesterCount}); } } - const maxAttestation = forkSeq >= ForkSeq.electra ? MAX_ATTESTATIONS_PER_GROUP_ELECTRA : MAX_ATTESTATIONS_PER_GROUP; + const maxAttestation = isPostElectra ? MAX_ATTESTATIONS_PER_GROUP_ELECTRA : MAX_ATTESTATIONS_PER_GROUP; if (attestations.length <= maxAttestation) { return attestations; } else { diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index f125b8c941db..887448b1e553 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -2,9 +2,11 @@ import {BitArray} from "@chainsafe/ssz"; import {Signature, aggregateSignatures} from "@chainsafe/blst"; import {Slot, RootHex, isElectraAttestation, Attestation} from "@lodestar/types"; import {MapDef, assert} from "@lodestar/utils"; +import {isForkPostElectra} from "@lodestar/params"; +import {ChainForkConfig} from "@lodestar/config"; import {IClock} from "../../util/clock.js"; import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js"; -import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; +import {isElectraAggregate, pruneBySlot, signatureFromBytesNoCheck} from "./utils.js"; /** * The number of slots that will be stored in the pool. @@ -28,14 +30,15 @@ type AggregateFastPhase0 = { signature: Signature; }; -type AggregateFastElectra = AggregateFastPhase0 & {committeeBits: BitArray}; +export type AggregateFastElectra = AggregateFastPhase0 & {committeeBits: BitArray}; -type AggregateFast = AggregateFastPhase0 | AggregateFastElectra; +export type AggregateFast = AggregateFastPhase0 | AggregateFastElectra; /** Hex string of DataRoot `TODO` */ type DataRootHex = string; -type CommitteeIndex = number; +/** CommitteeIndex must be null for pre-electra. Must not be null post-electra */ +type CommitteeIndex = number | null; /** * A pool of `Attestation` that is specially designed to store "unaggregated" attestations from @@ -68,6 +71,7 @@ export class AttestationPool { private lowestPermissibleSlot = 0; constructor( + private readonly config: ChainForkConfig, private readonly clock: IClock, private readonly cutOffSecFromSlot: number, private readonly preaggregateSlotDistance = 0 @@ -103,6 +107,7 @@ export class AttestationPool { */ add(committeeIndex: CommitteeIndex, attestation: Attestation, attDataRootHex: RootHex): InsertOutcome { const slot = attestation.data.slot; + const fork = this.config.getForkName(slot); const lowestPermissibleSlot = this.lowestPermissibleSlot; // Reject any attestations that are too old. @@ -121,8 +126,14 @@ export class AttestationPool { throw new OpPoolError({code: OpPoolErrorCode.REACHED_MAX_PER_SLOT}); } - // this should not happen because attestation should be validated before reaching this - assert.notNull(committeeIndex, "Committee index should not be null in attestation pool"); + if (isForkPostElectra(fork)) { + // Electra only: this should not happen because attestation should be validated before reaching this + assert.notNull(committeeIndex, "Committee index should not be null in attestation pool post-electra"); + assert.true(isElectraAttestation(attestation), "Attestation should be type electra.Attestation"); + } else { + assert.true(!isElectraAttestation(attestation), "Attestation should be type phase0.Attestation"); + committeeIndex = null; // For pre-electra, committee index info is encoded in attDataRootIndex + } // Pre-aggregate the contribution with existing items let aggregateByIndex = aggregateByRoot.get(attDataRootHex); @@ -144,14 +155,23 @@ export class AttestationPool { /** * For validator API to get an aggregate */ - // TODO Electra: Change attestation pool to accomodate pre-electra request getAggregate(slot: Slot, committeeIndex: CommitteeIndex, dataRootHex: RootHex): Attestation | null { + const fork = this.config.getForkName(slot); + const isPostElectra = isForkPostElectra(fork); + committeeIndex = isPostElectra ? committeeIndex : null; + const aggregate = this.aggregateByIndexByRootBySlot.get(slot)?.get(dataRootHex)?.get(committeeIndex); if (!aggregate) { // TODO: Add metric for missing aggregates return null; } + if (isPostElectra) { + assert.true(isElectraAggregate(aggregate), "Aggregate should be type AggregateFastElectra"); + } else { + assert.true(!isElectraAggregate(aggregate), "Aggregate should be type AggregateFastPhase0"); + } + return fastToAttestation(aggregate); } diff --git a/packages/beacon-node/src/chain/opPools/utils.ts b/packages/beacon-node/src/chain/opPools/utils.ts index 039e95af6c9f..e136bf1d4094 100644 --- a/packages/beacon-node/src/chain/opPools/utils.ts +++ b/packages/beacon-node/src/chain/opPools/utils.ts @@ -2,6 +2,7 @@ import {Signature} from "@chainsafe/blst"; import {BLS_WITHDRAWAL_PREFIX} from "@lodestar/params"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Slot, capella} from "@lodestar/types"; +import {AggregateFast, AggregateFastElectra} from "./attestationPool.js"; /** * Prune a Map indexed by slot to keep the most recent slots, up to `slotsRetained` @@ -58,3 +59,7 @@ export function isValidBlsToExecutionChangeForBlockInclusion( return true; } + +export function isElectraAggregate(aggregate: AggregateFast): aggregate is AggregateFastElectra { + return (aggregate as AggregateFastElectra).committeeBits !== undefined; +} diff --git a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts index 17343e386fd7..a0aa6db35893 100644 --- a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts +++ b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts @@ -2,9 +2,14 @@ import {BitArray} from "@chainsafe/ssz"; import {CommitteeIndex, phase0, RootHex, Slot} from "@lodestar/types"; import {MapDef} from "@lodestar/utils"; import {Metrics} from "../../metrics/metrics.js"; -import {SeenAttDataKey} from "../../util/sszBytes.js"; import {InsertOutcome} from "../opPools/types.js"; +export type SeenAttDataKey = AttDataBase64 | AttDataCommitteeBitsBase64; +// pre-electra, AttestationData is used to cache attestations +type AttDataBase64 = string; +// electra, AttestationData + CommitteeBits are used to cache attestations +type AttDataCommitteeBitsBase64 = string; + export type AttestationDataCacheEntry = { // part of shuffling data, so this does not take memory committeeValidatorIndices: Uint32Array; diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 02c6613b170f..39a3700aacf9 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -9,11 +9,11 @@ import {toRootHex} from "@lodestar/utils"; import {IBeaconChain} from ".."; import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/index.js"; import {RegenCaller} from "../regen/index.js"; -import {getSeenAttDataKeyFromSignedAggregateAndProof} from "../../util/sszBytes.js"; import {getSelectionProofSignatureSet, getAggregateAndProofSignatureSet} from "./signatureSets/index.js"; import { getAttestationDataSigningRoot, getCommitteeIndices, + getSeenAttDataKeyFromSignedAggregateAndProof, getShufflingForAttestationVerification, verifyHeadBlockAndTargetRoot, verifyPropagationSlotRange, @@ -71,9 +71,7 @@ async function validateAggregateAndProof( const attData = aggregate.data; const attSlot = attData.slot; - const seenAttDataKey = serializedData - ? getSeenAttDataKeyFromSignedAggregateAndProof(ForkSeq[fork], serializedData) - : null; + const seenAttDataKey = serializedData ? getSeenAttDataKeyFromSignedAggregateAndProof(fork, serializedData) : null; const cachedAttData = seenAttDataKey ? chain.seenAttestationDatas.get(attSlot, seenAttDataKey) : null; let attIndex; diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 1dcf76cdc5ae..4ceaf64d658d 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -13,7 +13,14 @@ import { IndexedAttestation, } from "@lodestar/types"; import {ProtoBlock} from "@lodestar/fork-choice"; -import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, ForkName, ForkSeq, DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; +import { + ATTESTATION_SUBNET_COUNT, + SLOTS_PER_EPOCH, + ForkName, + ForkSeq, + DOMAIN_BEACON_ATTESTER, + isForkPostElectra, +} from "@lodestar/params"; import { computeEpochAtSlot, createSingleSignatureSetFromComponents, @@ -30,12 +37,15 @@ import {AttestationError, AttestationErrorCode, GossipAction} from "../errors/in import {MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC} from "../../constants/index.js"; import {RegenCaller} from "../regen/index.js"; import { - SeenAttDataKey, getAggregationBitsFromAttestationSerialized, - getSeenAttDataKey, + getAttDataFromAttestationSerialized, + getAttDataFromSignedAggregateAndProofElectra, + getCommitteeBitsFromAttestationSerialized, + getCommitteeBitsFromSignedAggregateAndProofElectra, + getAttDataFromSignedAggregateAndProofPhase0, getSignatureFromAttestationSerialized, } from "../../util/sszBytes.js"; -import {AttestationDataCacheEntry} from "../seenCache/seenAttestationData.js"; +import {AttestationDataCacheEntry, SeenAttDataKey} from "../seenCache/seenAttestationData.js"; import {sszDeserializeAttestation} from "../../network/gossip/topic.js"; import {Result, wrapError} from "../../util/wrapError.js"; import {IBeaconChain} from "../interface.js"; @@ -67,7 +77,7 @@ export type GossipAttestation = { attSlot: Slot; // for old LIFO linear gossip queue we don't have attDataBase64 // for indexed gossip queue we have attDataBase64 - seenAttestationKey?: SeenAttDataKey | null; + attDataBase64?: SeenAttDataKey | null; }; export type Step0Result = AttestationValidationResult & { @@ -270,11 +280,7 @@ async function validateGossipAttestationNoSignatureCheck( if (attestationOrBytes.serializedData) { // gossip const attSlot = attestationOrBytes.attSlot; - attDataKey = - // we always have seenAttestationKey from the IndexedGossipQueue, getSeenAttDataKey() just for backward - // compatible in case beaconAttestationBatchValidation is false - // TODO: remove beaconAttestationBatchValidation flag since the batch attestation is stable - attestationOrBytes.seenAttestationKey ?? getSeenAttDataKey(ForkSeq[fork], attestationOrBytes.serializedData); + attDataKey = getSeenAttDataKeyFromGossipAttestation(fork, attestationOrBytes); const cachedAttData = attDataKey !== null ? chain.seenAttestationDatas.get(attSlot, attDataKey) : null; if (cachedAttData === null) { const attestation = sszDeserializeAttestation(fork, attestationOrBytes.serializedData); @@ -790,3 +796,45 @@ export function computeSubnetForSlot(shuffling: EpochShuffling, slot: number, co const committeesSinceEpochStart = shuffling.committeesPerSlot * slotsSinceEpochStart; return (committeesSinceEpochStart + committeeIndex) % ATTESTATION_SUBNET_COUNT; } + +/** + * Return fork-dependent seen attestation key + * - for pre-electra, it's the AttestationData base64 + * - for electra and later, it's the AttestationData base64 + committeeBits base64 + * + * we always have attDataBase64 from the IndexedGossipQueue, getAttDataFromAttestationSerialized() just for backward compatible when beaconAttestationBatchValidation is false + * TODO: remove beaconAttestationBatchValidation flag since the batch attestation is stable + */ +export function getSeenAttDataKeyFromGossipAttestation( + fork: ForkName, + attestation: GossipAttestation +): SeenAttDataKey | null { + const {attDataBase64, serializedData} = attestation; + if (isForkPostElectra(fork)) { + const attData = attDataBase64 ?? getAttDataFromAttestationSerialized(serializedData); + const committeeBits = getCommitteeBitsFromAttestationSerialized(serializedData); + return attData && committeeBits ? attDataBase64 + committeeBits : null; + } + + // pre-electra + return attDataBase64 ?? getAttDataFromAttestationSerialized(serializedData); +} + +/** + * Extract attestation data key from SignedAggregateAndProof Uint8Array to use cached data from SeenAttestationDatas + * - for pre-electra, it's the AttestationData base64 + * - for electra and later, it's the AttestationData base64 + committeeBits base64 + */ +export function getSeenAttDataKeyFromSignedAggregateAndProof( + fork: ForkName, + aggregateAndProof: Uint8Array +): SeenAttDataKey | null { + if (isForkPostElectra(fork)) { + const attData = getAttDataFromSignedAggregateAndProofElectra(aggregateAndProof); + const committeeBits = getCommitteeBitsFromSignedAggregateAndProofElectra(aggregateAndProof); + return attData && committeeBits ? attData + committeeBits : null; + } + + // pre-electra + return getAttDataFromSignedAggregateAndProofPhase0(aggregateAndProof); +} diff --git a/packages/beacon-node/src/network/processor/gossipQueues/index.ts b/packages/beacon-node/src/network/processor/gossipQueues/index.ts index b38ee74279ca..347458c91445 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues/index.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues/index.ts @@ -1,8 +1,7 @@ import {mapValues} from "@lodestar/utils"; -import {ForkSeq} from "@lodestar/params"; import {GossipType} from "../../gossip/interface.js"; import {PendingGossipsubMessage} from "../types.js"; -import {getSeenAttDataKey} from "../../../util/sszBytes.js"; +import {getGossipAttestationIndex} from "../../../util/sszBytes.js"; import {LinearGossipQueue} from "./linear.js"; import { DropType, @@ -88,8 +87,8 @@ const indexedGossipQueueOpts: { [GossipType.beacon_attestation]: { maxLength: 24576, indexFn: (item: PendingGossipsubMessage) => { - const {topic, msg} = item; - return getSeenAttDataKey(ForkSeq[topic.fork], msg.data); + // Note indexFn is fork agnostic despite changes introduced in Electra + return getGossipAttestationIndex(item.msg.data); }, minChunkSize: MIN_SIGNATURE_SETS_TO_BATCH_VERIFY, maxChunkSize: MAX_GOSSIP_ATTESTATION_BATCH_SIZE, diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index 89f1dd20a473..81821b5eb710 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -9,11 +9,10 @@ import { } from "@lodestar/params"; export type BlockRootHex = RootHex; -export type SeenAttDataKey = AttDataBase64 | AttDataCommitteeBitsBase64; // pre-electra, AttestationData is used to cache attestations export type AttDataBase64 = string; -// electra, AttestationData + CommitteeBits are used to cache attestations -export type AttDataCommitteeBitsBase64 = string; +// electra, CommitteeBits +export type CommitteeBitsBase64 = string; // pre-electra // class Attestation(Container): @@ -48,6 +47,7 @@ const SIGNATURE_SIZE = 96; // shared Buffers to convert bytes to hex/base64 const blockRootBuf = Buffer.alloc(ROOT_SIZE); const attDataBuf = Buffer.alloc(ATTESTATION_DATA_SIZE); +const committeeBitsDataBuf = Buffer.alloc(COMMITTEE_BITS_SIZE); /** * Extract slot from attestation serialized bytes. @@ -77,37 +77,10 @@ export function getBlockRootFromAttestationSerialized(data: Uint8Array): BlockRo } /** - * Extract attestation data key from an attestation Uint8Array in order to index gossip queue and cache later in SeenAttestationDatas - */ -export function getSeenAttDataKey(forkSeq: ForkSeq, data: Uint8Array): SeenAttDataKey | null { - return forkSeq >= ForkSeq.electra ? getSeenAttDataKeyElectra(data) : getSeenAttDataKeyPhase0(data); -} - -/** - * Extract attestation data + committeeBits base64 from electra attestation serialized bytes. - * Return null if data is not long enough to extract attestation data. - */ -export function getSeenAttDataKeyElectra(electraAttestationBytes: Uint8Array): AttDataCommitteeBitsBase64 | null { - const attestationData = getSeenAttDataKeyPhase0(electraAttestationBytes); - - if (attestationData === null) { - return null; - } - - const committeeBits = getCommitteeBitsFromAttestationSerialized(electraAttestationBytes); - - if (committeeBits === null) { - return null; - } - - return attestationData + toBase64(committeeBits.uint8Array); -} - -/** - * Extract attestation data base64 from phase0 attestation serialized bytes. + * Extract attestation data base64 from all forks' attestation serialized bytes. * Return null if data is not long enough to extract attestation data. */ -export function getSeenAttDataKeyPhase0(data: Uint8Array): AttDataBase64 | null { +export function getAttDataFromAttestationSerialized(data: Uint8Array): AttDataBase64 | null { if (data.length < VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE) { return null; } @@ -117,6 +90,13 @@ export function getSeenAttDataKeyPhase0(data: Uint8Array): AttDataBase64 | null return attDataBuf.toString("base64"); } +/** + * Alias of `getAttDataFromAttestationSerialized` specifically for batch handling indexing in gossip queue + */ +export function getGossipAttestationIndex(data: Uint8Array): AttDataBase64 | null { + return getAttDataFromAttestationSerialized(data); +} + /** * Extract aggregation bits from attestation serialized bytes. * Return null if data is not long enough to extract aggregation bits. @@ -153,16 +133,15 @@ export function getSignatureFromAttestationSerialized(data: Uint8Array): BLSSign * Extract committee bits from Electra attestation serialized bytes. * Return null if data is not long enough to extract committee bits. */ -export function getCommitteeBitsFromAttestationSerialized(data: Uint8Array): BitArray | null { +export function getCommitteeBitsFromAttestationSerialized(data: Uint8Array): CommitteeBitsBase64 | null { const committeeBitsStartIndex = VARIABLE_FIELD_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE; if (data.length < committeeBitsStartIndex + COMMITTEE_BITS_SIZE) { return null; } - const uint8Array = data.subarray(committeeBitsStartIndex, committeeBitsStartIndex + COMMITTEE_BITS_SIZE); - - return new BitArray(uint8Array, MAX_COMMITTEES_PER_SLOT); + committeeBitsDataBuf.set(data.subarray(committeeBitsStartIndex, committeeBitsStartIndex + COMMITTEE_BITS_SIZE)); + return committeeBitsDataBuf.toString("base64"); } // @@ -213,42 +192,39 @@ export function getBlockRootFromSignedAggregateAndProofSerialized(data: Uint8Arr } /** - * Extract attestation data key from SignedAggregateAndProof Uint8Array to use cached data from SeenAttestationDatas - */ -export function getSeenAttDataKeyFromSignedAggregateAndProof( - forkSeq: ForkSeq, - data: Uint8Array -): SeenAttDataKey | null { - return forkSeq >= ForkSeq.electra - ? getSeenAttDataKeyFromSignedAggregateAndProofElectra(data) - : getSeenAttDataKeyFromSignedAggregateAndProofPhase0(data); -} - -/** - * Extract AttestationData + CommitteeBits from SignedAggregateAndProof for electra + * Extract AttestationData base64 from SignedAggregateAndProof for electra * Return null if data is not long enough */ -export function getSeenAttDataKeyFromSignedAggregateAndProofElectra(data: Uint8Array): SeenAttDataKey | null { +export function getAttDataFromSignedAggregateAndProofElectra(data: Uint8Array): AttDataBase64 | null { const startIndex = SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET; const endIndex = startIndex + ATTESTATION_DATA_SIZE; if (data.length < endIndex + SIGNATURE_SIZE + COMMITTEE_BITS_SIZE) { return null; } + return toBase64(data.subarray(startIndex, endIndex)); +} - // base64 is a bit efficient than hex +/** + * Extract CommitteeBits base64 from SignedAggregateAndProof for electra + * Return null if data is not long enough + */ +export function getCommitteeBitsFromSignedAggregateAndProofElectra(data: Uint8Array): CommitteeBitsBase64 | null { + const startIndex = SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE + SIGNATURE_SIZE; + const endIndex = startIndex + COMMITTEE_BITS_SIZE; + + if (data.length < endIndex) { + return null; + } - return Buffer.concat([ - data.subarray(startIndex, endIndex), - data.subarray(endIndex + SIGNATURE_SIZE, endIndex + SIGNATURE_SIZE + COMMITTEE_BITS_SIZE), - ]).toString("base64"); + return toBase64(data.subarray(startIndex, endIndex)); } /** * Extract attestation data base64 from signed aggregate and proof serialized bytes. * Return null if data is not long enough to extract attestation data. */ -export function getSeenAttDataKeyFromSignedAggregateAndProofPhase0(data: Uint8Array): AttDataBase64 | null { +export function getAttDataFromSignedAggregateAndProofPhase0(data: Uint8Array): AttDataBase64 | null { if (data.length < SIGNED_AGGREGATE_AND_PROOF_SLOT_OFFSET + ATTESTATION_DATA_SIZE) { return null; } diff --git a/packages/beacon-node/test/mocks/clock.ts b/packages/beacon-node/test/mocks/clock.ts index c38794bf16d4..6f09bd292491 100644 --- a/packages/beacon-node/test/mocks/clock.ts +++ b/packages/beacon-node/test/mocks/clock.ts @@ -74,5 +74,6 @@ export function getMockedClock(): Mocked { }, currentSlotWithGossipDisparity: undefined, isCurrentSlotGivenGossipDisparity: vi.fn(), + secFromSlot: vi.fn(), } as unknown as Mocked; } diff --git a/packages/beacon-node/test/mocks/mockedBeaconChain.ts b/packages/beacon-node/test/mocks/mockedBeaconChain.ts index cc85cfd7d553..addeacf26a89 100644 --- a/packages/beacon-node/test/mocks/mockedBeaconChain.ts +++ b/packages/beacon-node/test/mocks/mockedBeaconChain.ts @@ -124,7 +124,7 @@ vi.mock("../../src/chain/chain.js", async (importActual) => { // @ts-expect-error eth1: new Eth1ForBlockProduction(), opPool: new OpPool(), - aggregatedAttestationPool: new AggregatedAttestationPool(), + aggregatedAttestationPool: new AggregatedAttestationPool(config), // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error beaconProposerCache: new BeaconProposerCache(), diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 60ff6ce48302..ec66f6dd8905 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -10,8 +10,9 @@ import { import {HISTORICAL_ROOTS_LIMIT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {ssz} from "@lodestar/types"; -// eslint-disable-next-line import/no-relative-packages -import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; + +import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; +import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test/perf/util.js"; import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; import {computeAnchorCheckpoint} from "../../../../src/chain/initState.js"; @@ -230,7 +231,9 @@ function getAggregatedAttestationPool( numMissedVotes: number, numBadVotes: number ): AggregatedAttestationPool { - const pool = new AggregatedAttestationPool(); + const config = createChainForkConfig(defaultChainConfig); + + const pool = new AggregatedAttestationPool(config); for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) { const slot = state.slot - 1 - epochSlot; const epoch = computeEpochAtSlot(slot); diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index e0e0a1e51169..f285317474d6 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -5,7 +5,7 @@ import {ssz} from "@lodestar/types"; import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; import {validateAttestation, validateGossipAttestationsSameAttData} from "../../../../src/chain/validation/index.js"; import {getAttestationValidData} from "../../../utils/validationData/attestation.js"; -import {getSeenAttDataKeyPhase0} from "../../../../src/util/sszBytes.js"; +import {getAttDataFromAttestationSerialized} from "../../../../src/util/sszBytes.js"; describe("validate gossip attestation", () => { setBenchOpts({ @@ -42,7 +42,7 @@ describe("validate gossip attestation", () => { attestation: null, serializedData, attSlot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet0 ); @@ -67,7 +67,7 @@ describe("validate gossip attestation", () => { attestation: null, serializedData, attSlot, - attDataBase64: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }; }); diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index c9c128e5485e..f00a300bbe4d 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -11,6 +11,7 @@ import { } from "@lodestar/params"; import {ssz, phase0} from "@lodestar/types"; import {CachedBeaconStateAltair} from "@lodestar/state-transition/src/types.js"; +import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; import {MockedForkChoice, getMockedForkChoice} from "../../../mocks/mockedBeaconChain.js"; import { aggregateConsolidation, @@ -36,6 +37,9 @@ const validSignature = fromHexString( describe("AggregatedAttestationPool", function () { let pool: AggregatedAttestationPool; const fork = ForkName.altair; + const config = createChainForkConfig({ + ...defaultChainConfig, + }); const altairForkEpoch = 2020; const currentEpoch = altairForkEpoch + 10; const currentSlot = SLOTS_PER_EPOCH * currentEpoch; @@ -79,7 +83,7 @@ describe("AggregatedAttestationPool", function () { let forkchoiceStub: MockedForkChoice; beforeEach(() => { - pool = new AggregatedAttestationPool(); + pool = new AggregatedAttestationPool(config); altairState = originalState.clone(); forkchoiceStub = getMockedForkChoice(); }); diff --git a/packages/beacon-node/test/unit/chain/opPools/attestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/attestationPool.test.ts new file mode 100644 index 000000000000..68efd0751585 --- /dev/null +++ b/packages/beacon-node/test/unit/chain/opPools/attestationPool.test.ts @@ -0,0 +1,120 @@ +import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {describe, it, expect, beforeEach, vi} from "vitest"; +import {GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {ssz} from "@lodestar/types"; +import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; +import {InsertOutcome} from "../../../../src/chain/opPools/types.js"; +import {AttestationPool} from "../../../../src/chain/opPools/attestationPool.js"; +import {getMockedClock} from "../../../mocks/clock.js"; + +/** Valid signature of random data to prevent BLS errors */ +export const validSignature = fromHexString( + "0xb2afb700f6c561ce5e1b4fedaec9d7c06b822d38c720cf588adfda748860a940adf51634b6788f298c552de40183b5a203b2bbe8b7dd147f0bb5bc97080a12efbb631c8888cb31a99cc4706eb3711865b8ea818c10126e4d818b542e9dbf9ae8" +); + +describe("AttestationPool", function () { + /* eslint-disable @typescript-eslint/naming-convention */ + const config = createChainForkConfig({ + ...defaultChainConfig, + ELECTRA_FORK_EPOCH: 5, + DENEB_FORK_EPOCH: 4, + CAPELLA_FORK_EPOCH: 3, + BELLATRIX_FORK_EPOCH: 2, + ALTAIR_FORK_EPOCH: 1, + }); + const clockStub = getMockedClock(); + vi.spyOn(clockStub, "secFromSlot").mockReturnValue(0); + + const cutOffSecFromSlot = (2 / 3) * config.SECONDS_PER_SLOT; + + // Mock attestations + const electraAttestationData = { + ...ssz.phase0.AttestationData.defaultValue(), + slot: config.ELECTRA_FORK_EPOCH * SLOTS_PER_EPOCH, + }; + const electraAttestation = { + ...ssz.electra.Attestation.defaultValue(), + data: electraAttestationData, + signature: validSignature, + }; + const phase0AttestationData = {...ssz.phase0.AttestationData.defaultValue(), slot: GENESIS_SLOT}; + const phase0Attestation = { + ...ssz.phase0.Attestation.defaultValue(), + data: phase0AttestationData, + signature: validSignature, + }; + + let pool: AttestationPool; + + beforeEach(() => { + pool = new AttestationPool(config, clockStub, cutOffSecFromSlot); + }); + + it("add correct electra attestation", () => { + const committeeIndex = 0; + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(electraAttestation.data)); + const outcome = pool.add(committeeIndex, electraAttestation, attDataRootHex); + + expect(outcome).equal(InsertOutcome.NewData); + expect(pool.getAggregate(electraAttestationData.slot, committeeIndex, attDataRootHex)).toEqual(electraAttestation); + }); + + it("add correct phase0 attestation", () => { + const committeeIndex = null; + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(phase0Attestation.data)); + const outcome = pool.add(committeeIndex, phase0Attestation, attDataRootHex); + + expect(outcome).equal(InsertOutcome.NewData); + expect(pool.getAggregate(phase0AttestationData.slot, committeeIndex, attDataRootHex)).toEqual(phase0Attestation); + expect(pool.getAggregate(phase0AttestationData.slot, 10, attDataRootHex)).toEqual(phase0Attestation); + expect(pool.getAggregate(phase0AttestationData.slot, 42, attDataRootHex)).toEqual(phase0Attestation); + expect(pool.getAggregate(phase0AttestationData.slot, null, attDataRootHex)).toEqual(phase0Attestation); + }); + + it("add electra attestation without committee index", () => { + const committeeIndex = null; + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(electraAttestation.data)); + + expect(() => pool.add(committeeIndex, electraAttestation, attDataRootHex)).toThrow(); + expect(pool.getAggregate(electraAttestationData.slot, committeeIndex, attDataRootHex)).toBeNull(); + }); + + it("add phase0 attestation with committee index", () => { + const committeeIndex = 0; + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(phase0Attestation.data)); + const outcome = pool.add(committeeIndex, phase0Attestation, attDataRootHex); + + expect(outcome).equal(InsertOutcome.NewData); + expect(pool.getAggregate(phase0AttestationData.slot, committeeIndex, attDataRootHex)).toEqual(phase0Attestation); + expect(pool.getAggregate(phase0AttestationData.slot, 123, attDataRootHex)).toEqual(phase0Attestation); + expect(pool.getAggregate(phase0AttestationData.slot, 456, attDataRootHex)).toEqual(phase0Attestation); + expect(pool.getAggregate(phase0AttestationData.slot, null, attDataRootHex)).toEqual(phase0Attestation); + }); + + it("add electra attestation with phase0 slot", () => { + const electraAttestationDataWithPhase0Slot = {...ssz.phase0.AttestationData.defaultValue(), slot: GENESIS_SLOT}; + const attestation = { + ...ssz.electra.Attestation.defaultValue(), + data: electraAttestationDataWithPhase0Slot, + signature: validSignature, + }; + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(electraAttestationDataWithPhase0Slot)); + + expect(() => pool.add(0, attestation, attDataRootHex)).toThrow(); + }); + + it("add phase0 attestation with electra slot", () => { + const phase0AttestationDataWithElectraSlot = { + ...ssz.phase0.AttestationData.defaultValue(), + slot: config.ELECTRA_FORK_EPOCH * SLOTS_PER_EPOCH, + }; + const attestation = { + ...ssz.phase0.Attestation.defaultValue(), + data: phase0AttestationDataWithElectraSlot, + signature: validSignature, + }; + const attDataRootHex = toHexString(ssz.phase0.AttestationData.hashTreeRoot(phase0AttestationDataWithElectraSlot)); + + expect(() => pool.add(0, attestation, attDataRootHex)).toThrow(); + }); +}); diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts index 4a1c3badae50..45d293ffbb33 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts @@ -1,6 +1,6 @@ import {BitArray} from "@chainsafe/ssz"; -import {describe, it} from "vitest"; -import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {describe, expect, it} from "vitest"; +import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../../state-transition/test/perf/util.js"; @@ -9,14 +9,17 @@ import {IBeaconChain} from "../../../../../src/chain/index.js"; import { ApiAttestation, GossipAttestation, + getSeenAttDataKeyFromGossipAttestation, + getSeenAttDataKeyFromSignedAggregateAndProof, validateApiAttestation, validateAttestation, } from "../../../../../src/chain/validation/index.js"; -import {getSeenAttDataKeyPhase0} from "../../../../../src/util/sszBytes.js"; +import {getAttDataFromAttestationSerialized} from "../../../../../src/util/sszBytes.js"; import {memoOnce} from "../../../../utils/cache.js"; import {expectRejectedWithLodestarError} from "../../../../utils/errors.js"; import {AttestationValidDataOpts, getAttestationValidData} from "../../../../utils/validationData/attestation.js"; +// TODO: more tests for electra describe("validateAttestation", () => { const vc = 64; const stateSlot = 100; @@ -52,7 +55,7 @@ describe("validateAttestation", () => { const {chain, subnet} = getValidData(); await expectGossipError( chain, - {attestation: null, serializedData: Buffer.alloc(0), attSlot: 0, seenAttestationKey: "invalid"}, + {attestation: null, serializedData: Buffer.alloc(0), attSlot: 0, attDataBase64: "invalid"}, subnet, GossipErrorCode.INVALID_SERIALIZED_BYTES_ERROR_CODE ); @@ -72,7 +75,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.BAD_TARGET_EPOCH @@ -91,7 +94,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.PAST_SLOT @@ -110,7 +113,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.FUTURE_SLOT @@ -135,7 +138,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -155,7 +158,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -179,7 +182,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT @@ -199,7 +202,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.INVALID_TARGET_ROOT @@ -226,7 +229,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS @@ -245,7 +248,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, invalidSubnet, AttestationErrorCode.INVALID_SUBNET_ID @@ -265,7 +268,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.ATTESTATION_ALREADY_KNOWN @@ -287,7 +290,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - seenAttestationKey: getSeenAttDataKeyPhase0(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData), }, subnet, AttestationErrorCode.INVALID_SIGNATURE @@ -314,3 +317,53 @@ describe("validateAttestation", () => { await expectRejectedWithLodestarError(validateAttestation(fork, chain, attestationOrBytes, subnet), errorCode); } }); + +describe("getSeenAttDataKey", () => { + const slot = 100; + const index = 0; + const blockRoot = Buffer.alloc(32, 1); + + it("phase0", () => { + const attestationData = ssz.phase0.AttestationData.defaultValue(); + attestationData.slot = slot; + attestationData.index = index; + attestationData.beaconBlockRoot = blockRoot; + const attestation = ssz.phase0.Attestation.defaultValue(); + attestation.data = attestationData; + const attDataBase64 = Buffer.from(ssz.phase0.AttestationData.serialize(attestationData)).toString("base64"); + const attestationBytes = ssz.phase0.Attestation.serialize(attestation); + const gossipAttestation = {attDataBase64, serializedData: attestationBytes, attSlot: slot} as GossipAttestation; + + const signedAggregateAndProof = ssz.phase0.SignedAggregateAndProof.defaultValue(); + signedAggregateAndProof.message.aggregate.data.slot = slot; + signedAggregateAndProof.message.aggregate.data.index = index; + signedAggregateAndProof.message.aggregate.data.beaconBlockRoot = blockRoot; + const aggregateAndProofBytes = ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof); + + expect(getSeenAttDataKeyFromGossipAttestation(ForkName.phase0, gossipAttestation)).toEqual( + getSeenAttDataKeyFromSignedAggregateAndProof(ForkName.phase0, aggregateAndProofBytes) + ); + }); + + it("electra", () => { + const attestationData = ssz.phase0.AttestationData.defaultValue(); + attestationData.slot = slot; + attestationData.index = index; + attestationData.beaconBlockRoot = blockRoot; + const attestation = ssz.electra.Attestation.defaultValue(); + attestation.data = attestationData; + const attDataBase64 = Buffer.from(ssz.phase0.AttestationData.serialize(attestationData)).toString("base64"); + const attestationBytes = ssz.electra.Attestation.serialize(attestation); + const gossipAttestation = {attDataBase64, serializedData: attestationBytes, attSlot: slot} as GossipAttestation; + + const signedAggregateAndProof = ssz.electra.SignedAggregateAndProof.defaultValue(); + signedAggregateAndProof.message.aggregate.data.slot = slot; + signedAggregateAndProof.message.aggregate.data.index = index; + signedAggregateAndProof.message.aggregate.data.beaconBlockRoot = blockRoot; + const aggregateAndProofBytes = ssz.electra.SignedAggregateAndProof.serialize(signedAggregateAndProof); + + expect(getSeenAttDataKeyFromGossipAttestation(ForkName.electra, gossipAttestation)).toEqual( + getSeenAttDataKeyFromSignedAggregateAndProof(ForkName.electra, aggregateAndProofBytes) + ); + }); +}); diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 6fe6ae15c448..612eea5a4388 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -4,8 +4,8 @@ import {deneb, electra, Epoch, isElectraAttestation, phase0, RootHex, Slot, ssz} import {fromHex, toHex} from "@lodestar/utils"; import {ForkName, MAX_COMMITTEES_PER_SLOT} from "@lodestar/params"; import { - getSeenAttDataKeyPhase0, - getSeenAttDataKeyFromSignedAggregateAndProofPhase0, + getAttDataFromAttestationSerialized, + getAttDataFromSignedAggregateAndProofPhase0, getAggregationBitsFromAttestationSerialized as getAggregationBitsFromAttestationSerialized, getBlockRootFromAttestationSerialized, getBlockRootFromSignedAggregateAndProofSerialized, @@ -15,7 +15,8 @@ import { getSlotFromSignedBeaconBlockSerialized, getSlotFromBlobSidecarSerialized, getCommitteeBitsFromAttestationSerialized, - getSeenAttDataKeyFromSignedAggregateAndProofElectra, + getCommitteeBitsFromSignedAggregateAndProofElectra, + getAttDataFromSignedAggregateAndProofElectra, } from "../../../src/util/sszBytes.js"; describe("attestation SSZ serialized picking", () => { @@ -53,7 +54,9 @@ describe("attestation SSZ serialized picking", () => { expect(getAggregationBitsFromAttestationSerialized(ForkName.electra, bytes)?.toBoolArray()).toEqual( attestation.aggregationBits.toBoolArray() ); - expect(getCommitteeBitsFromAttestationSerialized(bytes)).toEqual(attestation.committeeBits); + expect(getCommitteeBitsFromAttestationSerialized(bytes)).toEqual( + Buffer.from(attestation.committeeBits.uint8Array).toString("base64") + ); expect(getSignatureFromAttestationSerialized(bytes)).toEqual(attestation.signature); } else { expect(getAggregationBitsFromAttestationSerialized(ForkName.phase0, bytes)?.toBoolArray()).toEqual( @@ -63,7 +66,7 @@ describe("attestation SSZ serialized picking", () => { } const attDataBase64 = ssz.phase0.AttestationData.serialize(attestation.data); - expect(getSeenAttDataKeyPhase0(bytes)).toBe(Buffer.from(attDataBase64).toString("base64")); + expect(getAttDataFromAttestationSerialized(bytes)).toBe(Buffer.from(attDataBase64).toString("base64")); }); } @@ -81,10 +84,10 @@ describe("attestation SSZ serialized picking", () => { } }); - it("getAttDataBase64FromAttestationSerialized - invalid data", () => { + it("getAttDataFromAttestationSerialized - invalid data", () => { const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 131]; for (const size of invalidAttDataBase64DataSizes) { - expect(getSeenAttDataKeyPhase0(Buffer.alloc(size))).toBeNull(); + expect(getAttDataFromAttestationSerialized(Buffer.alloc(size))).toBeNull(); } }); @@ -128,9 +131,7 @@ describe("phase0 SignedAggregateAndProof SSZ serialized picking", () => { ); const attDataBase64 = ssz.phase0.AttestationData.serialize(signedAggregateAndProof.message.aggregate.data); - expect(getSeenAttDataKeyFromSignedAggregateAndProofPhase0(bytes)).toBe( - Buffer.from(attDataBase64).toString("base64") - ); + expect(getAttDataFromSignedAggregateAndProofPhase0(bytes)).toBe(Buffer.from(attDataBase64).toString("base64")); }); } @@ -151,7 +152,7 @@ describe("phase0 SignedAggregateAndProof SSZ serialized picking", () => { it("getAttDataBase64FromSignedAggregateAndProofSerialized - invalid data", () => { const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 339]; for (const size of invalidAttDataBase64DataSizes) { - expect(getSeenAttDataKeyFromSignedAggregateAndProofPhase0(Buffer.alloc(size))).toBeNull(); + expect(getAttDataFromSignedAggregateAndProofPhase0(Buffer.alloc(size))).toBeNull(); } }); }); @@ -182,8 +183,11 @@ describe("electra SignedAggregateAndProof SSZ serialized picking", () => { const committeeBits = ssz.electra.CommitteeBits.serialize( signedAggregateAndProof.message.aggregate.committeeBits ); - const seenKey = Buffer.concat([attDataBase64, committeeBits]).toString("base64"); - expect(getSeenAttDataKeyFromSignedAggregateAndProofElectra(bytes)).toBe(seenKey); + + expect(getAttDataFromSignedAggregateAndProofElectra(bytes)).toBe(Buffer.from(attDataBase64).toString("base64")); + expect(getCommitteeBitsFromSignedAggregateAndProofElectra(bytes)).toBe( + Buffer.from(committeeBits).toString("base64") + ); }); } @@ -204,7 +208,7 @@ describe("electra SignedAggregateAndProof SSZ serialized picking", () => { it("getAttDataBase64FromSignedAggregateAndProofSerialized - invalid data", () => { const invalidAttDataBase64DataSizes = [0, 4, 100, 128, 339]; for (const size of invalidAttDataBase64DataSizes) { - expect(getSeenAttDataKeyFromSignedAggregateAndProofPhase0(Buffer.alloc(size))).toBeNull(); + expect(getAttDataFromSignedAggregateAndProofPhase0(Buffer.alloc(size))).toBeNull(); } }); it("getSlotFromSignedAggregateAndProofSerialized - invalid data - large slots", () => { diff --git a/packages/state-transition/src/block/processAttestationPhase0.ts b/packages/state-transition/src/block/processAttestationPhase0.ts index ab21841069ec..ba6bc9089693 100644 --- a/packages/state-transition/src/block/processAttestationPhase0.ts +++ b/packages/state-transition/src/block/processAttestationPhase0.ts @@ -87,7 +87,6 @@ export function validateAttestation(fork: ForkSeq, state: CachedBeaconStateAllFo if (fork >= ForkSeq.electra) { assert.equal(data.index, 0, `AttestationData.index must be zero: index=${data.index}`); const attestationElectra = attestation as electra.Attestation; - // TODO Electra: this should be obsolete soon when the spec switches to committeeIndices const committeeIndices = attestationElectra.committeeBits.getTrueBitIndexes(); if (committeeIndices.length === 0) { From 3918aae2a0b2aae0bf678b73d62af01c2f7a4ac1 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 17 Aug 2024 18:44:52 +0100 Subject: [PATCH 089/145] fix: use attestation v1 endpoints pre-electra (#7024) * fix: use attestation v1 endpoints pre-electra * Adapt block attestations error message with pool apis --- .../src/api/impl/beacon/blocks/index.ts | 5 +- .../validator/src/services/attestation.ts | 29 ++++++-- .../test/unit/services/attestation.test.ts | 74 +++++++++++++------ 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 89e8a20d4f86..65e7b9373a22 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -410,7 +410,10 @@ export function getBeaconBlockApi({ const fork = config.getForkName(block.message.slot); if (isForkPostElectra(fork)) { - throw new ApiError(400, `Use getBlockAttestationsV2 to retrieve electra+ block attestations fork=${fork}`); + throw new ApiError( + 400, + `Use getBlockAttestationsV2 to retrieve block attestations for post-electra fork=${fork}` + ); } return { diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 604f4adeb47e..fc43b603c6b2 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -237,8 +237,11 @@ export class AttestationService { ...(this.opts?.disableAttestationGrouping && {index: attestationNoCommittee.index}), }; try { - // TODO Electra: Ensure calling V2 works in pre-electra - (await this.api.beacon.submitPoolAttestationsV2({signedAttestations})).assertOk(); + if (isPostElectra) { + (await this.api.beacon.submitPoolAttestationsV2({signedAttestations})).assertOk(); + } else { + (await this.api.beacon.submitPoolAttestations({signedAttestations})).assertOk(); + } this.logger.info("Published attestations", {...logCtx, count: signedAttestations.length}); this.metrics?.publishedAttestations.inc(signedAttestations.length); } catch (e) { @@ -264,6 +267,7 @@ export class AttestationService { duties: AttDutyAndProof[] ): Promise { const logCtx = {slot: attestation.slot, index: committeeIndex}; + const isPostElectra = this.config.getForkSeq(attestation.slot) >= ForkSeq.electra; // No validator is aggregator, skip if (duties.every(({selectionProof}) => selectionProof === null)) { @@ -271,11 +275,16 @@ export class AttestationService { } this.logger.verbose("Aggregating attestations", logCtx); - const res = await this.api.validator.getAggregatedAttestationV2({ - attestationDataRoot: ssz.phase0.AttestationData.hashTreeRoot(attestation), - slot: attestation.slot, - committeeIndex, - }); + const res = isPostElectra + ? await this.api.validator.getAggregatedAttestationV2({ + attestationDataRoot: ssz.phase0.AttestationData.hashTreeRoot(attestation), + slot: attestation.slot, + committeeIndex, + }) + : await this.api.validator.getAggregatedAttestation({ + attestationDataRoot: ssz.phase0.AttestationData.hashTreeRoot(attestation), + slot: attestation.slot, + }); const aggregate = res.value(); this.metrics?.numParticipantsInAggregate.observe(aggregate.aggregationBits.getTrueBitIndexes().length); @@ -302,7 +311,11 @@ export class AttestationService { if (signedAggregateAndProofs.length > 0) { try { - (await this.api.validator.publishAggregateAndProofsV2({signedAggregateAndProofs})).assertOk(); + if (isPostElectra) { + (await this.api.validator.publishAggregateAndProofsV2({signedAggregateAndProofs})).assertOk(); + } else { + (await this.api.validator.publishAggregateAndProofs({signedAggregateAndProofs})).assertOk(); + } this.logger.info("Published aggregateAndProofs", {...logCtx, count: signedAggregateAndProofs.length}); this.metrics?.publishedAggregates.inc(signedAggregateAndProofs.length); } catch (e) { diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index 591ac51fa05e..356503cbdd09 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -3,8 +3,8 @@ import {toHexString} from "@chainsafe/ssz"; import {SecretKey} from "@chainsafe/blst"; import {ssz} from "@lodestar/types"; import {routes} from "@lodestar/api"; -import {createChainForkConfig} from "@lodestar/config"; -import {config} from "@lodestar/config/default"; +import {ChainConfig, createChainForkConfig} from "@lodestar/config"; +import {config as defaultConfig} from "@lodestar/config/default"; import {ForkName} from "@lodestar/params"; import {AttestationService, AttestationServiceOpts} from "../../../src/services/attestation.js"; import {AttDutyAndProof} from "../../../src/services/attestationDuties.js"; @@ -52,16 +52,23 @@ describe("AttestationService", function () { vi.resetAllMocks(); }); - const testContexts: [string, AttestationServiceOpts][] = [ - ["With default configuration", {}], - ["With attestation grouping disabled", {disableAttestationGrouping: true}], - ["With distributed aggregation selection enabled", {distributedAggregationSelection: true}], + // eslint-disable-next-line @typescript-eslint/naming-convention + const electraConfig: Partial = {ELECTRA_FORK_EPOCH: 0}; + + const testContexts: [string, AttestationServiceOpts, Partial][] = [ + ["With default configuration", {}, {}], + ["With default configuration post-electra", {}, electraConfig], + ["With attestation grouping disabled", {disableAttestationGrouping: true}, {}], + ["With attestation grouping disabled post-electra", {disableAttestationGrouping: true}, electraConfig], + ["With distributed aggregation selection enabled", {distributedAggregationSelection: true}, {}], ]; - for (const [title, opts] of testContexts) { + for (const [title, opts, chainConfig] of testContexts) { describe(title, () => { it("Should produce, sign, and publish an attestation + aggregate", async () => { const clock = new ClockMock(); + const config = createChainForkConfig({...defaultConfig, ...chainConfig}); + const isPostElectra = chainConfig.ELECTRA_FORK_EPOCH === 0; const attestationService = new AttestationService( loggerVc, api, @@ -71,12 +78,16 @@ describe("AttestationService", function () { chainHeadTracker, syncingStatusTracker, null, - createChainForkConfig(config), + config, opts ); - const attestation = ssz.phase0.Attestation.defaultValue(); - const aggregate = ssz.phase0.SignedAggregateAndProof.defaultValue(); + const attestation = isPostElectra + ? ssz.electra.Attestation.defaultValue() + : ssz.phase0.Attestation.defaultValue(); + const aggregate = isPostElectra + ? ssz.electra.SignedAggregateAndProof.defaultValue() + : ssz.phase0.SignedAggregateAndProof.defaultValue(); const duties: AttDutyAndProof[] = [ { duty: { @@ -106,12 +117,17 @@ describe("AttestationService", function () { // Mock beacon's attestation and aggregates endpoints api.validator.produceAttestationData.mockResolvedValue(mockApiResponse({data: attestation.data})); - api.validator.getAggregatedAttestationV2.mockResolvedValue( - mockApiResponse({data: attestation, meta: {version: ForkName.phase0}}) - ); - - api.beacon.submitPoolAttestationsV2.mockResolvedValue(mockApiResponse({})); - api.validator.publishAggregateAndProofsV2.mockResolvedValue(mockApiResponse({})); + if (isPostElectra) { + api.validator.getAggregatedAttestationV2.mockResolvedValue( + mockApiResponse({data: attestation, meta: {version: ForkName.electra}}) + ); + api.beacon.submitPoolAttestationsV2.mockResolvedValue(mockApiResponse({})); + api.validator.publishAggregateAndProofsV2.mockResolvedValue(mockApiResponse({})); + } else { + api.validator.getAggregatedAttestation.mockResolvedValue(mockApiResponse({data: attestation})); + api.beacon.submitPoolAttestations.mockResolvedValue(mockApiResponse({})); + api.validator.publishAggregateAndProofs.mockResolvedValue(mockApiResponse({})); + } if (opts.distributedAggregationSelection) { // Mock distributed validator middleware client selections endpoint @@ -152,13 +168,25 @@ describe("AttestationService", function () { expect(api.validator.prepareBeaconCommitteeSubnet).toHaveBeenCalledWith({subscriptions: [subscription]}); } - // Must submit the attestation received through produceAttestationData() - expect(api.beacon.submitPoolAttestationsV2).toHaveBeenCalledOnce(); - expect(api.beacon.submitPoolAttestationsV2).toHaveBeenCalledWith({signedAttestations: [attestation]}); - - // Must submit the aggregate received through getAggregatedAttestationV2() then createAndSignAggregateAndProof() - expect(api.validator.publishAggregateAndProofsV2).toHaveBeenCalledOnce(); - expect(api.validator.publishAggregateAndProofsV2).toHaveBeenCalledWith({signedAggregateAndProofs: [aggregate]}); + if (isPostElectra) { + // Must submit the attestation received through produceAttestationData() + expect(api.beacon.submitPoolAttestationsV2).toHaveBeenCalledOnce(); + expect(api.beacon.submitPoolAttestationsV2).toHaveBeenCalledWith({signedAttestations: [attestation]}); + + // Must submit the aggregate received through getAggregatedAttestationV2() then createAndSignAggregateAndProof() + expect(api.validator.publishAggregateAndProofsV2).toHaveBeenCalledOnce(); + expect(api.validator.publishAggregateAndProofsV2).toHaveBeenCalledWith({ + signedAggregateAndProofs: [aggregate], + }); + } else { + // Must submit the attestation received through produceAttestationData() + expect(api.beacon.submitPoolAttestations).toHaveBeenCalledOnce(); + expect(api.beacon.submitPoolAttestations).toHaveBeenCalledWith({signedAttestations: [attestation]}); + + // Must submit the aggregate received through getAggregatedAttestation() then createAndSignAggregateAndProof() + expect(api.validator.publishAggregateAndProofs).toHaveBeenCalledOnce(); + expect(api.validator.publishAggregateAndProofs).toHaveBeenCalledWith({signedAggregateAndProofs: [aggregate]}); + } }); }); } From 5fd2d09368a22b4c080fb1189d94b739bc5ee64b Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 20 Aug 2024 16:57:00 +0530 Subject: [PATCH 090/145] fix: use correct partial withdrawal index to evaluate processing them --- packages/state-transition/src/block/processWithdrawals.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index 50113ef1741a..b06209167be3 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -105,9 +105,9 @@ export function getExpectedWithdrawals( if ( validator.exitEpoch === FAR_FUTURE_EPOCH && validator.effectiveBalance >= MIN_ACTIVATION_BALANCE && - balances.get(withdrawalIndex) > MIN_ACTIVATION_BALANCE + balances.get(withdrawal.index) > MIN_ACTIVATION_BALANCE ) { - const balanceOverMinActivationBalance = BigInt(balances.get(withdrawalIndex) - MIN_ACTIVATION_BALANCE); + const balanceOverMinActivationBalance = BigInt(balances.get(withdrawal.index) - MIN_ACTIVATION_BALANCE); const withdrawableBalance = balanceOverMinActivationBalance < withdrawal.amount ? balanceOverMinActivationBalance : withdrawal.amount; withdrawals.push({ From 23fb9fb442eb504f2ce52d1da8b45aeccdd7bc9a Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 20 Aug 2024 20:00:35 +0700 Subject: [PATCH 091/145] fix: ssz v0.17.1 (#7035) * fix: work around for sliceFrom() api * Revert "fix: work around for sliceFrom() api" This reverts commit 7aa6678cd46b1fa770f00923003a311a9d7a65da. * fix: upgrade ssz --- packages/api/package.json | 2 +- packages/beacon-node/package.json | 2 +- packages/cli/package.json | 2 +- packages/config/package.json | 2 +- packages/db/package.json | 2 +- packages/fork-choice/package.json | 2 +- packages/light-client/package.json | 2 +- packages/state-transition/package.json | 2 +- packages/types/package.json | 2 +- packages/validator/package.json | 2 +- yarn.lock | 30 ++++++++++++++++++++------ 11 files changed, 34 insertions(+), 16 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 49bcbbc34f59..138964ed946a 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -71,7 +71,7 @@ }, "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.8.0", - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index b56e1f1bb3e3..b41450f12d34 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -103,7 +103,7 @@ "@chainsafe/libp2p-noise": "^15.0.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/prometheus-gc-stats": "^1.0.0", - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@chainsafe/threads": "^1.11.1", "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^9.0.0", diff --git a/packages/cli/package.json b/packages/cli/package.json index b14d6d5ccafe..bf8621344dc5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -57,7 +57,7 @@ "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@chainsafe/threads": "^1.11.1", "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", diff --git a/packages/config/package.json b/packages/config/package.json index e306920aea49..45c462e19081 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -64,7 +64,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0" } diff --git a/packages/db/package.json b/packages/db/package.json index a68e4e6f2a8f..1a31603b6791 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -35,7 +35,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/config": "^1.21.0", "@lodestar/utils": "^1.21.0", "classic-level": "^1.4.1", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index ad493292566d..dfe4c942832d 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -36,7 +36,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", "@lodestar/state-transition": "^1.21.0", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 0bc1a5529c85..1188732c8870 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -76,7 +76,7 @@ "@chainsafe/bls": "7.1.3", "@chainsafe/blst": "^0.2.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/api": "^1.21.0", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index a892c5cace43..1dd43ef98f88 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -62,7 +62,7 @@ "@chainsafe/blst": "^2.0.3", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0", diff --git a/packages/types/package.json b/packages/types/package.json index f29229485e99..861dbdbee0ef 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -73,7 +73,7 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/params": "^1.21.0", "ethereum-cryptography": "^2.0.0" }, diff --git a/packages/validator/package.json b/packages/validator/package.json index 66de444c81b5..373e59817d2a 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -46,7 +46,7 @@ ], "dependencies": { "@chainsafe/blst": "^2.0.3", - "@chainsafe/ssz": "^0.17.0", + "@chainsafe/ssz": "^0.17.1", "@lodestar/api": "^1.21.0", "@lodestar/config": "^1.21.0", "@lodestar/db": "^1.21.0", diff --git a/yarn.lock b/yarn.lock index 7e2e3d8e6447..60e41c276e53 100644 --- a/yarn.lock +++ b/yarn.lock @@ -594,10 +594,10 @@ "@chainsafe/as-sha256" "^0.4.1" "@chainsafe/persistent-merkle-tree" "^0.6.1" -"@chainsafe/ssz@^0.17.0": - version "0.17.0" - resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.17.0.tgz#b154260f6885693fd77bfb2fff43dd8c3fa37edd" - integrity sha512-DEzyH9vF4zz+Zqe2EMZuxXyxV5+7cmmLwljL3VC3ApzmyPORsprGJM7xLaUJu3oMRKMdBpR8UjRNkfB2ROQJzQ== +"@chainsafe/ssz@^0.17.1": + version "0.17.1" + resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.17.1.tgz#7986afbcad5e6971006d596fdb7dfa34bc195131" + integrity sha512-1ay46QqYcVTBvUnDXTPTi5WTiENu7tIxpZGMDpUWps1/nYBmh/We/UoCF/jO+o/fkcDD3p8xQPlHbcCfy+jyjA== dependencies: "@chainsafe/as-sha256" "0.5.0" "@chainsafe/persistent-merkle-tree" "0.8.0" @@ -12080,7 +12080,16 @@ string-argv@~0.3.1: resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -13655,7 +13664,7 @@ workerpool@6.2.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343" integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -13673,6 +13682,15 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From cb7878bb7374aa7486352087c66f8d78d51ea52d Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 23 Aug 2024 14:42:14 +0530 Subject: [PATCH 092/145] chore: rebase fixes --- .../src/chain/opPools/aggregatedAttestationPool.ts | 2 +- packages/beacon-node/src/util/sszBytes.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index e656915552b4..9a3e6622d1ff 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -1,5 +1,5 @@ import {aggregateSignatures, Signature} from "@chainsafe/blst"; -import {BitArray, toHexString} from "@chainsafe/ssz"; +import {BitArray} from "@chainsafe/ssz"; import { ForkName, ForkSeq, diff --git a/packages/beacon-node/src/util/sszBytes.ts b/packages/beacon-node/src/util/sszBytes.ts index 81821b5eb710..c27df1a0fbf3 100644 --- a/packages/beacon-node/src/util/sszBytes.ts +++ b/packages/beacon-node/src/util/sszBytes.ts @@ -202,7 +202,8 @@ export function getAttDataFromSignedAggregateAndProofElectra(data: Uint8Array): if (data.length < endIndex + SIGNATURE_SIZE + COMMITTEE_BITS_SIZE) { return null; } - return toBase64(data.subarray(startIndex, endIndex)); + attDataBuf.set(data.subarray(startIndex, endIndex)); + return attDataBuf.toString("base64"); } /** @@ -217,7 +218,8 @@ export function getCommitteeBitsFromSignedAggregateAndProofElectra(data: Uint8Ar return null; } - return toBase64(data.subarray(startIndex, endIndex)); + committeeBitsDataBuf.set(data.subarray(startIndex, endIndex)); + return committeeBitsDataBuf.toString("base64"); } /** From 8ccbed70c888007126644add3d488e5db0be0f18 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 23 Aug 2024 12:11:19 +0100 Subject: [PATCH 093/145] chore: fix api handler used during testing (#7047) --- packages/beacon-node/test/utils/node/validator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index 4ec60dcc8b4f..285fa13fd01f 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -97,7 +97,7 @@ export function getApiFromServerHandlers(api: BeaconApiMethods): ApiClient { return async (args: unknown) => { try { const apiResponse = new ApiResponse({} as any, null, new Response(null, {status: HttpStatusCode.OK})); - const result = await api(args, {}); + const result = await api.call(apiModule, args, {}); apiResponse.value = () => result.data; apiResponse.meta = () => result.meta; return apiResponse; From ed18ff005d4482d0c372aebb2762873a335fa578 Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 27 Aug 2024 21:39:38 +0700 Subject: [PATCH 094/145] fix: improve processEffectiveBalanceUpdates (#7043) --- .../src/chain/historicalState/worker.ts | 4 +++ .../src/metrics/metrics/lodestar.ts | 4 +++ .../src/cache/epochTransitionCache.ts | 18 +++++++++-- packages/state-transition/src/epoch/index.ts | 3 +- .../epoch/processEffectiveBalanceUpdates.ts | 32 ++++++++++++------- .../src/epoch/processPendingConsolidations.ts | 4 +++ packages/state-transition/src/metrics.ts | 1 + .../test/perf/epoch/epochAltair.test.ts | 4 ++- .../test/perf/epoch/epochCapella.test.ts | 4 ++- .../test/perf/epoch/epochPhase0.test.ts | 4 ++- .../processEffectiveBalanceUpdates.test.ts | 4 ++- 11 files changed, 63 insertions(+), 19 deletions(-) diff --git a/packages/beacon-node/src/chain/historicalState/worker.ts b/packages/beacon-node/src/chain/historicalState/worker.ts index 9a9f9cc9cd0e..a07207cac5f5 100644 --- a/packages/beacon-node/src/chain/historicalState/worker.ts +++ b/packages/beacon-node/src/chain/historicalState/worker.ts @@ -82,6 +82,10 @@ if (metricsRegister) { buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5], labelNames: ["source"], }), + numEffectiveBalanceUpdates: metricsRegister.gauge({ + name: "lodestar_historical_state_stfn_num_effective_balance_updates_count", + help: "Count of effective balance updates in epoch transition", + }), preStateBalancesNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({ name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total", help: "Total count state.balances nodesPopulated is false on stfn", diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 6b4bdb111f41..a0cf0a185c2f 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -332,6 +332,10 @@ export function createLodestarMetrics( buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5], labelNames: ["source"], }), + numEffectiveBalanceUpdates: register.gauge({ + name: "lodestar_stfn_effective_balance_updates_count", + help: "Total count of effective balance updates", + }), preStateBalancesNodesPopulatedMiss: register.gauge<{source: StateCloneSource}>({ name: "lodestar_stfn_balances_nodes_populated_miss_total", help: "Total count state.balances nodesPopulated is false on stfn", diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 280524ec7beb..6f27ad96d1c8 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,4 +1,4 @@ -import {Epoch, ValidatorIndex} from "@lodestar/types"; +import {Epoch, ValidatorIndex, phase0} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; @@ -127,6 +127,18 @@ export interface EpochTransitionCache { flags: number[]; + /** + * Validators in the current epoch, should use it for read-only value instead of accessing state.validators directly. + * Note that during epoch processing, validators could be updated so need to use it with care. + */ + validators: phase0.Validator[]; + + /** + * This is for electra only + * Validators that're switched to compounding during processPendingConsolidations(), not available in beforeProcessEpoch() + */ + newCompoundingValidators?: Set; + /** * balances array will be populated by processRewardsAndPenalties() and consumed by processEffectiveBalanceUpdates(). * processRewardsAndPenalties() already has a regular Javascript array of balances. @@ -481,7 +493,9 @@ export function beforeProcessEpoch( proposerIndices, inclusionDelays, flags, - + validators, + // will be assigned in processPendingConsolidations() + newCompoundingValidators: undefined, // Will be assigned in processRewardsAndPenalties() balances: undefined, }; diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index 85e7c348dad3..bfb415b9ed6a 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -150,8 +150,9 @@ export function processEpoch( const timer = metrics?.epochTransitionStepTime.startTimer({ step: EpochTransitionStep.processEffectiveBalanceUpdates, }); - processEffectiveBalanceUpdates(fork, state, cache); + const numUpdate = processEffectiveBalanceUpdates(fork, state, cache); timer?.(); + metrics?.numEffectiveBalanceUpdates.set(numUpdate); } processSlashingsReset(state, cache); diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index fdbc99b9265e..0ea4b49dddf4 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -23,12 +23,14 @@ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; * * - On normal mainnet conditions 0 validators change their effective balance * - In case of big innactivity event a medium portion of validators may have their effectiveBalance updated + * + * Return number of validators updated */ export function processEffectiveBalanceUpdates( fork: ForkSeq, state: CachedBeaconStateAllForks, cache: EpochTransitionCache -): void { +): number { const HYSTERESIS_INCREMENT = EFFECTIVE_BALANCE_INCREMENT / HYSTERESIS_QUOTIENT; const DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER; const UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER; @@ -43,34 +45,38 @@ export function processEffectiveBalanceUpdates( // and updated in processPendingBalanceDeposits() and processPendingConsolidations() // so it's recycled here for performance. const balances = cache.balances ?? state.balances.getAll(); + const currentEpochValidators = cache.validators; + const newCompoundingValidators = cache.newCompoundingValidators ?? new Set(); + let numUpdate = 0; for (let i = 0, len = balances.length; i < len; i++) { const balance = balances[i]; // PERF: It's faster to access to get() every single element (4ms) than to convert to regular array then loop (9ms) let effectiveBalanceIncrement = effectiveBalanceIncrements[i]; let effectiveBalance = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT; - let effectiveBalanceLimit; + + let effectiveBalanceLimit: number; + if (fork < ForkSeq.electra) { + effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; + } else { + // from electra, effectiveBalanceLimit is per validator + const isCompoundingValidator = + hasCompoundingWithdrawalCredential(currentEpochValidators[i].withdrawalCredentials) || + newCompoundingValidators.has(i); + effectiveBalanceLimit = isCompoundingValidator ? MAX_EFFECTIVE_BALANCE_ELECTRA : MIN_ACTIVATION_BALANCE; + } if ( // Too big effectiveBalance > balance + DOWNWARD_THRESHOLD || // Too small. Check effectiveBalance < MAX_EFFECTIVE_BALANCE to prevent unnecessary updates - effectiveBalance + UPWARD_THRESHOLD < balance + (effectiveBalance < effectiveBalanceLimit && effectiveBalance + UPWARD_THRESHOLD < balance) ) { // Update the state tree // Should happen rarely, so it's fine to update the tree const validator = validators.get(i); - if (fork < ForkSeq.electra) { - effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; - } else { - // Electra or after - effectiveBalanceLimit = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials) - ? MAX_EFFECTIVE_BALANCE_ELECTRA - : MIN_ACTIVATION_BALANCE; - } - effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), effectiveBalanceLimit); validator.effectiveBalance = effectiveBalance; // Also update the fast cached version @@ -95,6 +101,7 @@ export function processEffectiveBalanceUpdates( effectiveBalanceIncrement = newEffectiveBalanceIncrement; effectiveBalanceIncrements[i] = effectiveBalanceIncrement; + numUpdate++; } // TODO: Do this in afterEpochTransitionCache, looping a Uint8Array should be very cheap @@ -105,4 +112,5 @@ export function processEffectiveBalanceUpdates( } cache.nextEpochTotalActiveBalanceByIncrement = nextEpochTotalActiveBalanceByIncrement; + return numUpdate; } diff --git a/packages/state-transition/src/epoch/processPendingConsolidations.ts b/packages/state-transition/src/epoch/processPendingConsolidations.ts index d6f760a6b27a..7d8548046e79 100644 --- a/packages/state-transition/src/epoch/processPendingConsolidations.ts +++ b/packages/state-transition/src/epoch/processPendingConsolidations.ts @@ -1,3 +1,4 @@ +import {ValidatorIndex} from "@lodestar/types"; import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js"; import {decreaseBalance, increaseBalance} from "../util/balance.js"; import {getActiveBalance} from "../util/validator.js"; @@ -20,6 +21,7 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca let nextPendingConsolidation = 0; const validators = state.validators; const cachedBalances = cache.balances; + const newCompoundingValidators = new Set(); for (const pendingConsolidation of state.pendingConsolidations.getAllReadonly()) { const {sourceIndex, targetIndex} = pendingConsolidation; @@ -35,6 +37,7 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca } // Churn any target excess active balance of target and raise its max switchToCompoundingValidator(state, targetIndex); + newCompoundingValidators.add(targetIndex); // Move active balance to target. Excess balance is withdrawable. const activeBalance = getActiveBalance(state, sourceIndex); decreaseBalance(state, sourceIndex, activeBalance); @@ -47,5 +50,6 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca nextPendingConsolidation++; } + cache.newCompoundingValidators = newCompoundingValidators; state.pendingConsolidations = state.pendingConsolidations.sliceFrom(nextPendingConsolidation); } diff --git a/packages/state-transition/src/metrics.ts b/packages/state-transition/src/metrics.ts index 48e0bc921876..a5e5463231fa 100644 --- a/packages/state-transition/src/metrics.ts +++ b/packages/state-transition/src/metrics.ts @@ -11,6 +11,7 @@ export type BeaconStateTransitionMetrics = { processBlockTime: Histogram; processBlockCommitTime: Histogram; stateHashTreeRootTime: Histogram<{source: StateHashTreeRootSource}>; + numEffectiveBalanceUpdates: Gauge; preStateBalancesNodesPopulatedMiss: Gauge<{source: StateCloneSource}>; preStateBalancesNodesPopulatedHit: Gauge<{source: StateCloneSource}>; preStateValidatorsNodesPopulatedMiss: Gauge<{source: StateCloneSource}>; diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 39e0a1b4c5c3..15cde849ce9f 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -141,7 +141,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - altair processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(ForkSeq.altair, state, cache.value), + fn: (state) => { + processEffectiveBalanceUpdates(ForkSeq.altair, state, cache.value); + }, }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index 5b8300df3f18..61bfad20b1ee 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -120,7 +120,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - capella processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(ForkSeq.capella, state, cache.value), + fn: (state) => { + processEffectiveBalanceUpdates(ForkSeq.capella, state, cache.value); + }, }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index 411577878102..3af3d4d4a832 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -123,7 +123,9 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - phase0 processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache.value), + fn: (state) => { + processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache.value); + }, }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts index d94daac9e59b..19f18df86c2e 100644 --- a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts @@ -36,7 +36,9 @@ describe("phase0 processEffectiveBalanceUpdates", () => { minRuns: 5, // Worst case is very slow before: () => getEffectiveBalanceTestData(vc, changeRatio), beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache), + fn: ({state, cache}) => { + processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache); + }, }); } }); From 7fe30b4627a1c0b02e9aead5858edee37aba1e61 Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Wed, 28 Aug 2024 12:51:47 -0400 Subject: [PATCH 095/145] fix: perf test relative import from state-transition (#7055) test: make import relative and add eslint-disable --- .../test/perf/chain/opPools/aggregatedAttestationPool.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index ec66f6dd8905..63fc4ee2e12c 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -12,7 +12,8 @@ import {ExecutionStatus, ForkChoice, IForkChoiceStore, ProtoArray, DataAvailabil import {ssz} from "@lodestar/types"; import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; -import {generatePerfTestCachedStateAltair} from "@lodestar/state-transition/test/perf/util.js"; +// eslint-disable-next-line import/no-relative-packages +import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; import {computeAnchorCheckpoint} from "../../../../src/chain/initState.js"; From 03f7396af7ffcac67080c6544df5324f9c8411a4 Mon Sep 17 00:00:00 2001 From: g11tech Date: Thu, 29 Aug 2024 00:12:08 +0530 Subject: [PATCH 096/145] fix: use next epoch for pending balance/consolidations processing (#7053) --- packages/beacon-node/test/spec/specTestVersioning.ts | 2 +- packages/beacon-node/test/spec/utils/specTestIterator.ts | 3 ++- .../src/epoch/processPendingBalanceDeposits.ts | 5 ++--- .../src/epoch/processPendingConsolidations.ts | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index e97770e8146c..7229a0236d84 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.5.0-alpha.3", + specVersion: "v1.5.0-alpha.5", // Target directory is the host package root: 'packages/*/spec-tests' outputDir: path.join(__dirname, "../../spec-tests"), specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests", diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index ac0d70307d33..48a002580043 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -65,9 +65,10 @@ export const defaultSkipOpts: SkipOpts = { skippedTestSuites: [ /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, + /^electra\/light_client\/.*/, ], // TODO Electra: Review this test in the next spec test release - skippedTests: [/incorrect_not_enough_consolidation_churn_available/], + skippedTests: [/^deneb\/light_client\/sync\/.*electra_fork.*/], skippedRunners: ["merkle_proof", "networking"], }; diff --git a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts index 81da34349e53..bef3ec0b2724 100644 --- a/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts +++ b/packages/state-transition/src/epoch/processPendingBalanceDeposits.ts @@ -2,7 +2,6 @@ import {FAR_FUTURE_EPOCH} from "@lodestar/params"; import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js"; import {increaseBalance} from "../util/balance.js"; import {getActivationExitChurnLimit} from "../util/validator.js"; -import {getCurrentEpoch} from "../util/epoch.js"; /** * Starting from Electra: @@ -14,8 +13,8 @@ import {getCurrentEpoch} from "../util/epoch.js"; * TODO Electra: Update ssz library to support batch push to `pendingBalanceDeposits` */ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra, cache: EpochTransitionCache): void { + const nextEpoch = state.epochCtx.epoch + 1; const availableForProcessing = state.depositBalanceToConsume + BigInt(getActivationExitChurnLimit(state.epochCtx)); - const currentEpoch = getCurrentEpoch(state); let processedAmount = 0n; let nextDepositIndex = 0; const depositsToPostpone = []; @@ -28,7 +27,7 @@ export function processPendingBalanceDeposits(state: CachedBeaconStateElectra, c // Validator is exiting, postpone the deposit until after withdrawable epoch if (validator.exitEpoch < FAR_FUTURE_EPOCH) { - if (currentEpoch <= validator.withdrawableEpoch) { + if (nextEpoch <= validator.withdrawableEpoch) { depositsToPostpone.push(deposit); } else { // Deposited balance will never become active. Increase balance but do not consume churn diff --git a/packages/state-transition/src/epoch/processPendingConsolidations.ts b/packages/state-transition/src/epoch/processPendingConsolidations.ts index 7d8548046e79..28178a509bba 100644 --- a/packages/state-transition/src/epoch/processPendingConsolidations.ts +++ b/packages/state-transition/src/epoch/processPendingConsolidations.ts @@ -18,6 +18,7 @@ import {switchToCompoundingValidator} from "../util/electra.js"; * */ export function processPendingConsolidations(state: CachedBeaconStateElectra, cache: EpochTransitionCache): void { + const nextEpoch = state.epochCtx.epoch + 1; let nextPendingConsolidation = 0; const validators = state.validators; const cachedBalances = cache.balances; @@ -32,7 +33,7 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca continue; } - if (sourceValidator.withdrawableEpoch > state.epochCtx.epoch) { + if (sourceValidator.withdrawableEpoch > nextEpoch) { break; } // Churn any target excess active balance of target and raise its max From 21afb72525457b1ed8838f94da5ad03e94e4b6be Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 30 Aug 2024 08:29:08 +0100 Subject: [PATCH 097/145] chore: relax node version restriction in package.json (#7062) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 06fac0de8891..85d9662ab920 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": ">=20.1.0 <21 || >=22 <22.5" + "node": ">=20.1.0 <21 || >=22 <23" }, "packageManager": "yarn@1.22.22+sha256.c17d3797fb9a9115bf375e31bfd30058cac6bc9c3b8807a3d8cb2094794b51ca", "workspaces": [ From 19ac678d4be3b08152ad7ef9e030fbdd7692ff83 Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 30 Aug 2024 19:22:29 +0700 Subject: [PATCH 098/145] feat: implement isomorphic utils for nodejs and browser (#7060) * feat: implement isomorphic utils for nodes and browser * fix: avoid async import * chore: revise toHexString() comment as in PR review Co-authored-by: Nico Flaig --------- Co-authored-by: Nico Flaig --- packages/utils/src/bytes.ts | 33 ++----------- packages/utils/src/bytes/browser.ts | 66 ++++++++++++++++++++++++++ packages/utils/src/bytes/index.ts | 14 ++++++ packages/utils/src/bytes/nodejs.ts | 33 +++++++++++++ packages/utils/src/index.ts | 1 + packages/utils/test/perf/bytes.test.ts | 38 +++++++++++++-- packages/utils/test/unit/bytes.test.ts | 23 ++++++++- 7 files changed, 173 insertions(+), 35 deletions(-) create mode 100644 packages/utils/src/bytes/browser.ts create mode 100644 packages/utils/src/bytes/index.ts create mode 100644 packages/utils/src/bytes/nodejs.ts diff --git a/packages/utils/src/bytes.ts b/packages/utils/src/bytes.ts index 820cf7d63300..6670b115ff8f 100644 --- a/packages/utils/src/bytes.ts +++ b/packages/utils/src/bytes.ts @@ -3,6 +3,9 @@ import {toBufferLE, toBigIntLE, toBufferBE, toBigIntBE} from "bigint-buffer"; type Endianness = "le" | "be"; const hexByByte: string[] = []; +/** + * @deprecated Use toHex() instead. + */ export function toHexString(bytes: Uint8Array): string { let hex = "0x"; for (const byte of bytes) { @@ -45,33 +48,3 @@ export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): } throw new Error("endianness must be either 'le' or 'be'"); } - -export function toHex(buffer: Uint8Array | Parameters[0]): string { - if (Buffer.isBuffer(buffer)) { - return "0x" + buffer.toString("hex"); - } else if (buffer instanceof Uint8Array) { - return "0x" + Buffer.from(buffer.buffer, buffer.byteOffset, buffer.length).toString("hex"); - } else { - return "0x" + Buffer.from(buffer).toString("hex"); - } -} - -// Shared buffer to convert root to hex -const rootBuf = Buffer.alloc(32); - -/** - * Convert a Uint8Array, length 32, to 0x-prefixed hex string - */ -export function toRootHex(root: Uint8Array): string { - if (root.length !== 32) { - throw Error(`Expect root to be 32 bytes, got ${root.length}`); - } - - rootBuf.set(root); - return `0x${rootBuf.toString("hex")}`; -} - -export function fromHex(hex: string): Uint8Array { - const b = Buffer.from(hex.replace("0x", ""), "hex"); - return new Uint8Array(b.buffer, b.byteOffset, b.length); -} diff --git a/packages/utils/src/bytes/browser.ts b/packages/utils/src/bytes/browser.ts new file mode 100644 index 000000000000..e6c04f79835f --- /dev/null +++ b/packages/utils/src/bytes/browser.ts @@ -0,0 +1,66 @@ +export function toHex(bytes: Uint8Array): string { + const charCodes = new Array(bytes.length * 2 + 2); + charCodes[0] = 48; + charCodes[1] = 120; + + for (let i = 0; i < bytes.length; i++) { + const byte = bytes[i]; + const first = (byte & 0xf0) >> 4; + const second = byte & 0x0f; + + // "0".charCodeAt(0) = 48 + // "a".charCodeAt(0) = 97 => delta = 87 + charCodes[2 + 2 * i] = first < 10 ? first + 48 : first + 87; + charCodes[2 + 2 * i + 1] = second < 10 ? second + 48 : second + 87; + } + return String.fromCharCode(...charCodes); +} + +const rootCharCodes = new Array(32 * 2 + 2); +// "0".charCodeAt(0) +rootCharCodes[0] = 48; +// "x".charCodeAt(0) +rootCharCodes[1] = 120; + +/** + * Convert a Uint8Array, length 32, to 0x-prefixed hex string + */ +export function toRootHex(root: Uint8Array): string { + if (root.length !== 32) { + throw Error(`Expect root to be 32 bytes, got ${root.length}`); + } + + for (let i = 0; i < root.length; i++) { + const byte = root[i]; + const first = (byte & 0xf0) >> 4; + const second = byte & 0x0f; + + // "0".charCodeAt(0) = 48 + // "a".charCodeAt(0) = 97 => delta = 87 + rootCharCodes[2 + 2 * i] = first < 10 ? first + 48 : first + 87; + rootCharCodes[2 + 2 * i + 1] = second < 10 ? second + 48 : second + 87; + } + return String.fromCharCode(...rootCharCodes); +} + +export function fromHex(hex: string): Uint8Array { + if (typeof hex !== "string") { + throw new Error(`hex argument type ${typeof hex} must be of type string`); + } + + if (hex.startsWith("0x")) { + hex = hex.slice(2); + } + + if (hex.length % 2 !== 0) { + throw new Error(`hex string length ${hex.length} must be multiple of 2`); + } + + const byteLen = hex.length / 2; + const bytes = new Uint8Array(byteLen); + for (let i = 0; i < byteLen; i++) { + const byte = parseInt(hex.slice(i * 2, (i + 1) * 2), 16); + bytes[i] = byte; + } + return bytes; +} diff --git a/packages/utils/src/bytes/index.ts b/packages/utils/src/bytes/index.ts new file mode 100644 index 000000000000..fe6a9fc40e54 --- /dev/null +++ b/packages/utils/src/bytes/index.ts @@ -0,0 +1,14 @@ +import {toHex as browserToHex, toRootHex as browserToRootHex, fromHex as browserFromHex} from "./browser.js"; +import {toHex as nodeToHex, toRootHex as nodeToRootHex, fromHex as nodeFromHex} from "./nodejs.js"; + +let toHex = browserToHex; +let toRootHex = browserToRootHex; +let fromHex = browserFromHex; + +if (typeof Buffer !== "undefined") { + toHex = nodeToHex; + toRootHex = nodeToRootHex; + fromHex = nodeFromHex; +} + +export {toHex, toRootHex, fromHex}; diff --git a/packages/utils/src/bytes/nodejs.ts b/packages/utils/src/bytes/nodejs.ts new file mode 100644 index 000000000000..7f0fe50d2a4a --- /dev/null +++ b/packages/utils/src/bytes/nodejs.ts @@ -0,0 +1,33 @@ +export function toHex(buffer: Uint8Array | Parameters[0]): string { + if (Buffer.isBuffer(buffer)) { + return "0x" + buffer.toString("hex"); + } else if (buffer instanceof Uint8Array) { + return "0x" + Buffer.from(buffer.buffer, buffer.byteOffset, buffer.length).toString("hex"); + } else { + return "0x" + Buffer.from(buffer).toString("hex"); + } +} + +// Shared buffer to convert root to hex +let rootBuf: Buffer | undefined; + +/** + * Convert a Uint8Array, length 32, to 0x-prefixed hex string + */ +export function toRootHex(root: Uint8Array): string { + if (root.length !== 32) { + throw Error(`Expect root to be 32 bytes, got ${root.length}`); + } + + if (rootBuf === undefined) { + rootBuf = Buffer.alloc(32); + } + + rootBuf.set(root); + return `0x${rootBuf.toString("hex")}`; +} + +export function fromHex(hex: string): Uint8Array { + const b = Buffer.from(hex.replace("0x", ""), "hex"); + return new Uint8Array(b.buffer, b.byteOffset, b.length); +} diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 2057e50e07bc..13ea1ffb7e69 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -2,6 +2,7 @@ export * from "./yaml/index.js"; export * from "./assert.js"; export * from "./base64.js"; export * from "./bytes.js"; +export * from "./bytes/index.js"; export * from "./command.js"; export * from "./err.js"; export * from "./errors.js"; diff --git a/packages/utils/test/perf/bytes.test.ts b/packages/utils/test/perf/bytes.test.ts index 368758d12efe..649ea0b51e3e 100644 --- a/packages/utils/test/perf/bytes.test.ts +++ b/packages/utils/test/perf/bytes.test.ts @@ -1,12 +1,14 @@ import {itBench} from "@dapplion/benchmark"; -import {toHex, toRootHex} from "../../src/bytes.js"; +import {toHex, toRootHex} from "../../src/bytes/nodejs.js"; +import {toHex as browserToHex, toRootHex as browserToRootHex} from "../../src/bytes/browser.js"; +import {toHexString} from "../../src/bytes.js"; describe("bytes utils", function () { const runsFactor = 1000; const blockRoot = new Uint8Array(Array.from({length: 32}, (_, i) => i)); itBench({ - id: "block root to RootHex using toHex", + id: "nodejs block root to RootHex using toHex", fn: () => { for (let i = 0; i < runsFactor; i++) { toHex(blockRoot); @@ -16,7 +18,7 @@ describe("bytes utils", function () { }); itBench({ - id: "block root to RootHex using toRootHex", + id: "nodejs block root to RootHex using toRootHex", fn: () => { for (let i = 0; i < runsFactor; i++) { toRootHex(blockRoot); @@ -24,4 +26,34 @@ describe("bytes utils", function () { }, runsFactor, }); + + itBench({ + id: "browser block root to RootHex using the deprecated toHexString", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + toHexString(blockRoot); + } + }, + runsFactor, + }); + + itBench({ + id: "browser block root to RootHex using toHex", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + browserToHex(blockRoot); + } + }, + runsFactor, + }); + + itBench({ + id: "browser block root to RootHex using toRootHex", + fn: () => { + for (let i = 0; i < runsFactor; i++) { + browserToRootHex(blockRoot); + } + }, + runsFactor, + }); }); diff --git a/packages/utils/test/unit/bytes.test.ts b/packages/utils/test/unit/bytes.test.ts index 8410e667187a..aefa3e240954 100644 --- a/packages/utils/test/unit/bytes.test.ts +++ b/packages/utils/test/unit/bytes.test.ts @@ -1,5 +1,5 @@ import {describe, it, expect} from "vitest"; -import {intToBytes, bytesToInt, toHex, fromHex, toHexString} from "../../src/index.js"; +import {intToBytes, bytesToInt, toHex, fromHex, toHexString, toRootHex} from "../../src/index.js"; describe("intToBytes", () => { const zeroedArray = (length: number): number[] => Array.from({length}, () => 0); @@ -48,7 +48,7 @@ describe("bytesToInt", () => { }); describe("toHex", () => { - const testCases: {input: Buffer | Uint8Array | string; output: string}[] = [ + const testCases: {input: Uint8Array; output: string}[] = [ {input: Buffer.from("Hello, World!", "utf-8"), output: "0x48656c6c6f2c20576f726c6421"}, {input: new Uint8Array([72, 101, 108, 108, 111]), output: "0x48656c6c6f"}, {input: Buffer.from([72, 101, 108, 108, 111]), output: "0x48656c6c6f"}, @@ -61,6 +61,25 @@ describe("toHex", () => { } }); +describe("toRootHex", () => { + const testCases: {input: Uint8Array; output: string}[] = [ + { + input: new Uint8Array(Array.from({length: 32}, (_, i) => i)), + output: "0x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + }, + { + input: new Uint8Array(Array.from({length: 32}, () => 0)), + output: "0x0000000000000000000000000000000000000000000000000000000000000000", + }, + ]; + + for (const {input, output} of testCases) { + it(`should convert root to hex string ${output}`, () => { + expect(toRootHex(input)).toBe(output); + }); + } +}); + describe("fromHex", () => { const testCases: {input: string; output: Buffer | Uint8Array}[] = [ { From ef99ed711437f8b86941fed6eb8b92010c91df3a Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 4 Sep 2024 08:20:35 +0700 Subject: [PATCH 099/145] fix: toPubkeyHex (#7065) * feat: add toPubkeyHex util * feat: consume toPubkeyHex * fix: use toPubkeyHex instead of toHex * chore: remove redundant comments --- packages/api/src/builder/routes.ts | 3 +- .../validator/keymanager/keystoreCache.ts | 6 +- .../validator/slashingProtection/export.ts | 5 +- .../cli/src/cmds/validator/voluntaryExit.ts | 4 +- packages/flare/src/cmds/selfSlashAttester.ts | 4 +- packages/flare/src/cmds/selfSlashProposer.ts | 4 +- .../src/cache/syncCommitteeCache.ts | 5 +- packages/utils/src/bytes/browser.ts | 71 ++++++++++++------- packages/utils/src/bytes/index.ts | 18 ++++- packages/utils/src/bytes/nodejs.ts | 16 +++++ packages/utils/test/unit/bytes.test.ts | 21 +++++- .../src/services/attestationDuties.ts | 7 +- packages/validator/src/services/block.ts | 5 +- .../validator/src/services/blockDuties.ts | 9 ++- packages/validator/src/services/indices.ts | 5 +- .../src/services/syncCommitteeDuties.ts | 4 +- .../validator/src/services/validatorStore.ts | 9 +-- .../validator/src/slashingProtection/index.ts | 7 +- .../interchange/formats/completeV4.ts | 3 +- .../interchange/formats/v5.ts | 3 +- 20 files changed, 138 insertions(+), 71 deletions(-) diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts index 297c4fc5e9a9..3d74101bb046 100644 --- a/packages/api/src/builder/routes.ts +++ b/packages/api/src/builder/routes.ts @@ -13,6 +13,7 @@ import { } from "@lodestar/types"; import {ForkName, isForkBlobs} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; +import {toPubkeyHex} from "@lodestar/utils"; import {Endpoint, RouteDefinitions, Schema} from "../utils/index.js"; import {MetaHeader, VersionCodec, VersionMeta} from "../utils/metadata.js"; @@ -105,7 +106,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - params: {slot, parent_hash: toHexString(parentHash), pubkey: toHexString(proposerPubKey)}, + params: {slot, parent_hash: toHexString(parentHash), pubkey: toPubkeyHex(proposerPubKey)}, }), parseReq: ({params}) => ({ slot: params.slot, diff --git a/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts index 85b1702892ee..2997a6b6b113 100644 --- a/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts +++ b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts @@ -3,7 +3,7 @@ import path from "node:path"; import {Keystore} from "@chainsafe/bls-keystore"; import {SecretKey} from "@chainsafe/blst"; import {SignerLocal, SignerType} from "@lodestar/validator"; -import {fromHex, toHex} from "@lodestar/utils"; +import {fromHex, toHex, toPubkeyHex} from "@lodestar/utils"; import {writeFile600Perm} from "../../../util/file.js"; import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js"; import {LocalKeystoreDefinition} from "./interface.js"; @@ -42,9 +42,9 @@ export async function loadKeystoreCache( const secretKey = SecretKey.fromBytes(secretKeyBytes); const publicKey = secretKey.toPublicKey().toBytes(); - if (toHex(publicKey) !== toHex(fromHex(k.pubkey))) { + if (toPubkeyHex(publicKey) !== toPubkeyHex(fromHex(k.pubkey))) { throw new Error( - `Keystore ${k.uuid} does not match the expected pubkey. expected=${toHex(fromHex(k.pubkey))}, found=${toHex( + `Keystore ${k.uuid} does not match the expected pubkey. expected=${toPubkeyHex(fromHex(k.pubkey))}, found=${toHex( publicKey )}` ); diff --git a/packages/cli/src/cmds/validator/slashingProtection/export.ts b/packages/cli/src/cmds/validator/slashingProtection/export.ts index 7d1a4f8e6e2f..c18b020f5782 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/export.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/export.ts @@ -1,8 +1,7 @@ import path from "node:path"; -import {toHexString} from "@chainsafe/ssz"; import {InterchangeFormatVersion} from "@lodestar/validator"; import {getNodeLogger} from "@lodestar/logger/node"; -import {CliCommand} from "@lodestar/utils"; +import {CliCommand, toPubkeyHex} from "@lodestar/utils"; import {YargsError, ensure0xPrefix, isValidatePubkeyHex, writeFile600Perm} from "../../../util/index.js"; import {parseLoggerArgs} from "../../../util/logger.js"; import {GlobalArgs} from "../../../options/index.js"; @@ -86,7 +85,7 @@ export const exportCmd: CliCommand toHexString(pubkey) === pubkeyHex); + const existingPubkey = allPubkeys.find((pubkey) => toPubkeyHex(pubkey) === pubkeyHex); if (!existingPubkey) { logger.warn("Pubkey not found in slashing protection db", {pubkey: pubkeyHex}); } else { diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index 279076b619f2..a399a763662d 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -8,7 +8,7 @@ import { } from "@lodestar/state-transition"; import {createBeaconConfig, BeaconConfig} from "@lodestar/config"; import {phase0, ssz, ValidatorIndex, Epoch} from "@lodestar/types"; -import {CliCommand, fromHex, toHex} from "@lodestar/utils"; +import {CliCommand, fromHex, toPubkeyHex} from "@lodestar/utils"; import {externalSignerPostSignature, SignableMessageType, Signer, SignerType} from "@lodestar/validator"; import {ApiClient, getClient} from "@lodestar/api"; import {ensure0xPrefix, YargsError, wrapError} from "../../util/index.js"; @@ -209,7 +209,7 @@ async function resolveValidatorIndexes(client: ApiClient, signersToExit: SignerP const validators = (await client.beacon.getStateValidators({stateId: "head", validatorIds: pubkeys})).value(); - const dataByPubkey = new Map(validators.map((item) => [toHex(item.validator.pubkey), item])); + const dataByPubkey = new Map(validators.map((item) => [toPubkeyHex(item.validator.pubkey), item])); return signersToExit.map(({signer, pubkey}) => { const item = dataByPubkey.get(pubkey); diff --git a/packages/flare/src/cmds/selfSlashAttester.ts b/packages/flare/src/cmds/selfSlashAttester.ts index e29a956a9306..a37c6c765bd3 100644 --- a/packages/flare/src/cmds/selfSlashAttester.ts +++ b/packages/flare/src/cmds/selfSlashAttester.ts @@ -4,7 +4,7 @@ import {AttesterSlashing, phase0, ssz} from "@lodestar/types"; import {config as chainConfig} from "@lodestar/config/default"; import {createBeaconConfig, BeaconConfig} from "@lodestar/config"; import {DOMAIN_BEACON_ATTESTER, MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; -import {CliCommand, toHexString} from "@lodestar/utils"; +import {CliCommand, toPubkeyHex} from "@lodestar/utils"; import {computeSigningRoot} from "@lodestar/state-transition"; import {deriveSecretKeys, SecretKeysArgs, secretKeysOptions} from "../util/deriveSecretKeys.js"; @@ -90,7 +90,7 @@ export async function selfSlashAttesterHandler(args: SelfSlashArgs): Promise; @@ -82,7 +83,7 @@ function computeSyncCommitteeIndices( for (const pubkey of pubkeys) { const validatorIndex = pubkey2index.get(pubkey); if (validatorIndex === undefined) { - throw Error(`SyncCommittee pubkey is unknown ${toHexString(pubkey)}`); + throw Error(`SyncCommittee pubkey is unknown ${toPubkeyHex(pubkey)}`); } validatorIndices.push(validatorIndex); diff --git a/packages/utils/src/bytes/browser.ts b/packages/utils/src/bytes/browser.ts index e6c04f79835f..f610e2912c04 100644 --- a/packages/utils/src/bytes/browser.ts +++ b/packages/utils/src/bytes/browser.ts @@ -1,26 +1,20 @@ +// "0".charCodeAt(0) = 48 +const CHAR_CODE_0 = 48; +// "x".charCodeAt(0) = 120 +const CHAR_CODE_X = 120; + export function toHex(bytes: Uint8Array): string { const charCodes = new Array(bytes.length * 2 + 2); - charCodes[0] = 48; - charCodes[1] = 120; - - for (let i = 0; i < bytes.length; i++) { - const byte = bytes[i]; - const first = (byte & 0xf0) >> 4; - const second = byte & 0x0f; + charCodes[0] = CHAR_CODE_0; + charCodes[1] = CHAR_CODE_X; - // "0".charCodeAt(0) = 48 - // "a".charCodeAt(0) = 97 => delta = 87 - charCodes[2 + 2 * i] = first < 10 ? first + 48 : first + 87; - charCodes[2 + 2 * i + 1] = second < 10 ? second + 48 : second + 87; - } + bytesIntoCharCodes(bytes, charCodes); return String.fromCharCode(...charCodes); } const rootCharCodes = new Array(32 * 2 + 2); -// "0".charCodeAt(0) -rootCharCodes[0] = 48; -// "x".charCodeAt(0) -rootCharCodes[1] = 120; +rootCharCodes[0] = CHAR_CODE_0; +rootCharCodes[1] = CHAR_CODE_X; /** * Convert a Uint8Array, length 32, to 0x-prefixed hex string @@ -30,17 +24,24 @@ export function toRootHex(root: Uint8Array): string { throw Error(`Expect root to be 32 bytes, got ${root.length}`); } - for (let i = 0; i < root.length; i++) { - const byte = root[i]; - const first = (byte & 0xf0) >> 4; - const second = byte & 0x0f; + bytesIntoCharCodes(root, rootCharCodes); + return String.fromCharCode(...rootCharCodes); +} - // "0".charCodeAt(0) = 48 - // "a".charCodeAt(0) = 97 => delta = 87 - rootCharCodes[2 + 2 * i] = first < 10 ? first + 48 : first + 87; - rootCharCodes[2 + 2 * i + 1] = second < 10 ? second + 48 : second + 87; +const pubkeyCharCodes = new Array(48 * 2 + 2); +pubkeyCharCodes[0] = CHAR_CODE_0; +pubkeyCharCodes[1] = CHAR_CODE_X; + +/** + * Convert a Uint8Array, length 48, to 0x-prefixed hex string + */ +export function toPubkeyHex(pubkey: Uint8Array): string { + if (pubkey.length !== CHAR_CODE_0) { + throw Error(`Expect pubkey to be 48 bytes, got ${pubkey.length}`); } - return String.fromCharCode(...rootCharCodes); + + bytesIntoCharCodes(pubkey, pubkeyCharCodes); + return String.fromCharCode(...pubkeyCharCodes); } export function fromHex(hex: string): Uint8Array { @@ -64,3 +65,23 @@ export function fromHex(hex: string): Uint8Array { } return bytes; } + +/** + * Populate charCodes from bytes. Note that charCodes index 0 and 1 ("0x") are not populated. + */ +function bytesIntoCharCodes(bytes: Uint8Array, charCodes: number[]): void { + if (bytes.length * 2 + 2 !== charCodes.length) { + throw Error(`Expect charCodes to be of length ${bytes.length * 2 + 2}, got ${charCodes.length}`); + } + + for (let i = 0; i < bytes.length; i++) { + const byte = bytes[i]; + const first = (byte & 0xf0) >> 4; + const second = byte & 0x0f; + + // "0".charCodeAt(0) = 48 + // "a".charCodeAt(0) = 97 => delta = 87 + charCodes[2 + 2 * i] = first < 10 ? first + 48 : first + 87; + charCodes[2 + 2 * i + 1] = second < 10 ? second + 48 : second + 87; + } +} diff --git a/packages/utils/src/bytes/index.ts b/packages/utils/src/bytes/index.ts index fe6a9fc40e54..a079764738b7 100644 --- a/packages/utils/src/bytes/index.ts +++ b/packages/utils/src/bytes/index.ts @@ -1,14 +1,26 @@ -import {toHex as browserToHex, toRootHex as browserToRootHex, fromHex as browserFromHex} from "./browser.js"; -import {toHex as nodeToHex, toRootHex as nodeToRootHex, fromHex as nodeFromHex} from "./nodejs.js"; +import { + toHex as browserToHex, + toRootHex as browserToRootHex, + fromHex as browserFromHex, + toPubkeyHex as browserToPubkeyHex, +} from "./browser.js"; +import { + toHex as nodeToHex, + toRootHex as nodeToRootHex, + fromHex as nodeFromHex, + toPubkeyHex as nodeToPubkeyHex, +} from "./nodejs.js"; let toHex = browserToHex; let toRootHex = browserToRootHex; +let toPubkeyHex = browserToPubkeyHex; let fromHex = browserFromHex; if (typeof Buffer !== "undefined") { toHex = nodeToHex; toRootHex = nodeToRootHex; + toPubkeyHex = nodeToPubkeyHex; fromHex = nodeFromHex; } -export {toHex, toRootHex, fromHex}; +export {toHex, toRootHex, toPubkeyHex, fromHex}; diff --git a/packages/utils/src/bytes/nodejs.ts b/packages/utils/src/bytes/nodejs.ts index 7f0fe50d2a4a..636f49bd8e76 100644 --- a/packages/utils/src/bytes/nodejs.ts +++ b/packages/utils/src/bytes/nodejs.ts @@ -27,6 +27,22 @@ export function toRootHex(root: Uint8Array): string { return `0x${rootBuf.toString("hex")}`; } +// Shared buffer to convert pubkey to hex +let pubkeyBuf: Buffer | undefined; + +export function toPubkeyHex(pubkey: Uint8Array): string { + if (pubkey.length !== 48) { + throw Error(`Expect pubkey to be 48 bytes, got ${pubkey.length}`); + } + + if (pubkeyBuf === undefined) { + pubkeyBuf = Buffer.alloc(48); + } + + pubkeyBuf.set(pubkey); + return `0x${pubkeyBuf.toString("hex")}`; +} + export function fromHex(hex: string): Uint8Array { const b = Buffer.from(hex.replace("0x", ""), "hex"); return new Uint8Array(b.buffer, b.byteOffset, b.length); diff --git a/packages/utils/test/unit/bytes.test.ts b/packages/utils/test/unit/bytes.test.ts index aefa3e240954..05789b839cfd 100644 --- a/packages/utils/test/unit/bytes.test.ts +++ b/packages/utils/test/unit/bytes.test.ts @@ -1,5 +1,5 @@ import {describe, it, expect} from "vitest"; -import {intToBytes, bytesToInt, toHex, fromHex, toHexString, toRootHex} from "../../src/index.js"; +import {intToBytes, bytesToInt, toHex, fromHex, toHexString, toRootHex, toPubkeyHex} from "../../src/index.js"; describe("intToBytes", () => { const zeroedArray = (length: number): number[] => Array.from({length}, () => 0); @@ -80,6 +80,25 @@ describe("toRootHex", () => { } }); +describe("toPubkeyHex", () => { + const testCases: {input: Uint8Array; output: string}[] = [ + { + input: new Uint8Array(Array.from({length: 48}, (_, i) => i)), + output: "0x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f", + }, + { + input: new Uint8Array(Array.from({length: 48}, () => 0)), + output: "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + }, + ]; + + for (const {input, output} of testCases) { + it(`should convert root to hex string ${output}`, () => { + expect(toPubkeyHex(input)).toBe(output); + }); + } +}); + describe("fromHex", () => { const testCases: {input: string; output: Buffer | Uint8Array}[] = [ { diff --git a/packages/validator/src/services/attestationDuties.ts b/packages/validator/src/services/attestationDuties.ts index ea82bf0a4c72..83838afe1492 100644 --- a/packages/validator/src/services/attestationDuties.ts +++ b/packages/validator/src/services/attestationDuties.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {sleep} from "@lodestar/utils"; +import {sleep, toPubkeyHex} from "@lodestar/utils"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength, isStartSlotOfEpoch} from "@lodestar/state-transition"; import {BLSSignature, Epoch, Slot, ValidatorIndex, RootHex} from "@lodestar/types"; import {ApiClient, routes} from "@lodestar/api"; @@ -97,7 +96,7 @@ export class AttestationDutiesService { removeDutiesForKey(pubkey: PubkeyHex): void { for (const [epoch, attDutiesAtEpoch] of this.dutiesByIndexByEpoch) { for (const [vIndex, attDutyAndProof] of attDutiesAtEpoch.dutiesByIndex) { - if (toHexString(attDutyAndProof.duty.pubkey) === pubkey) { + if (toPubkeyHex(attDutyAndProof.duty.pubkey) === pubkey) { attDutiesAtEpoch.dutiesByIndex.delete(vIndex); if (attDutiesAtEpoch.dutiesByIndex.size === 0) { this.dutiesByIndexByEpoch.delete(epoch); @@ -244,7 +243,7 @@ export class AttestationDutiesService { const attesterDuties = res.value(); const {dependentRoot} = res.meta(); const relevantDuties = attesterDuties.filter((duty) => { - const pubkeyHex = toHexString(duty.pubkey); + const pubkeyHex = toPubkeyHex(duty.pubkey); return this.validatorStore.hasVotingPubkey(pubkeyHex) && this.validatorStore.isDoppelgangerSafe(pubkeyHex); }); diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index 7792ef85be95..c3acf19c1669 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import { BLSPubkey, Slot, @@ -15,7 +14,7 @@ import { } from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {ForkPreBlobs, ForkBlobs, ForkSeq, ForkExecution, ForkName} from "@lodestar/params"; -import {extendError, prettyBytes, prettyWeiToEth} from "@lodestar/utils"; +import {extendError, prettyBytes, prettyWeiToEth, toPubkeyHex} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; import {IClock, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; @@ -110,7 +109,7 @@ export class BlockProposingService { /** Produce a block at the given slot for pubkey */ private async createAndPublishBlock(pubkey: BLSPubkey, slot: Slot): Promise { - const pubkeyHex = toHexString(pubkey); + const pubkeyHex = toPubkeyHex(pubkey); const logCtx = {slot, validator: prettyBytes(pubkeyHex)}; // Wrap with try catch here to re-use `logCtx` diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index 3282987f5d9e..d0e16f60e816 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -1,8 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {BLSPubkey, Epoch, RootHex, Slot} from "@lodestar/types"; import {ApiClient, routes} from "@lodestar/api"; -import {sleep} from "@lodestar/utils"; +import {sleep, toPubkeyHex} from "@lodestar/utils"; import {ChainConfig} from "@lodestar/config"; import {IClock, differenceHex, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; @@ -67,7 +66,7 @@ export class BlockDutiesService { if (dutyAtEpoch) { for (const proposer of dutyAtEpoch.data) { if (proposer.slot === slot) { - publicKeys.set(toHexString(proposer.pubkey), proposer.pubkey); + publicKeys.set(toPubkeyHex(proposer.pubkey), proposer.pubkey); } } } @@ -78,7 +77,7 @@ export class BlockDutiesService { removeDutiesForKey(pubkey: PubkeyHex): void { for (const blockDutyAtEpoch of this.proposers.values()) { blockDutyAtEpoch.data = blockDutyAtEpoch.data.filter((proposer) => { - return toHexString(proposer.pubkey) !== pubkey; + return toPubkeyHex(proposer.pubkey) !== pubkey; }); } } @@ -187,7 +186,7 @@ export class BlockDutiesService { const proposerDuties = res.value(); const {dependentRoot} = res.meta(); const relevantDuties = proposerDuties.filter((duty) => { - const pubkeyHex = toHexString(duty.pubkey); + const pubkeyHex = toPubkeyHex(duty.pubkey); return this.validatorStore.hasVotingPubkey(pubkeyHex) && this.validatorStore.isDoppelgangerSafe(pubkeyHex); }); diff --git a/packages/validator/src/services/indices.ts b/packages/validator/src/services/indices.ts index c6ef40b473e5..ec5155322918 100644 --- a/packages/validator/src/services/indices.ts +++ b/packages/validator/src/services/indices.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {ValidatorIndex} from "@lodestar/types"; -import {Logger, MapDef} from "@lodestar/utils"; +import {Logger, MapDef, toPubkeyHex} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; import {batchItems} from "../util/index.js"; import {Metrics} from "../metrics.js"; @@ -135,7 +134,7 @@ export class IndicesService { const status = statusToSimpleStatusMapping(validator.status); allValidatorStatuses.set(status, allValidatorStatuses.getOrDefault(status) + 1); - const pubkeyHex = toHexString(validator.validator.pubkey); + const pubkeyHex = toPubkeyHex(validator.validator.pubkey); if (!this.pubkey2index.has(pubkeyHex)) { this.logger.info("Validator seen on beacon chain", { validatorIndex: validator.index, diff --git a/packages/validator/src/services/syncCommitteeDuties.ts b/packages/validator/src/services/syncCommitteeDuties.ts index dd663528f751..ea448add15ec 100644 --- a/packages/validator/src/services/syncCommitteeDuties.ts +++ b/packages/validator/src/services/syncCommitteeDuties.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; import { computeEpochAtSlot, @@ -10,6 +9,7 @@ import { import {ChainForkConfig} from "@lodestar/config"; import {BLSSignature, Epoch, Slot, SyncPeriod, ValidatorIndex} from "@lodestar/types"; import {ApiClient, routes} from "@lodestar/api"; +import {toPubkeyHex} from "@lodestar/utils"; import {IClock, LoggerVc} from "../util/index.js"; import {PubkeyHex} from "../types.js"; import {Metrics} from "../metrics.js"; @@ -287,7 +287,7 @@ export class SyncCommitteeDutiesService { // Using `alreadyWarnedReorg` avoids excessive logs. // TODO: Use memory-efficient toHexString() - const pubkeyHex = toHexString(duty.pubkey); + const pubkeyHex = toPubkeyHex(duty.pubkey); dutiesByIndex.set(validatorIndex, {duty: {pubkey: pubkeyHex, validatorIndex, subnets}}); } diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index fa9d855aa24a..c6130f1fab95 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -42,6 +42,7 @@ import { SignedAggregateAndProof, } from "@lodestar/types"; import {routes} from "@lodestar/api"; +import {toPubkeyHex} from "@lodestar/utils"; import {ISlashingProtection} from "../slashingProtection/index.js"; import {PubkeyHex} from "../types.js"; import {externalSignerPostSignature, SignableMessageType, SignableMessage} from "../util/externalSignerClient.js"; @@ -723,7 +724,7 @@ export class ValidatorStore { regAttributes: {feeRecipient: Eth1Address; gasLimit: number}, slot: Slot ): Promise { - const pubkeyHex = typeof pubkeyMaybeHex === "string" ? pubkeyMaybeHex : toHexString(pubkeyMaybeHex); + const pubkeyHex = typeof pubkeyMaybeHex === "string" ? pubkeyMaybeHex : toPubkeyHex(pubkeyMaybeHex); const {feeRecipient, gasLimit} = regAttributes; const regFullKey = `${feeRecipient}-${gasLimit}`; const validatorData = this.validators.get(pubkeyHex); @@ -748,7 +749,7 @@ export class ValidatorStore { signableMessage: SignableMessage ): Promise { // TODO: Refactor indexing to not have to run toHexString() on the pubkey every time - const pubkeyHex = typeof pubkey === "string" ? pubkey : toHexString(pubkey); + const pubkeyHex = typeof pubkey === "string" ? pubkey : toPubkeyHex(pubkey); const signer = this.validators.get(pubkeyHex)?.signer; if (!signer) { @@ -787,7 +788,7 @@ export class ValidatorStore { private getSignerAndPubkeyHex(pubkey: BLSPubkeyMaybeHex): [Signer, string] { // TODO: Refactor indexing to not have to run toHexString() on the pubkey every time - const pubkeyHex = typeof pubkey === "string" ? pubkey : toHexString(pubkey); + const pubkeyHex = typeof pubkey === "string" ? pubkey : toPubkeyHex(pubkey); const signer = this.validators.get(pubkeyHex)?.signer; if (!signer) { throw Error(`Validator pubkey ${pubkeyHex} not known`); @@ -813,7 +814,7 @@ export class ValidatorStore { } private assertDoppelgangerSafe(pubKey: PubkeyHex | BLSPubkey): void { - const pubkeyHex = typeof pubKey === "string" ? pubKey : toHexString(pubKey); + const pubkeyHex = typeof pubKey === "string" ? pubKey : toPubkeyHex(pubKey); if (!this.isDoppelgangerSafe(pubkeyHex)) { throw new Error(`Doppelganger state for key ${pubkeyHex} is not safe`); } diff --git a/packages/validator/src/slashingProtection/index.ts b/packages/validator/src/slashingProtection/index.ts index dedbccf6cf94..bc57b0e51c13 100644 --- a/packages/validator/src/slashingProtection/index.ts +++ b/packages/validator/src/slashingProtection/index.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {BLSPubkey, Epoch, Root} from "@lodestar/types"; -import {Logger} from "@lodestar/utils"; +import {Logger, toPubkeyHex} from "@lodestar/utils"; import {LodestarValidatorDatabaseController} from "../types.js"; import {uniqueVectorArr} from "../slashingProtection/utils.js"; import {BlockBySlotRepository, SlashingProtectionBlockService} from "./block/index.js"; @@ -63,7 +62,7 @@ export class SlashingProtection implements ISlashingProtection { async importInterchange(interchange: Interchange, genesisValidatorsRoot: Root, logger?: Logger): Promise { const {data} = parseInterchange(interchange, genesisValidatorsRoot); for (const validator of data) { - logger?.info("Importing slashing protection", {pubkey: toHexString(validator.pubkey)}); + logger?.info("Importing slashing protection", {pubkey: toPubkeyHex(validator.pubkey)}); await this.blockService.importBlocks(validator.pubkey, validator.signedBlocks); await this.attestationService.importAttestations(validator.pubkey, validator.signedAttestations); } @@ -77,7 +76,7 @@ export class SlashingProtection implements ISlashingProtection { ): Promise { const validatorData: InterchangeLodestar["data"] = []; for (const pubkey of pubkeys) { - logger?.info("Exporting slashing protection", {pubkey: toHexString(pubkey)}); + logger?.info("Exporting slashing protection", {pubkey: toPubkeyHex(pubkey)}); validatorData.push({ pubkey, signedBlocks: await this.blockService.exportBlocks(pubkey), diff --git a/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts b/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts index 26d7f44f2e83..66aa31c52194 100644 --- a/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts +++ b/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {toPubkeyHex} from "@lodestar/utils"; import {InterchangeLodestar} from "../types.js"; import {fromOptionalHexString, numToString, toOptionalHexString} from "../../utils.js"; @@ -93,7 +94,7 @@ export function serializeInterchangeCompleteV4({ genesis_validators_root: toHexString(genesisValidatorsRoot), }, data: data.map((validator) => ({ - pubkey: toHexString(validator.pubkey), + pubkey: toPubkeyHex(validator.pubkey), signed_blocks: validator.signedBlocks.map((block) => ({ slot: numToString(block.slot), signing_root: toOptionalHexString(block.signingRoot), diff --git a/packages/validator/src/slashingProtection/interchange/formats/v5.ts b/packages/validator/src/slashingProtection/interchange/formats/v5.ts index c70dc84b1ed0..1c7f67b706a5 100644 --- a/packages/validator/src/slashingProtection/interchange/formats/v5.ts +++ b/packages/validator/src/slashingProtection/interchange/formats/v5.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {toPubkeyHex} from "@lodestar/utils"; import {InterchangeLodestar} from "../types.js"; import {fromOptionalHexString, numToString, toOptionalHexString} from "../../utils.js"; @@ -88,7 +89,7 @@ export function serializeInterchangeV5({data, genesisValidatorsRoot}: Interchang genesis_validators_root: toHexString(genesisValidatorsRoot), }, data: data.map((validator) => ({ - pubkey: toHexString(validator.pubkey), + pubkey: toPubkeyHex(validator.pubkey), signed_blocks: validator.signedBlocks.map((block) => ({ slot: numToString(block.slot), signing_root: toOptionalHexString(block.signingRoot), From 681bdcd775990d788c8674d6376b18778f6f588f Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 4 Sep 2024 08:21:12 +0700 Subject: [PATCH 100/145] fix: improve performance of getExpectedWithdrawals (#7045) * fix: improve performance of getExpectedWithdrawals * chore: use isPostElectra variable * chore: check pre-capella --- .../src/block/processWithdrawals.ts | 47 +++++++++++++--- packages/state-transition/src/util/electra.ts | 55 +------------------ 2 files changed, 41 insertions(+), 61 deletions(-) diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index b06209167be3..185ddd80eb32 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -7,6 +7,7 @@ import { MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, FAR_FUTURE_EPOCH, MIN_ACTIVATION_BALANCE, + MAX_EFFECTIVE_BALANCE, } from "@lodestar/params"; import {toRootHex} from "@lodestar/utils"; @@ -14,9 +15,9 @@ import {CachedBeaconStateCapella, CachedBeaconStateElectra} from "../types.js"; import { decreaseBalance, getValidatorMaxEffectiveBalance, + hasEth1WithdrawalCredential, + hasExecutionWithdrawalCredential, isCapellaPayloadHeader, - isFullyWithdrawableValidator, - isPartiallyWithdrawableValidator, } from "../util/index.js"; export function processWithdrawals( @@ -24,6 +25,8 @@ export function processWithdrawals( state: CachedBeaconStateCapella | CachedBeaconStateElectra, payload: capella.FullOrBlindedExecutionPayload ): void { + // partialWithdrawalsCount is withdrawals coming from EL since electra (EIP-7002) + // TODO - electra: may switch to executionWithdrawalsCount const {withdrawals: expectedWithdrawals, partialWithdrawalsCount} = getExpectedWithdrawals(fork, state); const numWithdrawals = expectedWithdrawals.length; @@ -86,16 +89,33 @@ export function getExpectedWithdrawals( sampledValidators: number; partialWithdrawalsCount: number; } { + if (fork < ForkSeq.capella) { + throw new Error(`getExpectedWithdrawals not supported at forkSeq=${fork} < ForkSeq.capella`); + } + const epoch = state.epochCtx.epoch; let withdrawalIndex = state.nextWithdrawalIndex; const {validators, balances, nextWithdrawalValidatorIndex} = state; const withdrawals: capella.Withdrawal[] = []; + const isPostElectra = fork >= ForkSeq.electra; - if (fork >= ForkSeq.electra) { + if (isPostElectra) { const stateElectra = state as CachedBeaconStateElectra; - for (const withdrawal of stateElectra.pendingPartialWithdrawals.getAllReadonly()) { + // MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP = 8, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728 so we should only call getAllReadonly() if it makes sense + // pendingPartialWithdrawals comes from EIP-7002 smart contract where it takes fee so it's more likely than not validator is in correct condition to withdraw + // also we may break early if withdrawableEpoch > epoch + const allPendingPartialWithdrawals = + stateElectra.pendingPartialWithdrawals.length <= MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + ? stateElectra.pendingPartialWithdrawals.getAllReadonly() + : null; + + // EIP-7002: Execution layer triggerable withdrawals + for (let i = 0; i < stateElectra.pendingPartialWithdrawals.length; i++) { + const withdrawal = allPendingPartialWithdrawals + ? allPendingPartialWithdrawals[i] + : stateElectra.pendingPartialWithdrawals.getReadonly(i); if (withdrawal.withdrawableEpoch > epoch || withdrawals.length === MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP) { break; } @@ -121,6 +141,7 @@ export function getExpectedWithdrawals( } } + // partialWithdrawalsCount is withdrawals coming from EL since electra (EIP-7002) const partialWithdrawalsCount = withdrawals.length; const bound = Math.min(validators.length, MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP); let n = 0; @@ -132,13 +153,18 @@ export function getExpectedWithdrawals( const validator = validators.getReadonly(validatorIndex); const balance = balances.get(validatorIndex); + const {withdrawableEpoch, withdrawalCredentials, effectiveBalance} = validator; + const hasWithdrawableCredentials = isPostElectra + ? hasExecutionWithdrawalCredential(withdrawalCredentials) + : hasEth1WithdrawalCredential(withdrawalCredentials); // early skip for balance = 0 as its now more likely that validator has exited/slahed with // balance zero than not have withdrawal credentials set - if (balance === 0) { + if (balance === 0 || !hasWithdrawableCredentials) { continue; } - if (isFullyWithdrawableValidator(fork, validator, balance, epoch)) { + // capella full withdrawal + if (withdrawableEpoch <= epoch) { withdrawals.push({ index: withdrawalIndex, validatorIndex, @@ -146,12 +172,17 @@ export function getExpectedWithdrawals( amount: BigInt(balance), }); withdrawalIndex++; - } else if (isPartiallyWithdrawableValidator(fork, validator, balance)) { + } else if ( + effectiveBalance === + (isPostElectra ? getValidatorMaxEffectiveBalance(withdrawalCredentials) : MAX_EFFECTIVE_BALANCE) && + balance > effectiveBalance + ) { + // capella partial withdrawal withdrawals.push({ index: withdrawalIndex, validatorIndex, address: validator.withdrawalCredentials.subarray(12), - amount: BigInt(balance - getValidatorMaxEffectiveBalance(validator.withdrawalCredentials)), + amount: BigInt(balance - effectiveBalance), }); withdrawalIndex++; } diff --git a/packages/state-transition/src/util/electra.ts b/packages/state-transition/src/util/electra.ts index 63f74bc96cc9..ac34da6407de 100644 --- a/packages/state-transition/src/util/electra.ts +++ b/packages/state-transition/src/util/electra.ts @@ -1,17 +1,8 @@ -import { - COMPOUNDING_WITHDRAWAL_PREFIX, - FAR_FUTURE_EPOCH, - ForkSeq, - MAX_EFFECTIVE_BALANCE, - MIN_ACTIVATION_BALANCE, -} from "@lodestar/params"; -import {ValidatorIndex, phase0, ssz} from "@lodestar/types"; +import {COMPOUNDING_WITHDRAWAL_PREFIX, FAR_FUTURE_EPOCH, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; +import {ValidatorIndex, ssz} from "@lodestar/types"; import {CachedBeaconStateElectra} from "../types.js"; -import {getValidatorMaxEffectiveBalance} from "./validator.js"; import {hasEth1WithdrawalCredential} from "./capella.js"; -type ValidatorInfo = Pick; - export function hasCompoundingWithdrawalCredential(withdrawalCredentials: Uint8Array): boolean { return withdrawalCredentials[0] === COMPOUNDING_WITHDRAWAL_PREFIX; } @@ -22,48 +13,6 @@ export function hasExecutionWithdrawalCredential(withdrawalCredentials: Uint8Arr ); } -export function isFullyWithdrawableValidator( - fork: ForkSeq, - validatorCredential: ValidatorInfo, - balance: number, - epoch: number -): boolean { - const {withdrawableEpoch, withdrawalCredentials} = validatorCredential; - - if (fork < ForkSeq.capella) { - throw new Error(`isFullyWithdrawableValidator not supported at forkSeq=${fork} < ForkSeq.capella`); - } - const hasWithdrawableCredentials = - fork >= ForkSeq.electra - ? hasExecutionWithdrawalCredential(withdrawalCredentials) - : hasEth1WithdrawalCredential(withdrawalCredentials); - - return hasWithdrawableCredentials && withdrawableEpoch <= epoch && balance > 0; -} - -export function isPartiallyWithdrawableValidator( - fork: ForkSeq, - validatorCredential: ValidatorInfo, - balance: number -): boolean { - const {effectiveBalance, withdrawalCredentials} = validatorCredential; - - if (fork < ForkSeq.capella) { - throw new Error(`isPartiallyWithdrawableValidator not supported at forkSeq=${fork} < ForkSeq.capella`); - } - const hasWithdrawableCredentials = - fork >= ForkSeq.electra - ? hasExecutionWithdrawalCredential(withdrawalCredentials) - : hasEth1WithdrawalCredential(withdrawalCredentials); - - const validatorMaxEffectiveBalance = - fork >= ForkSeq.electra ? getValidatorMaxEffectiveBalance(withdrawalCredentials) : MAX_EFFECTIVE_BALANCE; - const hasMaxEffectiveBalance = effectiveBalance === validatorMaxEffectiveBalance; - const hasExcessBalance = balance > validatorMaxEffectiveBalance; - - return hasWithdrawableCredentials && hasMaxEffectiveBalance && hasExcessBalance; -} - export function switchToCompoundingValidator(state: CachedBeaconStateElectra, index: ValidatorIndex): void { const validator = state.validators.get(index); From 4e22884db58ac119fdb0e59424841c92d4322ae9 Mon Sep 17 00:00:00 2001 From: twoeths Date: Wed, 4 Sep 2024 08:21:48 +0700 Subject: [PATCH 101/145] fix: improve regen state (#7033) * fix: improve regen state * fix: check for null block returned from db * feat: track state.hashTreeRoot() in regen.getState() * fix: transfer cache when regen state * fix: add caller as label to regenGetState metrics --- packages/beacon-node/src/chain/regen/regen.ts | 70 +++++++++++++++---- .../stateCache/persistentCheckpointsCache.ts | 4 ++ .../src/metrics/metrics/lodestar.ts | 28 ++++++++ .../state-transition/src/stateTransition.ts | 1 + 4 files changed, 91 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 409c12c77b21..04cf5b40b494 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -1,5 +1,5 @@ import {fromHexString} from "@chainsafe/ssz"; -import {phase0, Slot, RootHex, BeaconBlock} from "@lodestar/types"; +import {phase0, Slot, RootHex, BeaconBlock, SignedBeaconBlock} from "@lodestar/types"; import { CachedBeaconStateAllForks, computeEpochAtSlot, @@ -8,6 +8,7 @@ import { DataAvailableStatus, processSlots, stateTransition, + StateHashTreeRootSource, } from "@lodestar/state-transition"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {Logger, toRootHex} from "@lodestar/utils"; @@ -145,7 +146,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { */ async getState( stateRoot: RootHex, - _rCaller: RegenCaller, + caller: RegenCaller, opts?: StateCloneOpts, // internal option, don't want to expose to external caller allowDiskReload = false @@ -156,6 +157,13 @@ export class StateRegenerator implements IStateRegeneratorInternal { return cachedStateCtx; } + // in block gossip validation (getPreState() call), dontTransferCache is specified as true because we only want to transfer cache in verifyBlocksStateTransitionOnly() + // but here we want to process blocks as fast as possible so force to transfer cache in this case + if (opts && allowDiskReload) { + // if there is no `opts` specified, it already means "false" + opts.dontTransferCache = false; + } + // Otherwise we have to use the fork choice to traverse backwards, block by block, // searching the state caches // then replay blocks forward to the desired stateRoot @@ -166,6 +174,8 @@ export class StateRegenerator implements IStateRegeneratorInternal { const blocksToReplay = [block]; let state: CachedBeaconStateAllForks | null = null; const {checkpointStateCache} = this.modules; + + const getSeedStateTimer = this.modules.metrics?.regenGetState.getSeedState.startTimer({caller}); // iterateAncestorBlocks only returns ancestor blocks, not the block itself for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) { state = this.modules.blockStateCache.get(b.stateRoot, opts); @@ -181,6 +191,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { } blocksToReplay.push(b); } + getSeedStateTimer?.(); if (state === null) { throw new RegenError({ @@ -188,19 +199,50 @@ export class StateRegenerator implements IStateRegeneratorInternal { }); } + const blockCount = blocksToReplay.length; const MAX_EPOCH_TO_PROCESS = 5; - if (blocksToReplay.length > MAX_EPOCH_TO_PROCESS * SLOTS_PER_EPOCH) { + if (blockCount > MAX_EPOCH_TO_PROCESS * SLOTS_PER_EPOCH) { throw new RegenError({ code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED, stateRoot, }); } - const replaySlots = blocksToReplay.map((b) => b.slot).join(","); - this.modules.logger.debug("Replaying blocks to get state", {stateRoot, replaySlots}); - for (const b of blocksToReplay.reverse()) { - const block = await this.modules.db.block.get(fromHexString(b.blockRoot)); - if (!block) { + this.modules.metrics?.regenGetState.blockCount.observe({caller}, blockCount); + + const replaySlots = new Array(blockCount); + const blockPromises = new Array>(blockCount); + + const protoBlocksAsc = blocksToReplay.reverse(); + for (const [i, protoBlock] of protoBlocksAsc.entries()) { + replaySlots[i] = protoBlock.slot; + blockPromises[i] = this.modules.db.block.get(fromHexString(protoBlock.blockRoot)); + } + + const logCtx = {stateRoot, replaySlots: replaySlots.join(",")}; + this.modules.logger.debug("Replaying blocks to get state", logCtx); + + const loadBlocksTimer = this.modules.metrics?.regenGetState.loadBlocks.startTimer({caller}); + const blockOrNulls = await Promise.all(blockPromises); + loadBlocksTimer?.(); + + const blocksByRoot = new Map(); + for (const [i, blockOrNull] of blockOrNulls.entries()) { + // checking early here helps prevent unneccessary state transition below + if (blockOrNull === null) { + throw new RegenError({ + code: RegenErrorCode.BLOCK_NOT_IN_DB, + blockRoot: protoBlocksAsc[i].blockRoot, + }); + } + blocksByRoot.set(protoBlocksAsc[i].blockRoot, blockOrNull); + } + + const stateTransitionTimer = this.modules.metrics?.regenGetState.stateTransition.startTimer({caller}); + for (const b of protoBlocksAsc) { + const block = blocksByRoot.get(b.blockRoot); + // just to make compiler happy, we checked in the above for loop already + if (block === undefined) { throw new RegenError({ code: RegenErrorCode.BLOCK_NOT_IN_DB, blockRoot: b.blockRoot, @@ -224,7 +266,12 @@ export class StateRegenerator implements IStateRegeneratorInternal { this.modules.metrics ); + const hashTreeRootTimer = this.modules.metrics?.stateHashTreeRootTime.startTimer({ + source: StateHashTreeRootSource.regenState, + }); const stateRoot = toRootHex(state.hashTreeRoot()); + hashTreeRootTimer?.(); + if (b.stateRoot !== stateRoot) { throw new RegenError({ slot: b.slot, @@ -238,9 +285,6 @@ export class StateRegenerator implements IStateRegeneratorInternal { // also with allowDiskReload flag, we "reload" it to the state cache too this.modules.blockStateCache.add(state); } - - // this avoids keeping our node busy processing blocks - await nextEventLoop(); } catch (e) { throw new RegenError({ code: RegenErrorCode.STATE_TRANSITION_ERROR, @@ -248,7 +292,9 @@ export class StateRegenerator implements IStateRegeneratorInternal { }); } } - this.modules.logger.debug("Replayed blocks to get state", {stateRoot, replaySlots}); + stateTransitionTimer?.(); + + this.modules.logger.debug("Replayed blocks to get state", {...logCtx, stateSlot: state.slot}); return state; } diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index b315beba46d9..823f066abcd6 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -229,6 +229,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { newCachedState.commit(); const stateRoot = toRootHex(newCachedState.hashTreeRoot()); timer?.(); + + // load all cache in order for consumers (usually regen.getState()) to process blocks faster + newCachedState.validators.getAllReadonlyValues(); + newCachedState.balances.getAll(); this.logger.debug("Reload: cached state load successful", { ...logMeta, stateSlot: newCachedState.slot, diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index a0cf0a185c2f..55d43922d936 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1413,6 +1413,34 @@ export function createLodestarMetrics( help: "UnhandledPromiseRejection total count", }), + // regen.getState metrics + regenGetState: { + blockCount: register.histogram<{caller: RegenCaller}>({ + name: "lodestar_regen_get_state_block_count", + help: "Block count in regen.getState", + labelNames: ["caller"], + buckets: [4, 8, 16, 32, 64], + }), + getSeedState: register.histogram<{caller: RegenCaller}>({ + name: "lodestar_regen_get_state_get_seed_state_seconds", + help: "Duration of get seed state in regen.getState", + labelNames: ["caller"], + buckets: [0.1, 0.5, 1, 2, 3, 4], + }), + loadBlocks: register.histogram<{caller: RegenCaller}>({ + name: "lodestar_regen_get_state_load_blocks_seconds", + help: "Duration of load blocks in regen.getState", + labelNames: ["caller"], + buckets: [0.1, 0.5, 1, 2, 3, 4], + }), + stateTransition: register.histogram<{caller: RegenCaller}>({ + name: "lodestar_regen_get_state_state_transition_seconds", + help: "Duration of state transition in regen.getState", + labelNames: ["caller"], + buckets: [0.1, 0.5, 1, 2, 3, 4], + }), + }, + // Precompute next epoch transition precomputeNextEpochTransition: { count: register.counter<{result: string}>({ diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 40e87c8d07d2..3b97f19282a4 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -54,6 +54,7 @@ export enum StateHashTreeRootSource { blockTransition = "block_transition", prepareNextSlot = "prepare_next_slot", prepareNextEpoch = "prepare_next_epoch", + regenState = "regen_state", computeNewStateRoot = "compute_new_state_root", } From fe6c4acbfcc6ee3027b1c4833706b415417e448e Mon Sep 17 00:00:00 2001 From: twoeths Date: Thu, 5 Sep 2024 08:34:43 +0700 Subject: [PATCH 102/145] feat: archive state using BufferPool if provided (#7042) * fix: archive state using BufferPool if provided * chore: fix comment --- .../src/chain/archiver/archiveStates.ts | 14 ++++- .../beacon-node/src/chain/archiver/index.ts | 2 +- packages/beacon-node/src/chain/chain.ts | 6 +- packages/beacon-node/src/chain/interface.ts | 2 + .../beacon-node/src/chain/serializeState.ts | 33 +++++++++++ .../stateCache/persistentCheckpointsCache.ts | 59 ++++++------------- .../src/metrics/metrics/lodestar.ts | 11 ++-- packages/beacon-node/src/util/bufferPool.ts | 20 ++++--- .../test/unit/util/bufferPool.test.ts | 10 ++-- 9 files changed, 94 insertions(+), 63 deletions(-) create mode 100644 packages/beacon-node/src/chain/serializeState.ts diff --git a/packages/beacon-node/src/chain/archiver/archiveStates.ts b/packages/beacon-node/src/chain/archiver/archiveStates.ts index 2231cd3ff513..53f2033d80e8 100644 --- a/packages/beacon-node/src/chain/archiver/archiveStates.ts +++ b/packages/beacon-node/src/chain/archiver/archiveStates.ts @@ -6,6 +6,8 @@ import {CheckpointWithHex} from "@lodestar/fork-choice"; import {IBeaconDb} from "../../db/index.js"; import {IStateRegenerator} from "../regen/interface.js"; import {getStateSlotFromBytes} from "../../util/multifork.js"; +import {serializeState} from "../serializeState.js"; +import {AllocSource, BufferPool} from "../../util/bufferPool.js"; /** * Minimum number of epochs between single temp archived states @@ -30,7 +32,8 @@ export class StatesArchiver { private readonly regen: IStateRegenerator, private readonly db: IBeaconDb, private readonly logger: Logger, - private readonly opts: StatesArchiverOpts + private readonly opts: StatesArchiverOpts, + private readonly bufferPool?: BufferPool | null ) {} /** @@ -95,8 +98,13 @@ export class StatesArchiver { await this.db.stateArchive.putBinary(slot, finalizedStateOrBytes); this.logger.verbose("Archived finalized state bytes", {epoch: finalized.epoch, slot, root: rootHex}); } else { - // state - await this.db.stateArchive.put(finalizedStateOrBytes.slot, finalizedStateOrBytes); + // serialize state using BufferPool if provided + await serializeState( + finalizedStateOrBytes, + AllocSource.ARCHIVE_STATE, + (stateBytes) => this.db.stateArchive.putBinary(finalizedStateOrBytes.slot, stateBytes), + this.bufferPool + ); // don't delete states before the finalized state, auto-prune will take care of it this.logger.verbose("Archived finalized state", { epoch: finalized.epoch, diff --git a/packages/beacon-node/src/chain/archiver/index.ts b/packages/beacon-node/src/chain/archiver/index.ts index ee0711e05e4b..294c2281e19b 100644 --- a/packages/beacon-node/src/chain/archiver/index.ts +++ b/packages/beacon-node/src/chain/archiver/index.ts @@ -48,7 +48,7 @@ export class Archiver { opts: ArchiverOpts ) { this.archiveBlobEpochs = opts.archiveBlobEpochs; - this.statesArchiver = new StatesArchiver(chain.regen, db, logger, opts); + this.statesArchiver = new StatesArchiver(chain.regen, db, logger, opts, chain.bufferPool); this.prevFinalized = chain.forkChoice.getFinalizedCheckpoint(); this.jobQueue = new JobItemQueue<[CheckpointWithHex], void>(this.processFinalizedCheckpoint, { maxLength: PROCESS_FINALIZED_CHECKPOINT_QUEUE_LEN, diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index b410a1e84655..8dbb49798538 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -119,6 +119,7 @@ export class BeaconChain implements IBeaconChain { readonly config: BeaconConfig; readonly logger: Logger; readonly metrics: Metrics | null; + readonly bufferPool: BufferPool | null; readonly anchorStateLatestBlockSlot: Slot; @@ -272,6 +273,9 @@ export class BeaconChain implements IBeaconChain { const blockStateCache = this.opts.nHistoricalStates ? new FIFOBlockStateCache(this.opts, {metrics}) : new BlockStateCacheImpl({metrics}); + this.bufferPool = this.opts.nHistoricalStates + ? new BufferPool(anchorState.type.tree_serializedSize(anchorState.node), metrics) + : null; const checkpointStateCache = this.opts.nHistoricalStates ? new PersistentCheckpointStateCache( { @@ -280,7 +284,7 @@ export class BeaconChain implements IBeaconChain { clock, shufflingCache: this.shufflingCache, blockStateCache, - bufferPool: new BufferPool(anchorState.type.tree_serializedSize(anchorState.node), metrics), + bufferPool: this.bufferPool, datastore: fileDataStore ? // debug option if we want to investigate any issues with the DB new FileCPStateDatastore() diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index ca13dc604ea0..5185662eaa4f 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -30,6 +30,7 @@ import {IEth1ForBlockProduction} from "../eth1/index.js"; import {IExecutionEngine, IExecutionBuilder} from "../execution/index.js"; import {Metrics} from "../metrics/metrics.js"; import {IClock} from "../util/clock.js"; +import {BufferPool} from "../util/bufferPool.js"; import {ChainEventEmitter} from "./emitter.js"; import {IStateRegenerator, RegenCaller} from "./regen/index.js"; import {IBlsVerifier} from "./bls/index.js"; @@ -86,6 +87,7 @@ export interface IBeaconChain { readonly config: BeaconConfig; readonly logger: Logger; readonly metrics: Metrics | null; + readonly bufferPool: BufferPool | null; /** The initial slot that the chain is started with */ readonly anchorStateLatestBlockSlot: Slot; diff --git a/packages/beacon-node/src/chain/serializeState.ts b/packages/beacon-node/src/chain/serializeState.ts new file mode 100644 index 000000000000..cbb2ecd18cff --- /dev/null +++ b/packages/beacon-node/src/chain/serializeState.ts @@ -0,0 +1,33 @@ +import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import {AllocSource, BufferPool} from "../util/bufferPool.js"; + +type ProcessStateBytesFn = (stateBytes: Uint8Array) => Promise; + +/* + * Serialize state using the BufferPool if provided. + */ +export async function serializeState( + state: CachedBeaconStateAllForks, + source: AllocSource, + processFn: ProcessStateBytesFn, + bufferPool?: BufferPool | null +): Promise { + const size = state.type.tree_serializedSize(state.node); + let stateBytes: Uint8Array | null = null; + if (bufferPool) { + const bufferWithKey = bufferPool.alloc(size, source); + if (bufferWithKey) { + stateBytes = bufferWithKey.buffer; + const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength); + state.serializeToBytes({uint8Array: stateBytes, dataView}, 0); + } + } + + if (!stateBytes) { + // we already have metrics in BufferPool so no need to do it here + stateBytes = state.serialize(); + } + + return processFn(stateBytes); + // release the buffer back to the pool automatically +} diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 823f066abcd6..190b79e58cd6 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -8,8 +8,9 @@ import {INTERVALS_PER_SLOT} from "@lodestar/params"; import {Metrics} from "../../metrics/index.js"; import {IClock} from "../../util/clock.js"; import {ShufflingCache} from "../shufflingCache.js"; -import {BufferPool, BufferWithKey} from "../../util/bufferPool.js"; +import {AllocSource, BufferPool, BufferWithKey} from "../../util/bufferPool.js"; import {StateCloneOpts} from "../regen/interface.js"; +import {serializeState} from "../serializeState.js"; import {MapTracker} from "./mapMetrics.js"; import {CPStateDatastore, DatastoreKey, datastoreKeyToCheckpoint} from "./datastore/index.js"; import {CheckpointHex, CacheItemType, CheckpointStateCache, BlockStateCache} from "./types.js"; @@ -29,7 +30,7 @@ type PersistentCheckpointStateCacheModules = { shufflingCache: ShufflingCache; datastore: CPStateDatastore; blockStateCache: BlockStateCache; - bufferPool?: BufferPool; + bufferPool?: BufferPool | null; }; /** checkpoint serialized as a string */ @@ -106,7 +107,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { private readonly datastore: CPStateDatastore; private readonly shufflingCache: ShufflingCache; private readonly blockStateCache: BlockStateCache; - private readonly bufferPool?: BufferPool; + private readonly bufferPool?: BufferPool | null; constructor( { @@ -698,19 +699,20 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // persist and do not update epochIndex this.metrics?.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0); const cpPersist = {epoch: epoch, root: fromHexString(rootHex)}; - { - const timer = this.metrics?.stateSerializeDuration.startTimer(); - // automatically free the buffer pool after this scope - using stateBytesWithKey = this.serializeState(state); - let stateBytes = stateBytesWithKey?.buffer; - if (stateBytes == null) { - // fallback logic to use regular way to get state ssz bytes - this.metrics?.persistedStateAllocCount.inc(); - stateBytes = state.serialize(); - } - timer?.(); - persistedKey = await this.datastore.write(cpPersist, stateBytes); - } + // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. + // As monitored on holesky as of Jan 2024: + // - This does not increase heap allocation while gc time is the same + // - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s) + // - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s) + // - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization + const timer = this.metrics?.stateSerializeDuration.startTimer(); + persistedKey = await serializeState( + state, + AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, + (stateBytes) => this.datastore.write(cpPersist, stateBytes), + this.bufferPool + ); + timer?.(); persistCount++; this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", { ...logMeta, @@ -767,29 +769,6 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { }); } - /* - * It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. - * As monitored on holesky as of Jan 2024: - * - This does not increase heap allocation while gc time is the same - * - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s) - * - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s) - * - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization - */ - private serializeState(state: CachedBeaconStateAllForks): BufferWithKey | null { - const size = state.type.tree_serializedSize(state.node); - if (this.bufferPool) { - const bufferWithKey = this.bufferPool.alloc(size); - if (bufferWithKey) { - const stateBytes = bufferWithKey.buffer; - const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength); - state.serializeToBytes({uint8Array: stateBytes, dataView}, 0); - return bufferWithKey; - } - } - - return null; - } - /** * Serialize validators to bytes leveraging the buffer pool to save memory allocation. * - As monitored on holesky as of Jan 2024, it helps save ~500ms state reload time (4.3s vs 3.8s) @@ -800,7 +779,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const type = state.type.fields.validators; const size = type.tree_serializedSize(state.validators.node); if (this.bufferPool) { - const bufferWithKey = this.bufferPool.alloc(size); + const bufferWithKey = this.bufferPool.alloc(size, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_VALIDATORS); if (bufferWithKey) { const validatorsBytes = bufferWithKey.buffer; const dataView = new DataView(validatorsBytes.buffer, validatorsBytes.byteOffset, validatorsBytes.byteLength); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 55d43922d936..737a900e5f64 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -18,6 +18,7 @@ import {LodestarMetadata} from "../options.js"; import {RegistryMetricCreator} from "../utils/registryMetricCreator.js"; import {OpSource} from "../validatorMonitor.js"; import {CacheItemType} from "../../chain/stateCache/types.js"; +import {AllocSource} from "../../util/bufferPool.js"; export type LodestarMetrics = ReturnType; @@ -1165,13 +1166,15 @@ export function createLodestarMetrics( name: "lodestar_buffer_pool_length", help: "Buffer pool length", }), - hits: register.counter({ + hits: register.counter<{source: AllocSource}>({ name: "lodestar_buffer_pool_hits_total", help: "Total number of buffer pool hits", + labelNames: ["source"], }), - misses: register.counter({ + misses: register.counter<{source: AllocSource}>({ name: "lodestar_buffer_pool_misses_total", help: "Total number of buffer pool misses", + labelNames: ["source"], }), grows: register.counter({ name: "lodestar_buffer_pool_grows_total", @@ -1271,10 +1274,6 @@ export function createLodestarMetrics( name: "lodestar_cp_state_cache_persisted_state_remove_count", help: "Total number of persisted states removed", }), - persistedStateAllocCount: register.counter({ - name: "lodestar_cp_state_cache_persisted_state_alloc_count", - help: "Total number time to allocate memory for persisted state", - }), }, balancesCache: { diff --git a/packages/beacon-node/src/util/bufferPool.ts b/packages/beacon-node/src/util/bufferPool.ts index f9e18a6d64a5..e3cf10fa88b3 100644 --- a/packages/beacon-node/src/util/bufferPool.ts +++ b/packages/beacon-node/src/util/bufferPool.ts @@ -5,6 +5,12 @@ import {Metrics} from "../metrics/metrics.js"; */ const GROW_RATIO = 1.1; +export enum AllocSource { + PERSISTENT_CHECKPOINTS_CACHE_VALIDATORS = "persistent_checkpoints_cache_validators", + PERSISTENT_CHECKPOINTS_CACHE_STATE = "persistent_checkpoints_cache_state", + ARCHIVE_STATE = "archive_state", +} + /** * A simple implementation to manage a single buffer. * This is initially used for state serialization at every epoch and for state reload. @@ -36,24 +42,24 @@ export class BufferPool { * If the buffer is already in use, return null. * Grow the buffer if the requested size is larger than the current buffer. */ - alloc(size: number): BufferWithKey | null { - return this.doAlloc(size, false); + alloc(size: number, source: AllocSource): BufferWithKey | null { + return this.doAlloc(size, source, false); } /** * Same to alloc() but the buffer is not zeroed. */ - allocUnsafe(size: number): BufferWithKey | null { - return this.doAlloc(size, true); + allocUnsafe(size: number, source: AllocSource): BufferWithKey | null { + return this.doAlloc(size, source, true); } - private doAlloc(size: number, isUnsafe = false): BufferWithKey | null { + private doAlloc(size: number, source: AllocSource, isUnsafe = false): BufferWithKey | null { if (this.inUse) { - this.metrics?.misses.inc(); + this.metrics?.misses.inc({source}); return null; } this.inUse = true; - this.metrics?.hits.inc(); + this.metrics?.hits.inc({source}); this.currentKey += 1; if (size > this.buffer.length) { this.metrics?.grows.inc(); diff --git a/packages/beacon-node/test/unit/util/bufferPool.test.ts b/packages/beacon-node/test/unit/util/bufferPool.test.ts index 2c789c19f74d..ff66504ae65f 100644 --- a/packages/beacon-node/test/unit/util/bufferPool.test.ts +++ b/packages/beacon-node/test/unit/util/bufferPool.test.ts @@ -1,12 +1,12 @@ import {describe, it, expect} from "vitest"; -import {BufferPool} from "../../../src/util/bufferPool.js"; +import {AllocSource, BufferPool} from "../../../src/util/bufferPool.js"; describe("BufferPool", () => { const pool = new BufferPool(100); it("should increase length", () => { expect(pool.length).toEqual(110); - using mem = pool.alloc(200); + using mem = pool.alloc(200, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE); if (mem === null) { throw Error("Expected non-null mem"); } @@ -15,15 +15,15 @@ describe("BufferPool", () => { it("should not allow alloc if in use", () => { { - using mem = pool.alloc(20); + using mem = pool.alloc(20, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE); if (mem === null) { throw Error("Expected non-null mem"); } // in the same scope we can't allocate again - expect(pool.alloc(20)).toEqual(null); + expect(pool.alloc(20, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE)).toEqual(null); } // out of the scope we can allocate again - expect(pool.alloc(20)).not.toEqual(null); + expect(pool.alloc(20, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE)).not.toEqual(null); }); }); From b05c93e1471083d10fefda20c11514fd3afe94df Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 5 Sep 2024 02:35:10 +0100 Subject: [PATCH 103/145] feat: include more details in validator attestation logs (#7064) --- packages/validator/src/services/attestation.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index fc43b603c6b2..927bd3d92bb4 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -2,7 +2,7 @@ import {toHexString} from "@chainsafe/ssz"; import {BLSSignature, phase0, Slot, ssz, Attestation, SignedAggregateAndProof} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; -import {sleep} from "@lodestar/utils"; +import {prettyBytes, sleep} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; import {ChainForkConfig} from "@lodestar/config"; import {IClock, LoggerVc} from "../util/index.js"; @@ -242,7 +242,11 @@ export class AttestationService { } else { (await this.api.beacon.submitPoolAttestations({signedAttestations})).assertOk(); } - this.logger.info("Published attestations", {...logCtx, count: signedAttestations.length}); + this.logger.info("Published attestations", { + ...logCtx, + head: prettyBytes(headRootHex), + count: signedAttestations.length, + }); this.metrics?.publishedAttestations.inc(signedAttestations.length); } catch (e) { // Note: metric counts only 1 since we don't know how many signedAttestations are invalid @@ -286,7 +290,8 @@ export class AttestationService { slot: attestation.slot, }); const aggregate = res.value(); - this.metrics?.numParticipantsInAggregate.observe(aggregate.aggregationBits.getTrueBitIndexes().length); + const participants = aggregate.aggregationBits.getTrueBitIndexes().length; + this.metrics?.numParticipantsInAggregate.observe(participants); const signedAggregateAndProofs: SignedAggregateAndProof[] = []; @@ -316,7 +321,11 @@ export class AttestationService { } else { (await this.api.validator.publishAggregateAndProofs({signedAggregateAndProofs})).assertOk(); } - this.logger.info("Published aggregateAndProofs", {...logCtx, count: signedAggregateAndProofs.length}); + this.logger.info("Published aggregateAndProofs", { + ...logCtx, + participants, + count: signedAggregateAndProofs.length, + }); this.metrics?.publishedAggregates.inc(signedAggregateAndProofs.length); } catch (e) { this.logger.error("Error publishing aggregateAndProofs", logCtx, e as Error); From 0e79d29720185e180ff684162ffba188313e8df2 Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Mon, 9 Sep 2024 14:35:40 -0700 Subject: [PATCH 104/145] feat: rename getValidatorMaxEffectiveBalance (#7070) * Rename getValidatorMaxEffectiveBalance * Lint --- packages/state-transition/src/block/processWithdrawals.ts | 5 ++--- packages/state-transition/src/util/genesis.ts | 4 ++-- packages/state-transition/src/util/validator.ts | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/state-transition/src/block/processWithdrawals.ts b/packages/state-transition/src/block/processWithdrawals.ts index 185ddd80eb32..d4dfd47b4d94 100644 --- a/packages/state-transition/src/block/processWithdrawals.ts +++ b/packages/state-transition/src/block/processWithdrawals.ts @@ -14,7 +14,7 @@ import {toRootHex} from "@lodestar/utils"; import {CachedBeaconStateCapella, CachedBeaconStateElectra} from "../types.js"; import { decreaseBalance, - getValidatorMaxEffectiveBalance, + getMaxEffectiveBalance, hasEth1WithdrawalCredential, hasExecutionWithdrawalCredential, isCapellaPayloadHeader, @@ -173,8 +173,7 @@ export function getExpectedWithdrawals( }); withdrawalIndex++; } else if ( - effectiveBalance === - (isPostElectra ? getValidatorMaxEffectiveBalance(withdrawalCredentials) : MAX_EFFECTIVE_BALANCE) && + effectiveBalance === (isPostElectra ? getMaxEffectiveBalance(withdrawalCredentials) : MAX_EFFECTIVE_BALANCE) && balance > effectiveBalance ) { // capella partial withdrawal diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 02bcef00bb55..54507d0ef235 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -18,7 +18,7 @@ import {EpochCacheImmutableData} from "../cache/epochCache.js"; import {processDeposit} from "../block/processDeposit.js"; import {increaseBalance} from "../index.js"; import {computeEpochAtSlot} from "./epoch.js"; -import {getActiveValidatorIndices, getValidatorMaxEffectiveBalance} from "./validator.js"; +import {getActiveValidatorIndices, getMaxEffectiveBalance} from "./validator.js"; import {getTemporaryBlockHeader} from "./blockRoot.js"; import {newFilledArray} from "./array.js"; import {getNextSyncCommittee} from "./syncCommittee.js"; @@ -195,7 +195,7 @@ export function applyDeposits( const balance = balancesArr[i]; const effectiveBalance = Math.min( balance - (balance % EFFECTIVE_BALANCE_INCREMENT), - getValidatorMaxEffectiveBalance(validator.withdrawalCredentials) + getMaxEffectiveBalance(validator.withdrawalCredentials) ); validator.effectiveBalance = effectiveBalance; diff --git a/packages/state-transition/src/util/validator.ts b/packages/state-transition/src/util/validator.ts index 728f14587fde..ebad21d9d25c 100644 --- a/packages/state-transition/src/util/validator.ts +++ b/packages/state-transition/src/util/validator.ts @@ -75,7 +75,7 @@ export function getConsolidationChurnLimit(epochCtx: EpochCache): number { return getBalanceChurnLimit(epochCtx) - getActivationExitChurnLimit(epochCtx); } -export function getValidatorMaxEffectiveBalance(withdrawalCredentials: Uint8Array): number { +export function getMaxEffectiveBalance(withdrawalCredentials: Uint8Array): number { // Compounding withdrawal credential only available since Electra if (hasCompoundingWithdrawalCredential(withdrawalCredentials)) { return MAX_EFFECTIVE_BALANCE_ELECTRA; @@ -85,7 +85,7 @@ export function getValidatorMaxEffectiveBalance(withdrawalCredentials: Uint8Arra } export function getActiveBalance(state: CachedBeaconStateElectra, validatorIndex: ValidatorIndex): number { - const validatorMaxEffectiveBalance = getValidatorMaxEffectiveBalance( + const validatorMaxEffectiveBalance = getMaxEffectiveBalance( state.validators.getReadonly(validatorIndex).withdrawalCredentials ); From cbc00c7c16384fe9de82e6f4670792245efd0452 Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Mon, 9 Sep 2024 17:37:50 -0400 Subject: [PATCH 105/145] feat: add util to diff ssz objects (#7041) --- packages/utils/src/diff.ts | 232 ++++++++++++++++++++++++++++++++++++ packages/utils/src/index.ts | 1 + 2 files changed, 233 insertions(+) create mode 100644 packages/utils/src/diff.ts diff --git a/packages/utils/src/diff.ts b/packages/utils/src/diff.ts new file mode 100644 index 000000000000..204989016b46 --- /dev/null +++ b/packages/utils/src/diff.ts @@ -0,0 +1,232 @@ +/* eslint-disable no-console */ +import fs from "node:fs"; + +const primitiveTypeof = ["number", "string", "bigint", "boolean"]; +export type BufferType = Uint8Array | Uint32Array; +export type PrimitiveType = number | string | bigint | boolean | BufferType; +export type DiffableCollection = Record; +export type Diffable = PrimitiveType | Array | DiffableCollection; + +export interface Diff { + objectPath: string; + errorMessage?: string; + val1: Diffable; + val2: Diffable; +} + +export function diffUint8Array(val1: Uint8Array, val2: PrimitiveType, objectPath: string): Diff[] { + if (!(val2 instanceof Uint8Array)) { + return [ + { + objectPath, + errorMessage: `val1${objectPath} is a Uint8Array, but val2${objectPath} is not`, + val1, + val2, + }, + ]; + } + const hex1 = Buffer.from(val1).toString("hex"); + const hex2 = Buffer.from(val2).toString("hex"); + if (hex1 !== hex2) { + return [ + { + objectPath, + val1: `0x${hex1}`, + val2: `0x${hex2}`, + }, + ]; + } + return []; +} + +export function diffUint32Array(val1: Uint32Array, val2: PrimitiveType, objectPath: string): Diff[] { + if (!(val2 instanceof Uint32Array)) { + return [ + { + objectPath, + errorMessage: `val1${objectPath} is a Uint32Array, but val2${objectPath} is not`, + val1, + val2, + }, + ]; + } + const diffs: Diff[] = []; + val1.forEach((value, index) => { + const value2 = val2[index]; + if (value !== value2) { + diffs.push({ + objectPath: `${objectPath}[${index}]`, + val1: `0x${value.toString(16).padStart(8, "0")}`, + val2: value2 ? `0x${val2[index].toString(16).padStart(8, "0")}` : "undefined", + }); + } + }); + return diffs; +} + +function diffPrimitiveValue(val1: PrimitiveType, val2: PrimitiveType, objectPath: string): Diff[] { + if (val1 instanceof Uint8Array) { + return diffUint8Array(val1, val2, objectPath); + } + if (val1 instanceof Uint32Array) { + return diffUint32Array(val1, val2, objectPath); + } + + const diff = {objectPath, val1, val2} as Diff; + const type1 = typeof val1; + if (!primitiveTypeof.includes(type1)) { + diff.errorMessage = `val1${objectPath} is not a supported type`; + } + const type2 = typeof val2; + if (!primitiveTypeof.includes(type2)) { + diff.errorMessage = `val2${objectPath} is not a supported type`; + } + if (type1 !== type2) { + diff.errorMessage = `val1${objectPath} is not the same type as val2${objectPath}`; + } + if (val1 !== val2) { + return [diff]; + } + return []; +} + +function isPrimitiveValue(val: unknown): val is PrimitiveType { + if (Array.isArray(val)) return false; + if (typeof val === "object") { + return val instanceof Uint8Array || val instanceof Uint32Array; + } + return true; +} + +function isDiffable(val: unknown): val is Diffable { + return !(typeof val === "function" || typeof val === "symbol" || typeof val === "undefined" || val === null); +} + +export function getDiffs(val1: Diffable, val2: Diffable, objectPath: string): Diff[] { + if (isPrimitiveValue(val1)) { + if (!isPrimitiveValue(val2)) { + return [ + { + objectPath, + errorMessage: `val1${objectPath} is a primitive value and val2${objectPath} is not`, + val1, + val2, + }, + ]; + } + return diffPrimitiveValue(val1, val2, objectPath); + } + + const isArray = Array.isArray(val1); + let errorMessage: string | undefined; + if (isArray && !Array.isArray(val2)) { + errorMessage = `val1${objectPath} is an array and val2${objectPath} is not`; + } else if (typeof val1 === "object" && typeof val2 !== "object") { + errorMessage = `val1${objectPath} is a nested object and val2${objectPath} is not`; + } + if (errorMessage) { + return [ + { + objectPath, + errorMessage, + val1, + val2, + }, + ]; + } + + const diffs: Diff[] = []; + for (const [index, value] of Object.entries(val1)) { + if (!isDiffable(value)) { + diffs.push({objectPath, val1, val2, errorMessage: `val1${objectPath} is not Diffable`}); + continue; + } + const value2 = (val2 as DiffableCollection)[index]; + if (!isDiffable(value2)) { + diffs.push({objectPath, val1, val2, errorMessage: `val2${objectPath} is not Diffable`}); + continue; + } + const innerPath = isArray ? `${objectPath}[${index}]` : `${objectPath}.${index}`; + diffs.push(...getDiffs(value, value2, innerPath)); + } + return diffs; +} + +/** + * Find the different values on complex, nested objects. Outputs the path through the object to + * each value that does not match from val1 and val2. Optionally can output the values that differ. + * + * For objects that differ greatly, can write to a file instead of the terminal for analysis + * + * ## Example + * ```ts + * const obj1 = { + * key1: { + * key2: [ + * { key3: 1 }, + * { key3: new Uint8Array([1, 2, 3]) } + * ] + * }, + * key4: new Uint32Array([1, 2, 3]), + * key5: 362436 + * }; + * + * const obj2 = { + * key1: { + * key2: [ + * { key3: 1 }, + * { key3: new Uint8Array([1, 2, 4]) } + * ] + * }, + * key4: new Uint32Array([1, 2, 4]) + * key5: true + * }; + * + * diffObjects(obj1, obj2, true); + * + * + * ``` + * + * ## Output + * ```sh + * val.key1.key2[1].key3 + * - 0x010203 + * - 0x010204 + * val.key4[2] + * - 0x00000003 + * - 0x00000004 + * val.key5 + * val1.key5 is not the same type as val2.key5 + * - 362436 + * - true + * ``` + */ +export function diff(val1: unknown, val2: unknown, outputValues = false, filename?: string): void { + if (!isDiffable(val1)) { + console.log("val1 is not Diffable"); + return; + } + if (!isDiffable(val2)) { + console.log("val2 is not Diffable"); + return; + } + const diffs = getDiffs(val1, val2, ""); + let output = ""; + if (diffs.length) { + diffs.forEach((diff) => { + let diffOutput = `value${diff.objectPath}`; + if (diff.errorMessage) { + diffOutput += `\n ${diff.errorMessage}`; + } + if (outputValues) { + diffOutput += `\n - ${diff.val1.toString()}\n - ${diff.val2.toString()}\n`; + } + output += `${diffOutput}\n`; + }); + if (filename) { + fs.writeFileSync(filename, output); + } else { + console.log(output); + } + } +} diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 13ea1ffb7e69..4e0be0c592a1 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -4,6 +4,7 @@ export * from "./base64.js"; export * from "./bytes.js"; export * from "./bytes/index.js"; export * from "./command.js"; +export * from "./diff.js"; export * from "./err.js"; export * from "./errors.js"; export * from "./format.js"; From 6c1e335a5734bba7e38f160e26d134300eb7cee3 Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 10 Sep 2024 15:11:07 +0700 Subject: [PATCH 106/145] fix: single state tree at start up (#7056) * feat: use db state to load ws state * feat: log state size * fix: rename initStateFromAnchorState to checkAndPersistAnchorState * fix: only persist anchor state if it's cp state * fix: avoid redundant anchor state serialization --- packages/beacon-node/src/chain/initState.ts | 18 ++-- packages/beacon-node/src/index.ts | 4 +- .../cli/src/cmds/beacon/initBeaconState.ts | 101 ++++++++++++------ packages/cli/src/networks/index.ts | 35 ++++-- packages/state-transition/src/util/index.ts | 1 + .../src/util/loadState/index.ts | 2 +- .../src/util/loadState/loadState.ts | 19 ++++ .../test/unit/util/loadState.test.ts | 40 +++++++ packages/utils/src/bytes.ts | 20 ++++ packages/utils/test/unit/bytes.test.ts | 32 +++++- 10 files changed, 222 insertions(+), 50 deletions(-) create mode 100644 packages/state-transition/test/unit/util/loadState.test.ts diff --git a/packages/beacon-node/src/chain/initState.ts b/packages/beacon-node/src/chain/initState.ts index 19f0f1cc959d..e413bdff0f7d 100644 --- a/packages/beacon-node/src/chain/initState.ts +++ b/packages/beacon-node/src/chain/initState.ts @@ -37,17 +37,18 @@ export async function persistGenesisResult( export async function persistAnchorState( config: ChainForkConfig, db: IBeaconDb, - anchorState: BeaconStateAllForks + anchorState: BeaconStateAllForks, + anchorStateBytes: Uint8Array ): Promise { if (anchorState.slot === GENESIS_SLOT) { const genesisBlock = createGenesisBlock(config, anchorState); await Promise.all([ db.blockArchive.add(genesisBlock), db.block.add(genesisBlock), - db.stateArchive.add(anchorState), + db.stateArchive.putBinary(anchorState.slot, anchorStateBytes), ]); } else { - await db.stateArchive.add(anchorState); + await db.stateArchive.putBinary(anchorState.slot, anchorStateBytes); } } @@ -154,16 +155,17 @@ export async function initStateFromDb( /** * Initialize and persist an anchor state (either weak subjectivity or genesis) */ -export async function initStateFromAnchorState( +export async function checkAndPersistAnchorState( config: ChainForkConfig, db: IBeaconDb, logger: Logger, anchorState: BeaconStateAllForks, + anchorStateBytes: Uint8Array, { isWithinWeakSubjectivityPeriod, isCheckpointState, }: {isWithinWeakSubjectivityPeriod: boolean; isCheckpointState: boolean} -): Promise { +): Promise { const expectedFork = config.getForkInfo(computeStartSlotAtEpoch(anchorState.fork.epoch)); const expectedForkVersion = toHex(expectedFork.version); const stateFork = toHex(anchorState.fork.currentVersion); @@ -191,9 +193,9 @@ export async function initStateFromAnchorState( logger.warn("Checkpoint sync recommended, please use --help to see checkpoint sync options"); } - await persistAnchorState(config, db, anchorState); - - return anchorState; + if (isCheckpointState || anchorState.slot === GENESIS_SLOT) { + await persistAnchorState(config, db, anchorState, anchorStateBytes); + } } export function initBeaconMetrics(metrics: Metrics, state: BeaconStateAllForks): void { diff --git a/packages/beacon-node/src/index.ts b/packages/beacon-node/src/index.ts index aa555a1ab0ca..723b56d0b488 100644 --- a/packages/beacon-node/src/index.ts +++ b/packages/beacon-node/src/index.ts @@ -1,4 +1,4 @@ -export {initStateFromAnchorState, initStateFromDb, initStateFromEth1} from "./chain/index.js"; +export {checkAndPersistAnchorState, initStateFromDb, initStateFromEth1} from "./chain/index.js"; export {BeaconDb, type IBeaconDb} from "./db/index.js"; export {Eth1Provider, type IEth1Provider} from "./eth1/index.js"; export {createNodeJsLibp2p, type NodeJsLibp2pOpts} from "./network/index.js"; @@ -20,4 +20,4 @@ export {RestApiServer} from "./api/rest/base.js"; export type {RestApiServerOpts, RestApiServerModules, RestApiServerMetrics} from "./api/rest/base.js"; // Export type util for CLI - TEMP move to lodestar-types eventually -export {getStateTypeFromBytes} from "./util/multifork.js"; +export {getStateTypeFromBytes, getStateSlotFromBytes} from "./util/multifork.js"; diff --git a/packages/cli/src/cmds/beacon/initBeaconState.ts b/packages/cli/src/cmds/beacon/initBeaconState.ts index c8c444778991..67b578e9fdb9 100644 --- a/packages/cli/src/cmds/beacon/initBeaconState.ts +++ b/packages/cli/src/cmds/beacon/initBeaconState.ts @@ -1,15 +1,17 @@ import {ssz} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainForkConfig} from "@lodestar/config"; -import {Logger} from "@lodestar/utils"; +import {Logger, formatBytes} from "@lodestar/utils"; import { isWithinWeakSubjectivityPeriod, ensureWithinWeakSubjectivityPeriod, BeaconStateAllForks, + loadState, + loadStateAndValidators, } from "@lodestar/state-transition"; import { IBeaconDb, IBeaconNodeOptions, - initStateFromAnchorState, + checkAndPersistAnchorState, initStateFromEth1, getStateTypeFromBytes, } from "@lodestar/beacon-node"; @@ -25,19 +27,23 @@ import { } from "../../networks/index.js"; import {BeaconArgs} from "./options.js"; +type StateWithBytes = {state: BeaconStateAllForks; stateBytes: Uint8Array}; + async function initAndVerifyWeakSubjectivityState( config: BeaconConfig, db: IBeaconDb, logger: Logger, - store: BeaconStateAllForks, - wsState: BeaconStateAllForks, + dbStateBytes: StateWithBytes, + wsStateBytes: StateWithBytes, wsCheckpoint: Checkpoint, opts: {ignoreWeakSubjectivityCheck?: boolean} = {} ): Promise<{anchorState: BeaconStateAllForks; wsCheckpoint: Checkpoint}> { + const dbState = dbStateBytes.state; + const wsState = wsStateBytes.state; // Check if the store's state and wsState are compatible if ( - store.genesisTime !== wsState.genesisTime || - !ssz.Root.equals(store.genesisValidatorsRoot, wsState.genesisValidatorsRoot) + dbState.genesisTime !== wsState.genesisTime || + !ssz.Root.equals(dbState.genesisValidatorsRoot, wsState.genesisValidatorsRoot) ) { throw new Error( "Db state and checkpoint state are not compatible, either clear the db or verify your checkpoint source" @@ -45,12 +51,12 @@ async function initAndVerifyWeakSubjectivityState( } // Pick the state which is ahead as an anchor to initialize the beacon chain - let anchorState = wsState; + let anchorState = wsStateBytes; let anchorCheckpoint = wsCheckpoint; let isCheckpointState = true; - if (store.slot > wsState.slot) { - anchorState = store; - anchorCheckpoint = getCheckpointFromState(store); + if (dbState.slot > wsState.slot) { + anchorState = dbStateBytes; + anchorCheckpoint = getCheckpointFromState(dbState); isCheckpointState = false; logger.verbose( "Db state is ahead of the provided checkpoint state, using the db state to initialize the beacon chain" @@ -59,19 +65,19 @@ async function initAndVerifyWeakSubjectivityState( // Throw error unless user explicitly asked not to, in testnets can happen that wss period is too small // that even some epochs of non finalization can cause finalized checkpoint to be out of valid range - const wssCheck = wrapFnError(() => ensureWithinWeakSubjectivityPeriod(config, anchorState, anchorCheckpoint)); + const wssCheck = wrapFnError(() => ensureWithinWeakSubjectivityPeriod(config, anchorState.state, anchorCheckpoint)); const isWithinWeakSubjectivityPeriod = wssCheck.err === null; if (!isWithinWeakSubjectivityPeriod && !opts.ignoreWeakSubjectivityCheck) { throw wssCheck.err; } - anchorState = await initStateFromAnchorState(config, db, logger, anchorState, { + await checkAndPersistAnchorState(config, db, logger, anchorState.state, anchorState.stateBytes, { isWithinWeakSubjectivityPeriod, isCheckpointState, }); // Return the latest anchorState but still return original wsCheckpoint to validate in backfill - return {anchorState, wsCheckpoint}; + return {anchorState: anchorState.state, wsCheckpoint}; } /** @@ -96,8 +102,20 @@ export async function initBeaconState( } // fetch the latest state stored in the db which will be used in all cases, if it exists, either // i) used directly as the anchor state - // ii) used during verification of a weak subjectivity state, - const lastDbState = await db.stateArchive.lastValue(); + // ii) used to load and verify a weak subjectivity state, + const lastDbSlot = await db.stateArchive.lastKey(); + const stateBytes = lastDbSlot !== null ? await db.stateArchive.getBinary(lastDbSlot) : null; + let lastDbState: BeaconStateAllForks | null = null; + let lastDbValidatorsBytes: Uint8Array | null = null; + let lastDbStateWithBytes: StateWithBytes | null = null; + if (stateBytes) { + logger.verbose("Found the last archived state", {slot: lastDbSlot, size: formatBytes(stateBytes.length)}); + const {state, validatorsBytes} = loadStateAndValidators(chainForkConfig, stateBytes); + lastDbState = state; + lastDbValidatorsBytes = validatorsBytes; + lastDbStateWithBytes = {state, stateBytes: stateBytes}; + } + if (lastDbState) { const config = createBeaconConfig(chainForkConfig, lastDbState.genesisValidatorsRoot); const wssCheck = isWithinWeakSubjectivityPeriod(config, lastDbState, getCheckpointFromState(lastDbState)); @@ -107,7 +125,9 @@ export async function initBeaconState( // Forcing to sync from checkpoint is only recommended if node is taking too long to sync from last db state. // It is important to remind the user to remove this flag again unless it is absolutely necessary. if (wssCheck) { - logger.warn("Forced syncing from checkpoint even though db state is within weak subjectivity period"); + logger.warn( + `Forced syncing from checkpoint even though db state at slot ${lastDbState.slot} is within weak subjectivity period` + ); logger.warn("Please consider removing --forceCheckpointSync flag unless absolutely necessary"); } } else { @@ -115,11 +135,15 @@ export async function initBeaconState( // - if no checkpoint sync args provided, or // - the lastDbState is within weak subjectivity period: if ((!args.checkpointState && !args.checkpointSyncUrl) || wssCheck) { - const anchorState = await initStateFromAnchorState(config, db, logger, lastDbState, { + if (stateBytes === null) { + // this never happens + throw Error(`There is no stateBytes for the lastDbState at slot ${lastDbState.slot}`); + } + await checkAndPersistAnchorState(config, db, logger, lastDbState, stateBytes, { isWithinWeakSubjectivityPeriod: wssCheck, isCheckpointState: false, }); - return {anchorState}; + return {anchorState: lastDbState}; } } } @@ -127,7 +151,8 @@ export async function initBeaconState( // See if we can sync state using checkpoint sync args or else start from genesis if (args.checkpointState) { return readWSState( - lastDbState, + lastDbStateWithBytes, + lastDbValidatorsBytes, { checkpointState: args.checkpointState, wssCheckpoint: args.wssCheckpoint, @@ -139,7 +164,8 @@ export async function initBeaconState( ); } else if (args.checkpointSyncUrl) { return fetchWSStateFromBeaconApi( - lastDbState, + lastDbStateWithBytes, + lastDbValidatorsBytes, { checkpointSyncUrl: args.checkpointSyncUrl, wssCheckpoint: args.wssCheckpoint, @@ -153,10 +179,10 @@ export async function initBeaconState( const genesisStateFile = args.genesisStateFile || getGenesisFileUrl(args.network || defaultNetwork); if (genesisStateFile && !args.forceGenesis) { const stateBytes = await downloadOrLoadFile(genesisStateFile); - let anchorState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); + const anchorState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); const config = createBeaconConfig(chainForkConfig, anchorState.genesisValidatorsRoot); const wssCheck = isWithinWeakSubjectivityPeriod(config, anchorState, getCheckpointFromState(anchorState)); - anchorState = await initStateFromAnchorState(config, db, logger, anchorState, { + await checkAndPersistAnchorState(config, db, logger, anchorState, stateBytes, { isWithinWeakSubjectivityPeriod: wssCheck, isCheckpointState: true, }); @@ -170,7 +196,8 @@ export async function initBeaconState( } async function readWSState( - lastDbState: BeaconStateAllForks | null, + lastDbStateBytes: StateWithBytes | null, + lastDbValidatorsBytes: Uint8Array | null, wssOpts: {checkpointState: string; wssCheckpoint?: string; ignoreWeakSubjectivityCheck?: boolean}, chainForkConfig: ChainForkConfig, db: IBeaconDb, @@ -180,19 +207,28 @@ async function readWSState( // if a weak subjectivity checkpoint has been provided, it is used for additional verification // otherwise, the state itself is used for verification (not bad, because the trusted state has been explicitly provided) const {checkpointState, wssCheckpoint, ignoreWeakSubjectivityCheck} = wssOpts; + const lastDbState = lastDbStateBytes?.state ?? null; const stateBytes = await downloadOrLoadFile(checkpointState); - const wsState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); + let wsState: BeaconStateAllForks; + if (lastDbState && lastDbValidatorsBytes) { + // use lastDbState to load wsState if possible to share the same state tree + wsState = loadState(chainForkConfig, lastDbState, stateBytes, lastDbValidatorsBytes).state; + } else { + wsState = getStateTypeFromBytes(chainForkConfig, stateBytes).deserializeToViewDU(stateBytes); + } const config = createBeaconConfig(chainForkConfig, wsState.genesisValidatorsRoot); - const store = lastDbState ?? wsState; + const wsStateBytes = {state: wsState, stateBytes}; + const store = lastDbStateBytes ?? wsStateBytes; const checkpoint = wssCheckpoint ? getCheckpointFromArg(wssCheckpoint) : getCheckpointFromState(wsState); - return initAndVerifyWeakSubjectivityState(config, db, logger, store, wsState, checkpoint, { + return initAndVerifyWeakSubjectivityState(config, db, logger, store, wsStateBytes, checkpoint, { ignoreWeakSubjectivityCheck, }); } async function fetchWSStateFromBeaconApi( - lastDbState: BeaconStateAllForks | null, + lastDbStateBytes: StateWithBytes | null, + lastDbValidatorsBytes: Uint8Array | null, wssOpts: {checkpointSyncUrl: string; wssCheckpoint?: string; ignoreWeakSubjectivityCheck?: boolean}, chainForkConfig: ChainForkConfig, db: IBeaconDb, @@ -213,10 +249,15 @@ async function fetchWSStateFromBeaconApi( throw e; } - const {wsState, wsCheckpoint} = await fetchWeakSubjectivityState(chainForkConfig, logger, wssOpts); + const {wsState, wsStateBytes, wsCheckpoint} = await fetchWeakSubjectivityState(chainForkConfig, logger, wssOpts, { + lastDbState: lastDbStateBytes?.state ?? null, + lastDbValidatorsBytes, + }); + const config = createBeaconConfig(chainForkConfig, wsState.genesisValidatorsRoot); - const store = lastDbState ?? wsState; - return initAndVerifyWeakSubjectivityState(config, db, logger, store, wsState, wsCheckpoint, { + const wsStateWithBytes = {state: wsState, stateBytes: wsStateBytes}; + const store = lastDbStateBytes ?? wsStateWithBytes; + return initAndVerifyWeakSubjectivityState(config, db, logger, store, wsStateWithBytes, wsCheckpoint, { ignoreWeakSubjectivityCheck: wssOpts.ignoreWeakSubjectivityCheck, }); } diff --git a/packages/cli/src/networks/index.ts b/packages/cli/src/networks/index.ts index 2d605335b0e8..0831b78cd2f6 100644 --- a/packages/cli/src/networks/index.ts +++ b/packages/cli/src/networks/index.ts @@ -3,12 +3,17 @@ import got from "got"; import {ENR} from "@chainsafe/enr"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {HttpHeader, MediaType, WireFormat, getClient} from "@lodestar/api"; -import {getStateTypeFromBytes} from "@lodestar/beacon-node"; +import {getStateSlotFromBytes} from "@lodestar/beacon-node"; import {ChainConfig, ChainForkConfig} from "@lodestar/config"; import {Checkpoint} from "@lodestar/types/phase0"; import {Slot} from "@lodestar/types"; -import {fromHex, callFnWhenAwait, Logger} from "@lodestar/utils"; -import {BeaconStateAllForks, getLatestBlockRoot, computeCheckpointEpochAtStateSlot} from "@lodestar/state-transition"; +import {fromHex, callFnWhenAwait, Logger, formatBytes} from "@lodestar/utils"; +import { + BeaconStateAllForks, + getLatestBlockRoot, + computeCheckpointEpochAtStateSlot, + loadState, +} from "@lodestar/state-transition"; import {parseBootnodesFile} from "../util/format.js"; import * as mainnet from "./mainnet.js"; import * as dev from "./dev.js"; @@ -140,8 +145,12 @@ export function readBootnodes(bootnodesFilePath: string): string[] { export async function fetchWeakSubjectivityState( config: ChainForkConfig, logger: Logger, - {checkpointSyncUrl, wssCheckpoint}: {checkpointSyncUrl: string; wssCheckpoint?: string} -): Promise<{wsState: BeaconStateAllForks; wsCheckpoint: Checkpoint}> { + {checkpointSyncUrl, wssCheckpoint}: {checkpointSyncUrl: string; wssCheckpoint?: string}, + { + lastDbState, + lastDbValidatorsBytes, + }: {lastDbState: BeaconStateAllForks | null; lastDbValidatorsBytes: Uint8Array | null} +): Promise<{wsState: BeaconStateAllForks; wsStateBytes: Uint8Array; wsCheckpoint: Checkpoint}> { try { let wsCheckpoint: Checkpoint | null; let stateId: Slot | "finalized"; @@ -169,7 +178,7 @@ export async function fetchWeakSubjectivityState( } ); - const stateBytes = await callFnWhenAwait( + const wsStateBytes = await callFnWhenAwait( getStatePromise, () => logger.info("Download in progress, please wait..."), GET_STATE_LOG_INTERVAL @@ -177,13 +186,23 @@ export async function fetchWeakSubjectivityState( return res.ssz(); }); - logger.info("Download completed", {stateId}); + const wsSlot = getStateSlotFromBytes(wsStateBytes); + const logData = {stateId, size: formatBytes(wsStateBytes.length)}; + logger.info("Download completed", typeof stateId === "number" ? logData : {...logData, slot: wsSlot}); // It should not be required to get fork type from bytes but Checkpointz does not return // Eth-Consensus-Version header, see https://github.com/ethpandaops/checkpointz/issues/164 - const wsState = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes); + let wsState: BeaconStateAllForks; + if (lastDbState && lastDbValidatorsBytes) { + // use lastDbState to load wsState if possible to share the same state tree + wsState = loadState(config, lastDbState, wsStateBytes, lastDbValidatorsBytes).state; + } else { + const stateType = config.getForkTypes(wsSlot).BeaconState; + wsState = stateType.deserializeToViewDU(wsStateBytes); + } return { wsState, + wsStateBytes, wsCheckpoint: wsCheckpoint ?? getCheckpointFromState(wsState), }; } catch (e) { diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index 6a839fbe103d..9b9916f1d49e 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -25,3 +25,4 @@ export * from "./validator.js"; export * from "./weakSubjectivity.js"; export * from "./deposit.js"; export * from "./electra.js"; +export * from "./loadState/index.js"; diff --git a/packages/state-transition/src/util/loadState/index.ts b/packages/state-transition/src/util/loadState/index.ts index 706de3c11540..78ffe7877c09 100644 --- a/packages/state-transition/src/util/loadState/index.ts +++ b/packages/state-transition/src/util/loadState/index.ts @@ -1 +1 @@ -export {loadState} from "./loadState.js"; +export {loadState, loadStateAndValidators} from "./loadState.js"; diff --git a/packages/state-transition/src/util/loadState/loadState.ts b/packages/state-transition/src/util/loadState/loadState.ts index dc9f8fe4fcab..6e3e9c6719fa 100644 --- a/packages/state-transition/src/util/loadState/loadState.ts +++ b/packages/state-transition/src/util/loadState/loadState.ts @@ -66,6 +66,25 @@ export function loadState( return {state: migratedState, modifiedValidators}; } +/** + * Load state and validators Uint8Array from state bytes. + */ +export function loadStateAndValidators( + chainForkConfig: ChainForkConfig, + stateBytes: Uint8Array +): {state: BeaconStateAllForks; validatorsBytes: Uint8Array} { + // stateType could be any types, casting just to make typescript happy + const stateType = getStateTypeFromBytes(chainForkConfig, stateBytes) as typeof ssz.phase0.BeaconState; + const state = stateType.deserializeToViewDU(stateBytes); + const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength); + const fieldRanges = stateType.getFieldRanges(dataView, 0, stateBytes.length); + const allFields = Object.keys(stateType.fields); + const validatorFieldIndex = allFields.indexOf("validators"); + const validatorRange = fieldRanges[validatorFieldIndex]; + const validatorsBytes = stateBytes.subarray(validatorRange.start, validatorRange.end); + return {state, validatorsBytes}; +} + /** * This value is rarely changed as monitored 3 month state diffs on mainnet as of Sep 2023. * Reusing this data helps save hashTreeRoot time of state ~500ms diff --git a/packages/state-transition/test/unit/util/loadState.test.ts b/packages/state-transition/test/unit/util/loadState.test.ts new file mode 100644 index 000000000000..97a792a28adb --- /dev/null +++ b/packages/state-transition/test/unit/util/loadState.test.ts @@ -0,0 +1,40 @@ +import {describe, it, expect} from "vitest"; +import {ssz} from "@lodestar/types"; +import {mainnetChainConfig} from "@lodestar/config/networks"; +import {createChainForkConfig} from "@lodestar/config"; +import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {loadStateAndValidators} from "../../../src/util/loadState/loadState.js"; + +describe("loadStateAndValidators", () => { + const numValidator = 10; + const config = createChainForkConfig(mainnetChainConfig); + + const testCases: {name: ForkName; slot: number}[] = [ + {name: ForkName.phase0, slot: 100}, + {name: ForkName.altair, slot: mainnetChainConfig.ALTAIR_FORK_EPOCH * SLOTS_PER_EPOCH + 100}, + {name: ForkName.capella, slot: mainnetChainConfig.CAPELLA_FORK_EPOCH * SLOTS_PER_EPOCH + 100}, + {name: ForkName.deneb, slot: mainnetChainConfig.DENEB_FORK_EPOCH * SLOTS_PER_EPOCH + 100}, + ]; + + for (const {name, slot} of testCases) { + it(`fork: ${name}, slot: ${slot}`, () => { + const state = config.getForkTypes(slot).BeaconState.defaultViewDU(); + state.slot = slot; + for (let i = 0; i < numValidator; i++) { + const validator = ssz.phase0.Validator.defaultViewDU(); + validator.pubkey = Buffer.alloc(48, i); + state.validators.push(validator); + state.balances.push(32 * 1e9); + } + state.commit(); + + const stateBytes = state.serialize(); + const stateRoot = state.hashTreeRoot(); + const {state: loadedState, validatorsBytes} = loadStateAndValidators(config, stateBytes); + expect(loadedState.hashTreeRoot()).toEqual(stateRoot); + // serialize() somehow takes time, however comparing state root would be enough + // expect(loadedState.serialize()).toEqual(stateBytes); + expect(validatorsBytes).toEqual(state.validators.serialize()); + }); + } +}); diff --git a/packages/utils/src/bytes.ts b/packages/utils/src/bytes.ts index 6670b115ff8f..95bb62ebd548 100644 --- a/packages/utils/src/bytes.ts +++ b/packages/utils/src/bytes.ts @@ -48,3 +48,23 @@ export function bytesToBigInt(value: Uint8Array, endianness: Endianness = "le"): } throw new Error("endianness must be either 'le' or 'be'"); } + +export function formatBytes(bytes: number): string { + if (bytes < 0) { + throw new Error("bytes must be a positive number, got " + bytes); + } + + if (bytes === 0) { + return "0 Bytes"; + } + + // size of a kb + const k = 1024; + + // only support up to GB + const units = ["Bytes", "KB", "MB", "GB"]; + const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), units.length - 1); + const formattedSize = (bytes / Math.pow(k, i)).toFixed(2); + + return `${formattedSize} ${units[i]}`; +} diff --git a/packages/utils/test/unit/bytes.test.ts b/packages/utils/test/unit/bytes.test.ts index 05789b839cfd..af4df6652f13 100644 --- a/packages/utils/test/unit/bytes.test.ts +++ b/packages/utils/test/unit/bytes.test.ts @@ -1,5 +1,14 @@ import {describe, it, expect} from "vitest"; -import {intToBytes, bytesToInt, toHex, fromHex, toHexString, toRootHex, toPubkeyHex} from "../../src/index.js"; +import { + intToBytes, + bytesToInt, + toHex, + fromHex, + toHexString, + toRootHex, + toPubkeyHex, + formatBytes, +} from "../../src/index.js"; describe("intToBytes", () => { const zeroedArray = (length: number): number[] => Array.from({length}, () => 0); @@ -135,3 +144,24 @@ describe("toHexString", () => { }); } }); + +describe("formatBytes", () => { + const testCases: {input: number; output: string}[] = [ + {input: 0, output: "0 Bytes"}, + {input: 1, output: "1.00 Bytes"}, + {input: 1024, output: "1.00 KB"}, + {input: 1024 + 0.12 * 1024, output: "1.12 KB"}, + {input: 1024 * 1024, output: "1.00 MB"}, + {input: 1024 * 1024 + 0.12 * (1024 * 1024), output: "1.12 MB"}, + {input: 1024 * 1024 * 1024, output: "1.00 GB"}, + {input: 1024 * 1024 * 1024 + 0.12 * 1024 * 1024 * 1024, output: "1.12 GB"}, + // too big + {input: 1024 * 1024 * 1024 * 1024, output: "1024.00 GB"}, + ]; + + for (const {input, output} of testCases) { + it(`should format ${input} bytes as ${output}`, () => { + expect(formatBytes(input)).toBe(output); + }); + } +}); From 94288e9f2d6d7f6d49b0a58c7dfdc7495de7849f Mon Sep 17 00:00:00 2001 From: Phil Ngo Date: Tue, 10 Sep 2024 12:47:50 -0400 Subject: [PATCH 107/145] chore: bump package versions to 1.22.0 --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 6 +++--- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 +++++++------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 12 ++++++------ packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 6 +++--- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 99 insertions(+), 99 deletions(-) diff --git a/lerna.json b/lerna.json index 043a5c48569a..0ffc65fe5402 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.21.0", + "version": "1.22.0", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 138964ed946a..d2f8a621f02e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -72,10 +72,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index b41450f12d34..ade89ebc6cc1 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -119,18 +119,18 @@ "@libp2p/peer-id-factory": "^4.1.0", "@libp2p/prometheus-metrics": "^3.0.21", "@libp2p/tcp": "9.0.23", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/db": "^1.21.0", - "@lodestar/fork-choice": "^1.21.0", - "@lodestar/light-client": "^1.21.0", - "@lodestar/logger": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/reqresp": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", - "@lodestar/validator": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/db": "^1.22.0", + "@lodestar/fork-choice": "^1.22.0", + "@lodestar/light-client": "^1.22.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/reqresp": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", + "@lodestar/validator": "^1.22.0", "@multiformats/multiaddr": "^12.1.3", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index bf8621344dc5..579725eba04e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.21.0", + "version": "1.22.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -62,17 +62,17 @@ "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", "@libp2p/peer-id-factory": "^4.1.0", - "@lodestar/api": "^1.21.0", - "@lodestar/beacon-node": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/db": "^1.21.0", - "@lodestar/light-client": "^1.21.0", - "@lodestar/logger": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", - "@lodestar/validator": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/beacon-node": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/db": "^1.22.0", + "@lodestar/light-client": "^1.22.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", + "@lodestar/validator": "^1.22.0", "@multiformats/multiaddr": "^12.1.3", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -88,7 +88,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "@types/debug": "^4.1.7", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", diff --git a/packages/config/package.json b/packages/config/package.json index 45c462e19081..adf21542a890 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.21.0", + "version": "1.22.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,7 +65,7 @@ ], "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0" + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0" } } diff --git a/packages/db/package.json b/packages/db/package.json index 1a31603b6791..0e3a2c847d4b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.21.0", + "version": "1.22.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -36,12 +36,12 @@ }, "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/utils": "^1.22.0", "classic-level": "^1.4.1", "it-all": "^3.0.4" }, "devDependencies": { - "@lodestar/logger": "^1.21.0" + "@lodestar/logger": "^1.22.0" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index 461147274591..3a11e9b49ebf 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.21.0", + "version": "1.22.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/blst": "^2.0.3", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index dfe4c942832d..728036b365d3 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -37,11 +37,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0" + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 1188732c8870..940a78ae66d5 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -77,11 +77,11 @@ "@chainsafe/blst": "^0.2.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "mitt": "^3.0.0" }, "devDependencies": { diff --git a/packages/logger/package.json b/packages/logger/package.json index 1ab176dbfd93..196fb306a8b0 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.21.0", + "@lodestar/utils": "^1.22.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "@types/triple-beam": "^1.3.2", "triple-beam": "^1.3.0" }, diff --git a/packages/params/package.json b/packages/params/package.json index c246918959e4..c281f97a41ec 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.21.0", + "version": "1.22.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index 974125e2a17f..6a31b5b1baa0 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/light-client": "^1.21.0", - "@lodestar/logger": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/light-client": "^1.22.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "ethereum-cryptography": "^2.0.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 1b9d99b000a8..44dcb8048f99 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -54,9 +54,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.3.0", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/utils": "^1.22.0", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -65,8 +65,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.21.0", - "@lodestar/types": "^1.21.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/types": "^1.22.0", "libp2p": "1.4.3" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 166df67a7409..b4aabf0140e8 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.21.0", + "version": "1.22.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,7 +62,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.21.0", + "@lodestar/utils": "^1.22.0", "axios": "^1.3.4", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 1dd43ef98f88..2df87a522a63 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -63,10 +63,10 @@ "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "bigint-buffer": "^1.1.5", "immutable": "^4.3.2" }, diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 6c8c9e62894f..5bf61891a9d5 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.21.0", + "version": "1.22.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -59,8 +59,8 @@ "dependencies": { "@chainsafe/bls-keystore": "^3.1.0", "@chainsafe/blst": "^2.0.3", - "@lodestar/params": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/params": "^1.22.0", + "@lodestar/utils": "^1.22.0", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index 861dbdbee0ef..f3e034ec1b35 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -74,7 +74,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/params": "^1.21.0", + "@lodestar/params": "^1.22.0", "ethereum-cryptography": "^2.0.0" }, "keywords": [ diff --git a/packages/utils/package.json b/packages/utils/package.json index 492c65606dfb..0723f5f1815c 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index 373e59817d2a..65ed656f010d 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.21.0", + "version": "1.22.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -47,17 +47,17 @@ "dependencies": { "@chainsafe/blst": "^2.0.3", "@chainsafe/ssz": "^0.17.1", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/db": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/db": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From 9f4bf50408fe51cf783a475ab6f0d2db40850753 Mon Sep 17 00:00:00 2001 From: twoeths Date: Thu, 12 Sep 2024 08:34:15 +0700 Subject: [PATCH 108/145] fix: avoid toHexString() (#7075) * fix: avoid toHexString() * fix: use toRootHex and toPubkeyHex where applicable --- packages/api/src/beacon/routes/proof.ts | 7 ++++--- packages/api/src/beacon/routes/validator.ts | 15 ++++++++------- packages/api/src/builder/routes.ts | 6 +++--- packages/api/src/utils/serdes.ts | 5 +++-- .../beacon-node/src/chain/blocks/importBlock.ts | 7 +++---- packages/beacon-node/src/chain/opPools/opPool.ts | 6 +++--- .../src/chain/stateCache/datastore/file.ts | 9 +++++---- .../stateCache/persistentCheckpointsCache.ts | 12 ++++++------ .../beacon-node/src/eth1/provider/eth1Provider.ts | 5 ++--- packages/beacon-node/src/eth1/provider/utils.ts | 8 ++++---- packages/beacon-node/src/eth1/utils/eth1Vote.ts | 1 - .../network/peers/utils/assertPeerRelevance.ts | 5 ++--- .../cli/src/cmds/validator/keymanager/server.ts | 4 ++-- packages/config/package.json | 1 + packages/config/src/chainConfig/json.ts | 5 +++-- packages/config/src/genesisConfig/index.ts | 4 ++-- packages/fork-choice/src/forkChoice/forkChoice.ts | 5 ++--- packages/fork-choice/src/protoArray/protoArray.ts | 4 ++-- packages/light-client/src/index.ts | 6 +++--- packages/logger/src/utils/json.ts | 6 +++--- .../src/block/processBlsToExecutionChange.ts | 7 +++---- .../src/block/processExecutionPayload.ts | 9 ++++----- .../src/block/processWithdrawalRequest.ts | 6 +++--- packages/utils/src/format.ts | 8 ++++---- packages/validator/src/services/attestation.ts | 5 ++--- packages/validator/src/services/validatorStore.ts | 14 +++++++------- .../interchange/formats/completeV4.ts | 6 +++--- .../slashingProtection/interchange/formats/v5.ts | 6 +++--- .../validator/src/slashingProtection/utils.ts | 7 ++++--- packages/validator/src/util/difference.ts | 6 +++--- .../validator/src/util/externalSignerClient.ts | 11 ++++++----- packages/validator/src/validator.ts | 9 ++++----- 32 files changed, 107 insertions(+), 108 deletions(-) diff --git a/packages/api/src/beacon/routes/proof.ts b/packages/api/src/beacon/routes/proof.ts index 5c20a0194fc5..a48caf02c9e7 100644 --- a/packages/api/src/beacon/routes/proof.ts +++ b/packages/api/src/beacon/routes/proof.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {CompactMultiProof, ProofType} from "@chainsafe/persistent-merkle-tree"; -import {ByteListType, ContainerType, fromHexString, toHexString} from "@chainsafe/ssz"; +import {ByteListType, ContainerType, fromHexString} from "@chainsafe/ssz"; +import {toHex} from "@lodestar/utils"; import {ChainForkConfig} from "@lodestar/config"; import {ssz} from "@lodestar/types"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; @@ -45,7 +46,7 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({params: {state_id: stateId}, query: {format: toHexString(descriptor)}}), + writeReq: ({stateId, descriptor}) => ({params: {state_id: stateId}, query: {format: toHex(descriptor)}}), parseReq: ({params, query}) => ({stateId: params.state_id, descriptor: fromHexString(query.format)}), schema: {params: {state_id: Schema.StringRequired}, query: {format: Schema.StringRequired}}, }, @@ -63,7 +64,7 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({params: {block_id: blockId}, query: {format: toHexString(descriptor)}}), + writeReq: ({blockId, descriptor}) => ({params: {block_id: blockId}, query: {format: toHex(descriptor)}}), parseReq: ({params, query}) => ({blockId: params.block_id, descriptor: fromHexString(query.format)}), schema: {params: {block_id: Schema.StringRequired}, query: {format: Schema.StringRequired}}, }, diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 664caf44a2c4..3acbfec3e800 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {ContainerType, fromHexString, toHexString, Type, ValueOf} from "@chainsafe/ssz"; +import {ContainerType, fromHexString, Type, ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {isForkBlobs, isForkPostElectra} from "@lodestar/params"; import { @@ -20,6 +20,7 @@ import { Attestation, sszTypesFor, } from "@lodestar/types"; +import {toHex, toRootHex} from "@lodestar/utils"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; import {fromGraffitiHex, toBoolean, toGraffitiHex} from "../../utils/serdes.js"; import {getExecutionForkTypes, toForkName} from "../../utils/fork.js"; @@ -623,7 +624,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ params: {slot}, query: { - randao_reveal: toHexString(randaoReveal), + randao_reveal: toHex(randaoReveal), graffiti: toGraffitiHex(graffiti), fee_recipient: feeRecipient, builder_selection: builderSelection, @@ -674,7 +675,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ params: {slot}, query: { - randao_reveal: toHexString(randaoReveal), + randao_reveal: toHex(randaoReveal), graffiti: toGraffitiHex(graffiti), skip_randao_verification: writeSkipRandaoVerification(skipRandaoVerification), fee_recipient: feeRecipient, @@ -765,7 +766,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ params: {slot}, - query: {randao_reveal: toHexString(randaoReveal), graffiti: toGraffitiHex(graffiti)}, + query: {randao_reveal: toHex(randaoReveal), graffiti: toGraffitiHex(graffiti)}, }), parseReq: ({params, query}) => ({ slot: params.slot, @@ -805,7 +806,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - query: {slot, subcommittee_index: subcommitteeIndex, beacon_block_root: toHexString(beaconBlockRoot)}, + query: {slot, subcommittee_index: subcommitteeIndex, beacon_block_root: toRootHex(beaconBlockRoot)}, }), parseReq: ({query}) => ({ slot: query.slot, @@ -830,7 +831,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - query: {attestation_data_root: toHexString(attestationDataRoot), slot}, + query: {attestation_data_root: toRootHex(attestationDataRoot), slot}, }), parseReq: ({query}) => ({ attestationDataRoot: fromHexString(query.attestation_data_root), @@ -853,7 +854,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - query: {attestation_data_root: toHexString(attestationDataRoot), slot, committee_index: committeeIndex}, + query: {attestation_data_root: toHex(attestationDataRoot), slot, committee_index: committeeIndex}, }), parseReq: ({query}) => ({ attestationDataRoot: fromHexString(query.attestation_data_root), diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts index 3d74101bb046..ffed5a91824c 100644 --- a/packages/api/src/builder/routes.ts +++ b/packages/api/src/builder/routes.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import { ssz, bellatrix, @@ -13,7 +13,7 @@ import { } from "@lodestar/types"; import {ForkName, isForkBlobs} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; -import {toPubkeyHex} from "@lodestar/utils"; +import {toPubkeyHex, toRootHex} from "@lodestar/utils"; import {Endpoint, RouteDefinitions, Schema} from "../utils/index.js"; import {MetaHeader, VersionCodec, VersionMeta} from "../utils/metadata.js"; @@ -106,7 +106,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - params: {slot, parent_hash: toHexString(parentHash), pubkey: toPubkeyHex(proposerPubKey)}, + params: {slot, parent_hash: toRootHex(parentHash), pubkey: toPubkeyHex(proposerPubKey)}, }), parseReq: ({params}) => ({ slot: params.slot, diff --git a/packages/api/src/utils/serdes.ts b/packages/api/src/utils/serdes.ts index 233d7db9e7f8..8fc746ac8264 100644 --- a/packages/api/src/utils/serdes.ts +++ b/packages/api/src/utils/serdes.ts @@ -1,4 +1,5 @@ -import {fromHexString, JsonPath, toHexString} from "@chainsafe/ssz"; +import {fromHexString, JsonPath} from "@chainsafe/ssz"; +import {toHex} from "@lodestar/utils"; /** * Serialize proof path to JSON. @@ -82,7 +83,7 @@ export function toGraffitiHex(utf8?: string): string | undefined { return undefined; } - const hex = toHexString(new TextEncoder().encode(utf8)); + const hex = toHex(new TextEncoder().encode(utf8)); if (hex.length > GRAFFITI_HEX_LENGTH) { // remove characters from the end if hex string is too long diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index de5ecf607d95..4c46bdae53ee 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {capella, ssz, altair, BeaconBlock} from "@lodestar/types"; import {ForkLightClient, ForkSeq, INTERVALS_PER_SLOT, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; import { @@ -10,7 +9,7 @@ import { } from "@lodestar/state-transition"; import {routes} from "@lodestar/api"; import {ForkChoiceError, ForkChoiceErrorCode, EpochDifference, AncestorStatus} from "@lodestar/fork-choice"; -import {isErrorAborted, toRootHex} from "@lodestar/utils"; +import {isErrorAborted, toHex, toRootHex} from "@lodestar/utils"; import {ZERO_HASH_HEX} from "../../constants/index.js"; import {toCheckpointHex} from "../stateCache/index.js"; import {isOptimisticBlock} from "../../util/forkChoice.js"; @@ -433,8 +432,8 @@ export async function importBlock( blockRoot: blockRootHex, slot: blockSlot, index, - kzgCommitment: toHexString(kzgCommitment), - versionedHash: toHexString(kzgCommitmentToVersionedHash(kzgCommitment)), + kzgCommitment: toHex(kzgCommitment), + versionedHash: toHex(kzgCommitmentToVersionedHash(kzgCommitment)), }); } } diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index a2180a718fee..dc3fee0ac8a8 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -1,4 +1,4 @@ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAllForks, computeEpochAtSlot, @@ -16,7 +16,7 @@ import { ForkSeq, MAX_ATTESTER_SLASHINGS_ELECTRA, } from "@lodestar/params"; -import {toRootHex} from "@lodestar/utils"; +import {toHex, toRootHex} from "@lodestar/utils"; import {Epoch, phase0, capella, ssz, ValidatorIndex, SignedBeaconBlock, AttesterSlashing} from "@lodestar/types"; import {IBeaconDb} from "../../db/index.js"; import {SignedBLSToExecutionChangeVersioned} from "../../util/types.js"; @@ -88,7 +88,7 @@ export class OpPool { key: fromHexString(key), value: value.attesterSlashing, })), - toHexString + toHex ), persistDiff( db.proposerSlashing, diff --git a/packages/beacon-node/src/chain/stateCache/datastore/file.ts b/packages/beacon-node/src/chain/stateCache/datastore/file.ts index 6529d12f84db..8a07df90d168 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/file.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/file.ts @@ -1,6 +1,7 @@ import path from "node:path"; -import {toHexString, fromHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {phase0, ssz} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js"; import {CPStateDatastore, DatastoreKey} from "./types.js"; @@ -28,18 +29,18 @@ export class FileCPStateDatastore implements CPStateDatastore { async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise { const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey); - const filePath = path.join(this.folderPath, toHexString(serializedCheckpoint)); + const filePath = path.join(this.folderPath, toHex(serializedCheckpoint)); await writeIfNotExist(filePath, stateBytes); return serializedCheckpoint; } async remove(serializedCheckpoint: DatastoreKey): Promise { - const filePath = path.join(this.folderPath, toHexString(serializedCheckpoint)); + const filePath = path.join(this.folderPath, toHex(serializedCheckpoint)); await removeFile(filePath); } async read(serializedCheckpoint: DatastoreKey): Promise { - const filePath = path.join(this.folderPath, toHexString(serializedCheckpoint)); + const filePath = path.join(this.folderPath, toHex(serializedCheckpoint)); return readFile(filePath); } diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 190b79e58cd6..c919c3c86233 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -1,7 +1,7 @@ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {phase0, Epoch, RootHex} from "@lodestar/types"; import {CachedBeaconStateAllForks, computeStartSlotAtEpoch, getBlockRootAtSlot} from "@lodestar/state-transition"; -import {Logger, MapDef, sleep, toRootHex} from "@lodestar/utils"; +import {Logger, MapDef, sleep, toHex, toRootHex} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {loadCachedBeaconState} from "@lodestar/state-transition"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; @@ -193,7 +193,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { return stateOrStateBytesData?.clone(opts?.dontTransferCache) ?? null; } const {persistedKey, stateBytes} = stateOrStateBytesData; - const logMeta = {persistedKey: toHexString(persistedKey)}; + const logMeta = {persistedKey: toHex(persistedKey)}; this.logger.debug("Reload: read state successful", logMeta); this.metrics?.stateReloadSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0); const seedState = this.findSeedStateToReload(cp); @@ -341,7 +341,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.logger.verbose("Added checkpoint state to memory but a persisted key existed", { epoch: cp.epoch, rootHex: cpHex.rootHex, - persistedKey: toHexString(persistedKey), + persistedKey: toHex(persistedKey), }); } else { this.cache.set(key, {type: CacheItemType.inMemory, state}); @@ -688,7 +688,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { stateSlot: state.slot, rootHex, epochBoundaryHex, - persistedKey: persistedKey ? toHexString(persistedKey) : "", + persistedKey: persistedKey ? toHex(persistedKey) : "", }; if (persistedRootHexes.has(rootHex)) { @@ -716,7 +716,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { persistCount++; this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", { ...logMeta, - persistedKey: toHexString(persistedKey), + persistedKey: toHex(persistedKey), }); } // overwrite cpKey, this means the state is deleted from memory diff --git a/packages/beacon-node/src/eth1/provider/eth1Provider.ts b/packages/beacon-node/src/eth1/provider/eth1Provider.ts index 3af909cd132e..c86991eb7d15 100644 --- a/packages/beacon-node/src/eth1/provider/eth1Provider.ts +++ b/packages/beacon-node/src/eth1/provider/eth1Provider.ts @@ -1,7 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {phase0} from "@lodestar/types"; import {ChainConfig} from "@lodestar/config"; -import {fromHex, isErrorAborted, createElapsedTimeTracker, toPrintableUrl} from "@lodestar/utils"; +import {fromHex, isErrorAborted, createElapsedTimeTracker, toPrintableUrl, toHex} from "@lodestar/utils"; import {Logger} from "@lodestar/logger"; import {FetchError, isFetchError} from "@lodestar/api"; @@ -72,7 +71,7 @@ export class Eth1Provider implements IEth1Provider { ) { this.logger = opts.logger; this.deployBlock = opts.depositContractDeployBlock ?? 0; - this.depositContractAddress = toHexString(config.DEPOSIT_CONTRACT_ADDRESS); + this.depositContractAddress = toHex(config.DEPOSIT_CONTRACT_ADDRESS); const providerUrls = opts.providerUrls ?? DEFAULT_PROVIDER_URLS; this.rpc = new JsonRpcHttpClient(providerUrls, { diff --git a/packages/beacon-node/src/eth1/provider/utils.ts b/packages/beacon-node/src/eth1/provider/utils.ts index 506e4e48711a..cddf48c40d5a 100644 --- a/packages/beacon-node/src/eth1/provider/utils.ts +++ b/packages/beacon-node/src/eth1/provider/utils.ts @@ -1,6 +1,6 @@ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {RootHex} from "@lodestar/types"; -import {bytesToBigInt, bigIntToBytes} from "@lodestar/utils"; +import {bytesToBigInt, bigIntToBytes, toHex} from "@lodestar/utils"; import {ErrorParseJson} from "./jsonRpcHttpClient.js"; /** QUANTITY as defined in ethereum execution layer JSON RPC https://eth.wiki/json-rpc/API */ @@ -32,7 +32,7 @@ export function bytesToHex(bytes: Uint8Array): string { return "0x" + bytes[0].toString(16); } - return toHexString(bytes); + return toHex(bytes); } /** @@ -100,7 +100,7 @@ export function bytesToQuantity(bytes: Uint8Array): QUANTITY { * - WRONG: 004200 (must be prefixed 0x) */ export function bytesToData(bytes: Uint8Array): DATA { - return toHexString(bytes); + return toHex(bytes); } /** diff --git a/packages/beacon-node/src/eth1/utils/eth1Vote.ts b/packages/beacon-node/src/eth1/utils/eth1Vote.ts index cdc8fa04163e..7a4e3ddca9b6 100644 --- a/packages/beacon-node/src/eth1/utils/eth1Vote.ts +++ b/packages/beacon-node/src/eth1/utils/eth1Vote.ts @@ -120,7 +120,6 @@ function getKeysWithMaxValue(map: Map): T[] { * ✓ pickEth1Vote - max votes 37.89912 ops/s 26.38583 ms/op - 29 runs 1.27 s */ function getEth1DataKey(eth1Data: phase0.Eth1Data): string { - // return toHexString(ssz.phase0.Eth1Data.hashTreeRoot(eth1Data)); return fastSerializeEth1Data(eth1Data); } diff --git a/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts b/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts index 96e35e9d317d..e588b1ae0308 100644 --- a/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts +++ b/packages/beacon-node/src/network/peers/utils/assertPeerRelevance.ts @@ -1,6 +1,5 @@ -import {toHexString} from "@chainsafe/ssz"; import {ForkDigest, Root, Slot, phase0, ssz} from "@lodestar/types"; -import {toRootHex} from "@lodestar/utils"; +import {toHex, toRootHex} from "@lodestar/utils"; // TODO: Why this value? (From Lighthouse) const FUTURE_SLOT_TOLERANCE = 1; @@ -79,7 +78,7 @@ export function isZeroRoot(root: Root): boolean { export function renderIrrelevantPeerType(type: IrrelevantPeerType): string { switch (type.code) { case IrrelevantPeerCode.INCOMPATIBLE_FORKS: - return `INCOMPATIBLE_FORKS ours: ${toHexString(type.ours)} theirs: ${toHexString(type.theirs)}`; + return `INCOMPATIBLE_FORKS ours: ${toHex(type.ours)} theirs: ${toHex(type.theirs)}`; case IrrelevantPeerCode.DIFFERENT_CLOCKS: return `DIFFERENT_CLOCKS slotDiff: ${type.slotDiff}`; case IrrelevantPeerCode.DIFFERENT_FINALIZED: diff --git a/packages/cli/src/cmds/validator/keymanager/server.ts b/packages/cli/src/cmds/validator/keymanager/server.ts index 03880c8b8842..cca1a1f3b76b 100644 --- a/packages/cli/src/cmds/validator/keymanager/server.ts +++ b/packages/cli/src/cmds/validator/keymanager/server.ts @@ -1,10 +1,10 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import {toHexString} from "@chainsafe/ssz"; import {RestApiServer, RestApiServerOpts, RestApiServerModules} from "@lodestar/beacon-node"; import {KeymanagerApiMethods, registerRoutes} from "@lodestar/api/keymanager/server"; import {ChainForkConfig} from "@lodestar/config"; +import {toHex} from "@lodestar/utils"; import {writeFile600Perm} from "../../../util/index.js"; export type KeymanagerRestApiServerOpts = RestApiServerOpts & { @@ -50,7 +50,7 @@ export class KeymanagerRestApiServer extends RestApiServer { if (opts.isAuthEnabled) { // Generate a new token if token file does not exist or file do exist, but is empty - bearerToken = readFileIfExists(apiTokenPath) ?? `api-token-${toHexString(crypto.randomBytes(32))}`; + bearerToken = readFileIfExists(apiTokenPath) ?? `api-token-${toHex(crypto.randomBytes(32))}`; writeFile600Perm(apiTokenPath, bearerToken, {encoding: "utf8"}); } diff --git a/packages/config/package.json b/packages/config/package.json index 45c462e19081..d13713d376c6 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -66,6 +66,7 @@ "dependencies": { "@chainsafe/ssz": "^0.17.1", "@lodestar/params": "^1.21.0", + "@lodestar/utils": "^1.21.0", "@lodestar/types": "^1.21.0" } } diff --git a/packages/config/src/chainConfig/json.ts b/packages/config/src/chainConfig/json.ts index 4e61333cbdee..887409cd4c5b 100644 --- a/packages/config/src/chainConfig/json.ts +++ b/packages/config/src/chainConfig/json.ts @@ -1,4 +1,5 @@ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; +import {toHex} from "@lodestar/utils"; import {ChainConfig, chainConfigTypes, SpecValue, SpecValueTypeName} from "./types.js"; const MAX_UINT64_JSON = "18446744073709551615"; @@ -69,7 +70,7 @@ export function serializeSpecValue(value: SpecValue, typeName: SpecValueTypeName if (!(value instanceof Uint8Array)) { throw Error(`Invalid value ${value.toString()} expected Uint8Array`); } - return toHexString(value); + return toHex(value); case "string": if (typeof value !== "string") { diff --git a/packages/config/src/genesisConfig/index.ts b/packages/config/src/genesisConfig/index.ts index 52fdd03880a6..d2dfae2a8e08 100644 --- a/packages/config/src/genesisConfig/index.ts +++ b/packages/config/src/genesisConfig/index.ts @@ -1,6 +1,6 @@ -import {toHexString} from "@chainsafe/ssz"; import {ForkName, SLOTS_PER_EPOCH, DOMAIN_VOLUNTARY_EXIT} from "@lodestar/params"; import {DomainType, ForkDigest, phase0, Root, Slot, ssz, Version} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; import {ChainForkConfig} from "../beaconConfig.js"; import {ForkDigestHex, CachedGenesis} from "./types.js"; export type {ForkDigestContext} from "./types.js"; @@ -139,7 +139,7 @@ function computeForkDataRoot(currentVersion: Version, genesisValidatorsRoot: Roo } function toHexStringNoPrefix(hex: string | Uint8Array): string { - return strip0xPrefix(typeof hex === "string" ? hex : toHexString(hex)); + return strip0xPrefix(typeof hex === "string" ? hex : toHex(hex)); } function strip0xPrefix(hex: string): string { diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 2405a442c8cd..6ae3f52e7d3a 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {SLOTS_PER_HISTORICAL_ROOT, SLOTS_PER_EPOCH, INTERVALS_PER_SLOT} from "@lodestar/params"; import {bellatrix, Slot, ValidatorIndex, phase0, ssz, RootHex, Epoch, Root, BeaconBlock} from "@lodestar/types"; @@ -643,7 +642,7 @@ export class ForkChoice implements IForkChoice { ...(isExecutionBlockBodyType(block.body) && isExecutionStateType(state) && isExecutionEnabled(state, block) ? { - executionPayloadBlockHash: toHexString(block.body.executionPayload.blockHash), + executionPayloadBlockHash: toRootHex(block.body.executionPayload.blockHash), executionPayloadNumber: block.body.executionPayload.blockNumber, executionStatus: this.getPostMergeExecStatus(executionStatus), dataAvailabilityStatus, @@ -1484,7 +1483,7 @@ export function assertValidTerminalPowBlock( // powBock.blockHash is hex, so we just pick the corresponding root if (!ssz.Root.equals(block.body.executionPayload.parentHash, config.TERMINAL_BLOCK_HASH)) throw new Error( - `Invalid terminal block hash, expected: ${toHexString(config.TERMINAL_BLOCK_HASH)}, actual: ${toHexString( + `Invalid terminal block hash, expected: ${toRootHex(config.TERMINAL_BLOCK_HASH)}, actual: ${toRootHex( block.body.executionPayload.parentHash )}` ); diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index eaa86b2f0ee1..0b793d2be099 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -1,8 +1,8 @@ -import {toHexString} from "@chainsafe/ssz"; import {Epoch, RootHex, Slot} from "@lodestar/types"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {GENESIS_EPOCH} from "@lodestar/params"; +import {toRootHex} from "@lodestar/utils"; import {ForkChoiceError, ForkChoiceErrorCode} from "../forkChoice/errors.js"; import {ProtoBlock, ProtoNode, HEX_ZERO_HASH, ExecutionStatus, LVHExecResponse} from "./interface.js"; import {ProtoArrayError, ProtoArrayErrorCode, LVHExecError, LVHExecErrorCode} from "./errors.js"; @@ -10,7 +10,7 @@ import {ProtoArrayError, ProtoArrayErrorCode, LVHExecError, LVHExecErrorCode} fr export const DEFAULT_PRUNE_THRESHOLD = 0; type ProposerBoost = {root: RootHex; score: number}; -const ZERO_HASH_HEX = toHexString(Buffer.alloc(32, 0)); +const ZERO_HASH_HEX = toRootHex(Buffer.alloc(32, 0)); export class ProtoArray { // Do not attempt to prune the tree unless it has at least this many nodes. diff --git a/packages/light-client/src/index.ts b/packages/light-client/src/index.ts index 16ecf1adf939..03be60eca05d 100644 --- a/packages/light-client/src/index.ts +++ b/packages/light-client/src/index.ts @@ -1,5 +1,5 @@ import mitt from "mitt"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@lodestar/params"; import { LightClientBootstrap, @@ -13,7 +13,7 @@ import { SyncPeriod, } from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainForkConfig} from "@lodestar/config"; -import {isErrorAborted, sleep} from "@lodestar/utils"; +import {isErrorAborted, sleep, toRootHex} from "@lodestar/utils"; import {getCurrentSlot, slotWithFutureTolerance, timeUntilNextEpoch} from "./utils/clock.js"; import {chunkifyInclusiveRange} from "./utils/chunkify.js"; import {LightclientEmitter, LightclientEvent} from "./events.js"; @@ -162,7 +162,7 @@ export class Lightclient { const {transport, checkpointRoot} = args; // Fetch bootstrap state with proof at the trusted block root - const {data: bootstrap} = await transport.getBootstrap(toHexString(checkpointRoot)); + const {data: bootstrap} = await transport.getBootstrap(toRootHex(checkpointRoot)); validateLightClientBootstrap(args.config, checkpointRoot, bootstrap); diff --git a/packages/logger/src/utils/json.ts b/packages/logger/src/utils/json.ts index f6f2d85487c7..7408de582dd1 100644 --- a/packages/logger/src/utils/json.ts +++ b/packages/logger/src/utils/json.ts @@ -1,4 +1,4 @@ -import {LodestarError, mapValues, toHexString} from "@lodestar/utils"; +import {LodestarError, mapValues, toHex} from "@lodestar/utils"; const MAX_DEPTH = 0; @@ -29,7 +29,7 @@ export function logCtxToJson(arg: unknown, depth = 0, fromError = false): LogDat if (arg === null) return "null"; if (arg instanceof Uint8Array) { - return toHexString(arg); + return toHex(arg); } // For any type that may include recursiveness break early at the first level @@ -90,7 +90,7 @@ export function logCtxToString(arg: unknown, depth = 0, fromError = false): stri if (arg === null) return "null"; if (arg instanceof Uint8Array) { - return toHexString(arg); + return toHex(arg); } // For any type that may include recursiveness break early at the first level diff --git a/packages/state-transition/src/block/processBlsToExecutionChange.ts b/packages/state-transition/src/block/processBlsToExecutionChange.ts index 1cc3706a756f..be79f06f3f21 100644 --- a/packages/state-transition/src/block/processBlsToExecutionChange.ts +++ b/packages/state-transition/src/block/processBlsToExecutionChange.ts @@ -1,7 +1,8 @@ -import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {digest} from "@chainsafe/as-sha256"; import {capella} from "@lodestar/types"; import {BLS_WITHDRAWAL_PREFIX, ETH1_ADDRESS_WITHDRAWAL_PREFIX} from "@lodestar/params"; +import {toHex} from "@lodestar/utils"; import {verifyBlsToExecutionChangeSignature} from "../signatureSets/index.js"; import {CachedBeaconStateCapella} from "../types.js"; @@ -60,9 +61,7 @@ export function isValidBlsToExecutionChange( return { valid: false, error: Error( - `Invalid withdrawalCredentials expected=${toHexString(withdrawalCredentials)} actual=${toHexString( - digestCredentials - )}` + `Invalid withdrawalCredentials expected=${toHex(withdrawalCredentials)} actual=${toHex(digestCredentials)}` ), }; } diff --git a/packages/state-transition/src/block/processExecutionPayload.ts b/packages/state-transition/src/block/processExecutionPayload.ts index 3c28a400d3bf..3d70e46d40fd 100644 --- a/packages/state-transition/src/block/processExecutionPayload.ts +++ b/packages/state-transition/src/block/processExecutionPayload.ts @@ -1,6 +1,7 @@ -import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; +import {byteArrayEquals} from "@chainsafe/ssz"; import {BeaconBlockBody, BlindedBeaconBlockBody, deneb, isExecutionPayload} from "@lodestar/types"; import {ForkSeq, MAX_BLOBS_PER_BLOCK} from "@lodestar/params"; +import {toHex, toRootHex} from "@lodestar/utils"; import {CachedBeaconStateBellatrix, CachedBeaconStateCapella} from "../types.js"; import {getRandaoMix} from "../util/index.js"; import { @@ -23,7 +24,7 @@ export function processExecutionPayload( const {latestExecutionPayloadHeader} = state; if (!byteArrayEquals(payload.parentHash, latestExecutionPayloadHeader.blockHash)) { throw Error( - `Invalid execution payload parentHash ${toHexString(payload.parentHash)} latest blockHash ${toHexString( + `Invalid execution payload parentHash ${toRootHex(payload.parentHash)} latest blockHash ${toRootHex( latestExecutionPayloadHeader.blockHash )}` ); @@ -33,9 +34,7 @@ export function processExecutionPayload( // Verify random const expectedRandom = getRandaoMix(state, state.epochCtx.epoch); if (!byteArrayEquals(payload.prevRandao, expectedRandom)) { - throw Error( - `Invalid execution payload random ${toHexString(payload.prevRandao)} expected=${toHexString(expectedRandom)}` - ); + throw Error(`Invalid execution payload random ${toHex(payload.prevRandao)} expected=${toHex(expectedRandom)}`); } // Verify timestamp diff --git a/packages/state-transition/src/block/processWithdrawalRequest.ts b/packages/state-transition/src/block/processWithdrawalRequest.ts index cff1ea03bd80..0587a06ee179 100644 --- a/packages/state-transition/src/block/processWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processWithdrawalRequest.ts @@ -1,4 +1,3 @@ -import {toHexString} from "@chainsafe/ssz"; import {electra, phase0, ssz} from "@lodestar/types"; import { FAR_FUTURE_EPOCH, @@ -8,6 +7,7 @@ import { ForkSeq, } from "@lodestar/params"; +import {toHex} from "@lodestar/utils"; import {CachedBeaconStateElectra} from "../types.js"; import {hasCompoundingWithdrawalCredential, hasExecutionWithdrawalCredential} from "../util/electra.js"; import {getPendingBalanceToWithdraw, isActiveValidator} from "../util/validator.js"; @@ -85,8 +85,8 @@ function isValidatorEligibleForWithdrawOrExit( state: CachedBeaconStateElectra ): boolean { const {withdrawalCredentials} = validator; - const addressStr = toHexString(withdrawalCredentials.subarray(12)); - const sourceAddressStr = toHexString(sourceAddress); + const addressStr = toHex(withdrawalCredentials.subarray(12)); + const sourceAddressStr = toHex(sourceAddress); const {epoch: currentEpoch, config} = state.epochCtx; return ( diff --git a/packages/utils/src/format.ts b/packages/utils/src/format.ts index 8bd8a40273f1..5567eb89cc68 100644 --- a/packages/utils/src/format.ts +++ b/packages/utils/src/format.ts @@ -1,4 +1,4 @@ -import {toHexString} from "./bytes.js"; +import {toRootHex} from "./bytes/index.js"; import {ETH_TO_WEI} from "./ethConversion.js"; /** @@ -6,7 +6,7 @@ import {ETH_TO_WEI} from "./ethConversion.js"; * 4 bytes can represent 4294967296 values, so the chance of collision is low */ export function prettyBytes(root: Uint8Array | string): string { - const str = typeof root === "string" ? root : toHexString(root); + const str = typeof root === "string" ? root : toRootHex(root); return `${str.slice(0, 6)}…${str.slice(-4)}`; } @@ -15,7 +15,7 @@ export function prettyBytes(root: Uint8Array | string): string { * Paired with block numbers or slots, it can still act as a decent identify-able format */ export function prettyBytesShort(root: Uint8Array | string): string { - const str = typeof root === "string" ? root : toHexString(root); + const str = typeof root === "string" ? root : toRootHex(root); return `${str.slice(0, 6)}…`; } @@ -25,7 +25,7 @@ export function prettyBytesShort(root: Uint8Array | string): string { * values on explorers like beaconcha.in while improving readability of logs */ export function truncBytes(root: Uint8Array | string): string { - const str = typeof root === "string" ? root : toHexString(root); + const str = typeof root === "string" ? root : toRootHex(root); return str.slice(0, 14); } diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 927bd3d92bb4..7f0dffa3e970 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -1,8 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {BLSSignature, phase0, Slot, ssz, Attestation, SignedAggregateAndProof} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {computeEpochAtSlot, isAggregatorFromCommitteeLength} from "@lodestar/state-transition"; -import {prettyBytes, sleep} from "@lodestar/utils"; +import {prettyBytes, sleep, toRootHex} from "@lodestar/utils"; import {ApiClient, routes} from "@lodestar/api"; import {ChainForkConfig} from "@lodestar/config"; import {IClock, LoggerVc} from "../util/index.js"; @@ -195,7 +194,7 @@ export class AttestationService { duties: AttDutyAndProof[] ): Promise { const signedAttestations: Attestation[] = []; - const headRootHex = toHexString(attestationNoCommittee.beaconBlockRoot); + const headRootHex = toRootHex(attestationNoCommittee.beaconBlockRoot); const currentEpoch = computeEpochAtSlot(slot); const isPostElectra = currentEpoch >= this.config.ELECTRA_FORK_EPOCH; diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index c6130f1fab95..b64917ec03ae 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -1,4 +1,4 @@ -import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; +import {BitArray, fromHexString} from "@chainsafe/ssz"; import {SecretKey} from "@chainsafe/blst"; import { computeEpochAtSlot, @@ -42,7 +42,7 @@ import { SignedAggregateAndProof, } from "@lodestar/types"; import {routes} from "@lodestar/api"; -import {toPubkeyHex} from "@lodestar/utils"; +import {toPubkeyHex, toRootHex} from "@lodestar/utils"; import {ISlashingProtection} from "../slashingProtection/index.js"; import {PubkeyHex} from "../types.js"; import {externalSignerPostSignature, SignableMessageType, SignableMessage} from "../util/externalSignerClient.js"; @@ -459,8 +459,8 @@ export class ValidatorStore { logger?.debug("Signing the block proposal", { slot: signingSlot, - blockRoot: toHexString(blockRoot), - signingRoot: toHexString(signingRoot), + blockRoot: toRootHex(blockRoot), + signingRoot: toRootHex(signingRoot), }); try { @@ -748,7 +748,7 @@ export class ValidatorStore { signingSlot: Slot, signableMessage: SignableMessage ): Promise { - // TODO: Refactor indexing to not have to run toHexString() on the pubkey every time + // TODO: Refactor indexing to not have to run toHex() on the pubkey every time const pubkeyHex = typeof pubkey === "string" ? pubkey : toPubkeyHex(pubkey); const signer = this.validators.get(pubkeyHex)?.signer; @@ -787,7 +787,7 @@ export class ValidatorStore { } private getSignerAndPubkeyHex(pubkey: BLSPubkeyMaybeHex): [Signer, string] { - // TODO: Refactor indexing to not have to run toHexString() on the pubkey every time + // TODO: Refactor indexing to not have to run toHex() on the pubkey every time const pubkeyHex = typeof pubkey === "string" ? pubkey : toPubkeyHex(pubkey); const signer = this.validators.get(pubkeyHex)?.signer; if (!signer) { @@ -824,7 +824,7 @@ export class ValidatorStore { function getSignerPubkeyHex(signer: Signer): PubkeyHex { switch (signer.type) { case SignerType.Local: - return toHexString(signer.secretKey.toPublicKey().toBytes()); + return toPubkeyHex(signer.secretKey.toPublicKey().toBytes()); case SignerType.Remote: if (!isValidatePubkeyHex(signer.pubkey)) { diff --git a/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts b/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts index 66aa31c52194..d1e9475daa0e 100644 --- a/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts +++ b/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {toPubkeyHex} from "@lodestar/utils"; +import {fromHexString} from "@chainsafe/ssz"; +import {toPubkeyHex, toRootHex} from "@lodestar/utils"; import {InterchangeLodestar} from "../types.js"; import {fromOptionalHexString, numToString, toOptionalHexString} from "../../utils.js"; @@ -91,7 +91,7 @@ export function serializeInterchangeCompleteV4({ metadata: { interchange_format: "complete", interchange_format_version: "4", - genesis_validators_root: toHexString(genesisValidatorsRoot), + genesis_validators_root: toRootHex(genesisValidatorsRoot), }, data: data.map((validator) => ({ pubkey: toPubkeyHex(validator.pubkey), diff --git a/packages/validator/src/slashingProtection/interchange/formats/v5.ts b/packages/validator/src/slashingProtection/interchange/formats/v5.ts index 1c7f67b706a5..20d11aefe91a 100644 --- a/packages/validator/src/slashingProtection/interchange/formats/v5.ts +++ b/packages/validator/src/slashingProtection/interchange/formats/v5.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {toPubkeyHex} from "@lodestar/utils"; +import {fromHexString} from "@chainsafe/ssz"; +import {toPubkeyHex, toRootHex} from "@lodestar/utils"; import {InterchangeLodestar} from "../types.js"; import {fromOptionalHexString, numToString, toOptionalHexString} from "../../utils.js"; @@ -86,7 +86,7 @@ export function serializeInterchangeV5({data, genesisValidatorsRoot}: Interchang return { metadata: { interchange_format_version: "5", - genesis_validators_root: toHexString(genesisValidatorsRoot), + genesis_validators_root: toRootHex(genesisValidatorsRoot), }, data: data.map((validator) => ({ pubkey: toPubkeyHex(validator.pubkey), diff --git a/packages/validator/src/slashingProtection/utils.ts b/packages/validator/src/slashingProtection/utils.ts index f1fc0f24b9ae..294036981b14 100644 --- a/packages/validator/src/slashingProtection/utils.ts +++ b/packages/validator/src/slashingProtection/utils.ts @@ -1,5 +1,6 @@ -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {Epoch, Root, ssz} from "@lodestar/types"; +import {toHex, toRootHex} from "@lodestar/utils"; export const blsPubkeyLen = 48; export const ZERO_ROOT = ssz.Root.defaultValue(); @@ -17,7 +18,7 @@ export function fromOptionalHexString(hex: string | undefined): Root { } export function toOptionalHexString(root: Root): string | undefined { - return isEqualRoot(root, ZERO_ROOT) ? undefined : toHexString(root); + return isEqualRoot(root, ZERO_ROOT) ? undefined : toRootHex(root); } /** @@ -34,7 +35,7 @@ export function minEpoch(epochs: Epoch[]): Epoch | null { export function uniqueVectorArr(buffers: Uint8Array[]): Uint8Array[] { const bufferStr = new Set(); return buffers.filter((buffer) => { - const str = toHexString(buffer); + const str = toHex(buffer); const seen = bufferStr.has(str); bufferStr.add(str); return !seen; diff --git a/packages/validator/src/util/difference.ts b/packages/validator/src/util/difference.ts index bafc6ff8f42f..b8c3c5e7b089 100644 --- a/packages/validator/src/util/difference.ts +++ b/packages/validator/src/util/difference.ts @@ -1,10 +1,10 @@ -import {toHexString} from "@chainsafe/ssz"; import {Root} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; /** * Return items included in `next` but not in `prev` */ export function differenceHex(prev: T[], next: T[]): T[] { - const existing = new Set(prev.map((item) => toHexString(item))); - return next.filter((item) => !existing.has(toHexString(item))); + const existing = new Set(prev.map((item) => toHex(item))); + return next.filter((item) => !existing.has(toHex(item))); } diff --git a/packages/validator/src/util/externalSignerClient.ts b/packages/validator/src/util/externalSignerClient.ts index dc1d0d0f1dd5..4fd615ce5280 100644 --- a/packages/validator/src/util/externalSignerClient.ts +++ b/packages/validator/src/util/externalSignerClient.ts @@ -1,4 +1,4 @@ -import {ContainerType, toHexString, ValueOf} from "@chainsafe/ssz"; +import {ContainerType, ValueOf} from "@chainsafe/ssz"; import {fetch} from "@lodestar/api"; import {phase0, altair, capella, BeaconBlock, BlindedBeaconBlock} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; @@ -6,6 +6,7 @@ import {ValidatorRegistrationV1} from "@lodestar/types/bellatrix"; import {BeaconConfig} from "@lodestar/config"; import {computeEpochAtSlot, blindedOrFullBlockToHeader} from "@lodestar/state-transition"; import {Epoch, Root, RootHex, Slot, ssz} from "@lodestar/types"; +import {toHex, toRootHex} from "@lodestar/utils"; import {PubkeyHex} from "../types.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -131,17 +132,17 @@ export async function externalSignerPostSignature( const requestObj = serializerSignableMessagePayload(config, signableMessage) as Web3SignerSerializedRequest; requestObj.type = signableMessage.type; - requestObj.signingRoot = toHexString(signingRoot); + requestObj.signingRoot = toRootHex(signingRoot); if (requiresForkInfo[signableMessage.type]) { const forkInfo = config.getForkInfo(signingSlot); requestObj.fork_info = { fork: { - previous_version: toHexString(forkInfo.prevVersion), - current_version: toHexString(forkInfo.version), + previous_version: toHex(forkInfo.prevVersion), + current_version: toHex(forkInfo.version), epoch: String(computeEpochAtSlot(signingSlot)), }, - genesis_validators_root: toHexString(config.genesisValidatorsRoot), + genesis_validators_root: toRootHex(config.genesisValidatorsRoot), }; } diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 8ec8f3330af2..980e64f7eac2 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -1,8 +1,7 @@ -import {toHexString} from "@chainsafe/ssz"; import {BLSPubkey, phase0, ssz} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {Genesis} from "@lodestar/types/phase0"; -import {Logger, toPrintableUrl} from "@lodestar/utils"; +import {Logger, toPrintableUrl, toRootHex} from "@lodestar/utils"; import {getClient, ApiClient, routes, ApiRequestInit, defaultInit} from "@lodestar/api"; import {computeEpochAtSlot, getCurrentSlot} from "@lodestar/state-transition"; import {Clock, IClock} from "./util/clock.js"; @@ -397,14 +396,14 @@ async function assertEqualGenesis(opts: ValidatorOptions, genesis: Genesis): Pro if (!ssz.Root.equals(genesisValidatorsRoot, nodeGenesisValidatorRoot)) { // this happens when the existing validator db served another network before opts.logger.error("Not the same genesisValidatorRoot", { - expected: toHexString(nodeGenesisValidatorRoot), - actual: toHexString(genesisValidatorsRoot), + expected: toRootHex(nodeGenesisValidatorRoot), + actual: toRootHex(genesisValidatorsRoot), }); throw new NotEqualParamsError("Not the same genesisValidatorRoot"); } } else { await metaDataRepository.setGenesisValidatorsRoot(nodeGenesisValidatorRoot); - opts.logger.info("Persisted genesisValidatorRoot", toHexString(nodeGenesisValidatorRoot)); + opts.logger.info("Persisted genesisValidatorRoot", toRootHex(nodeGenesisValidatorRoot)); } const nodeGenesisTime = genesis.genesisTime; From d6e8c057d23fcb2e6f8ed1c13d27dbdc6a52e1d0 Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 13 Sep 2024 20:07:32 +0700 Subject: [PATCH 109/145] chore: avoid fromHexString (#7080) * fix: fromHexString() to fromHex() * fix: throw error for NodeJS fromHex() if malformed string * fix: correct hex value in sim test --- packages/api/src/beacon/routes/proof.ts | 8 ++++---- packages/api/src/beacon/routes/validator.ts | 16 ++++++++-------- packages/api/src/builder/routes.ts | 7 +++---- packages/api/src/utils/serdes.ts | 6 +++--- .../src/chain/archiver/archiveBlocks.ts | 9 ++++----- packages/beacon-node/src/chain/chain.ts | 16 ++++++++-------- packages/beacon-node/src/chain/opPools/opPool.ts | 5 ++--- packages/beacon-node/src/chain/regen/regen.ts | 5 ++--- .../src/chain/stateCache/datastore/file.ts | 5 ++--- .../stateCache/persistentCheckpointsCache.ts | 7 +++---- packages/beacon-node/src/eth1/index.ts | 4 ++-- packages/beacon-node/src/eth1/provider/utils.ts | 5 ++--- .../src/eth1/utils/depositContract.ts | 10 +++++----- .../beacon-node/src/network/core/networkCore.ts | 5 ++--- .../reqresp/beaconBlocksMaybeBlobsByRoot.ts | 4 ++-- packages/beacon-node/src/sync/unknownBlock.ts | 7 +++---- packages/cli/src/cmds/lightclient/handler.ts | 4 ++-- .../src/cmds/validator/blsToExecutionChange.ts | 7 +++---- .../cli/src/cmds/validator/keymanager/impl.ts | 5 ++--- packages/cli/src/util/format.ts | 4 ++-- .../utils/crucible/assertions/blobsAssertion.ts | 2 +- .../config/src/chainConfig/configs/mainnet.ts | 2 +- .../config/src/chainConfig/configs/minimal.ts | 2 +- packages/config/src/chainConfig/json.ts | 5 ++--- .../config/src/chainConfig/networks/chiado.ts | 2 +- .../config/src/chainConfig/networks/ephemery.ts | 2 +- .../config/src/chainConfig/networks/gnosis.ts | 2 +- .../config/src/chainConfig/networks/holesky.ts | 2 +- .../config/src/chainConfig/networks/mainnet.ts | 2 +- .../config/src/chainConfig/networks/sepolia.ts | 2 +- packages/light-client/src/index.ts | 5 ++--- .../state-transition/src/cache/epochCache.ts | 5 ++--- packages/utils/src/bytes/nodejs.ts | 14 +++++++++++++- .../validator/src/services/chainHeaderTracker.ts | 5 ++--- .../src/services/doppelgangerService.ts | 5 ++--- .../validator/src/services/externalSignerSync.ts | 5 ++--- .../validator/src/services/validatorStore.ts | 10 +++++----- .../interchange/formats/completeV4.ts | 7 +++---- .../slashingProtection/interchange/formats/v5.ts | 7 +++---- .../validator/src/slashingProtection/utils.ts | 5 ++--- 40 files changed, 111 insertions(+), 119 deletions(-) diff --git a/packages/api/src/beacon/routes/proof.ts b/packages/api/src/beacon/routes/proof.ts index a48caf02c9e7..2ef4dad15e81 100644 --- a/packages/api/src/beacon/routes/proof.ts +++ b/packages/api/src/beacon/routes/proof.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {CompactMultiProof, ProofType} from "@chainsafe/persistent-merkle-tree"; -import {ByteListType, ContainerType, fromHexString} from "@chainsafe/ssz"; -import {toHex} from "@lodestar/utils"; +import {ByteListType, ContainerType} from "@chainsafe/ssz"; +import {fromHex, toHex} from "@lodestar/utils"; import {ChainForkConfig} from "@lodestar/config"; import {ssz} from "@lodestar/types"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; @@ -47,7 +47,7 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({params: {state_id: stateId}, query: {format: toHex(descriptor)}}), - parseReq: ({params, query}) => ({stateId: params.state_id, descriptor: fromHexString(query.format)}), + parseReq: ({params, query}) => ({stateId: params.state_id, descriptor: fromHex(query.format)}), schema: {params: {state_id: Schema.StringRequired}, query: {format: Schema.StringRequired}}, }, resp: { @@ -65,7 +65,7 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({params: {block_id: blockId}, query: {format: toHex(descriptor)}}), - parseReq: ({params, query}) => ({blockId: params.block_id, descriptor: fromHexString(query.format)}), + parseReq: ({params, query}) => ({blockId: params.block_id, descriptor: fromHex(query.format)}), schema: {params: {block_id: Schema.StringRequired}, query: {format: Schema.StringRequired}}, }, resp: { diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 3acbfec3e800..a9a1423e4da2 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {ContainerType, fromHexString, Type, ValueOf} from "@chainsafe/ssz"; +import {ContainerType, Type, ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {isForkBlobs, isForkPostElectra} from "@lodestar/params"; import { @@ -20,7 +20,7 @@ import { Attestation, sszTypesFor, } from "@lodestar/types"; -import {toHex, toRootHex} from "@lodestar/utils"; +import {fromHex, toHex, toRootHex} from "@lodestar/utils"; import {Endpoint, RouteDefinitions, Schema} from "../../utils/index.js"; import {fromGraffitiHex, toBoolean, toGraffitiHex} from "../../utils/serdes.js"; import {getExecutionForkTypes, toForkName} from "../../utils/fork.js"; @@ -633,7 +633,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ slot: params.slot, - randaoReveal: fromHexString(query.randao_reveal), + randaoReveal: fromHex(query.randao_reveal), graffiti: fromGraffitiHex(query.graffiti), feeRecipient: query.fee_recipient, builderSelection: query.builder_selection as BuilderSelection, @@ -687,7 +687,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ slot: params.slot, - randaoReveal: fromHexString(query.randao_reveal), + randaoReveal: fromHex(query.randao_reveal), graffiti: fromGraffitiHex(query.graffiti), skipRandaoVerification: parseSkipRandaoVerification(query.skip_randao_verification), feeRecipient: query.fee_recipient, @@ -770,7 +770,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ slot: params.slot, - randaoReveal: fromHexString(query.randao_reveal), + randaoReveal: fromHex(query.randao_reveal), graffiti: fromGraffitiHex(query.graffiti), }), schema: { @@ -811,7 +811,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ slot: query.slot, subcommitteeIndex: query.subcommittee_index, - beaconBlockRoot: fromHexString(query.beacon_block_root), + beaconBlockRoot: fromHex(query.beacon_block_root), }), schema: { query: { @@ -834,7 +834,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - attestationDataRoot: fromHexString(query.attestation_data_root), + attestationDataRoot: fromHex(query.attestation_data_root), slot: query.slot, }), schema: { @@ -857,7 +857,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ - attestationDataRoot: fromHexString(query.attestation_data_root), + attestationDataRoot: fromHex(query.attestation_data_root), slot: query.slot, committeeIndex: query.committee_index, }), diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts index ffed5a91824c..7459e46abd0b 100644 --- a/packages/api/src/builder/routes.ts +++ b/packages/api/src/builder/routes.ts @@ -1,5 +1,4 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString} from "@chainsafe/ssz"; import { ssz, bellatrix, @@ -13,7 +12,7 @@ import { } from "@lodestar/types"; import {ForkName, isForkBlobs} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; -import {toPubkeyHex, toRootHex} from "@lodestar/utils"; +import {fromHex, toPubkeyHex, toRootHex} from "@lodestar/utils"; import {Endpoint, RouteDefinitions, Schema} from "../utils/index.js"; import {MetaHeader, VersionCodec, VersionMeta} from "../utils/metadata.js"; @@ -110,8 +109,8 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions ({ slot: params.slot, - parentHash: fromHexString(params.parent_hash), - proposerPubkey: fromHexString(params.pubkey), + parentHash: fromHex(params.parent_hash), + proposerPubkey: fromHex(params.pubkey), }), schema: { params: {slot: Schema.UintRequired, parent_hash: Schema.StringRequired, pubkey: Schema.StringRequired}, diff --git a/packages/api/src/utils/serdes.ts b/packages/api/src/utils/serdes.ts index 8fc746ac8264..282c2514e00d 100644 --- a/packages/api/src/utils/serdes.ts +++ b/packages/api/src/utils/serdes.ts @@ -1,5 +1,5 @@ -import {fromHexString, JsonPath} from "@chainsafe/ssz"; -import {toHex} from "@lodestar/utils"; +import {JsonPath} from "@chainsafe/ssz"; +import {fromHex, toHex} from "@lodestar/utils"; /** * Serialize proof path to JSON. @@ -103,7 +103,7 @@ export function fromGraffitiHex(hex?: string): string | undefined { return undefined; } try { - return new TextDecoder("utf8").decode(fromHexString(hex)); + return new TextDecoder("utf8").decode(fromHex(hex)); } catch { // allow malformed graffiti hex string return hex; diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts index ed6dd5bfc091..8e1ac456579a 100644 --- a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts @@ -1,7 +1,6 @@ -import {fromHexString} from "@chainsafe/ssz"; import {Epoch, Slot, RootHex} from "@lodestar/types"; import {IForkChoice} from "@lodestar/fork-choice"; -import {Logger, toRootHex} from "@lodestar/utils"; +import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {KeyValue} from "@lodestar/db"; @@ -48,7 +47,7 @@ export async function archiveBlocks( const finalizedCanonicalBlockRoots: BlockRootSlot[] = finalizedCanonicalBlocks.map((block) => ({ slot: block.slot, - root: fromHexString(block.blockRoot), + root: fromHex(block.blockRoot), })); if (finalizedCanonicalBlockRoots.length > 0) { @@ -68,7 +67,7 @@ export async function archiveBlocks( // deleteNonCanonicalBlocks // loop through forkchoice single time - const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHexString(summary.blockRoot)); + const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHex(summary.blockRoot)); if (nonCanonicalBlockRoots.length > 0) { await db.block.batchDelete(nonCanonicalBlockRoots); logger.verbose("Deleted non canonical blocks from hot DB", { @@ -144,7 +143,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot value: blockBuffer, slot: block.slot, blockRoot: block.root, - // TODO: Benchmark if faster to slice Buffer or fromHexString() + // TODO: Benchmark if faster to slice Buffer or fromHex() parentRoot: getParentRootFromSignedBlock(blockBuffer), }; }) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 8dbb49798538..ee70231c7d40 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz"; +import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import { BeaconStateAllForks, CachedBeaconStateAllForks, @@ -35,7 +35,7 @@ import { } from "@lodestar/types"; import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice"; import {ProcessShutdownCallback} from "@lodestar/validator"; -import {Logger, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils"; +import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils"; import {ForkSeq, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; @@ -521,7 +521,7 @@ export class BeaconChain implements IBeaconChain { }; } - const data = await this.db.stateArchive.getByRoot(fromHexString(stateRoot)); + const data = await this.db.stateArchive.getByRoot(fromHex(stateRoot)); return data && {state: data, executionOptimistic: false, finalized: true}; } @@ -568,7 +568,7 @@ export class BeaconChain implements IBeaconChain { // Unfinalized slot, attempt to find in fork-choice const block = this.forkChoice.getCanonicalBlockAtSlot(slot); if (block) { - const data = await this.db.block.get(fromHexString(block.blockRoot)); + const data = await this.db.block.get(fromHex(block.blockRoot)); if (data) { return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false}; } @@ -587,7 +587,7 @@ export class BeaconChain implements IBeaconChain { ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null> { const block = this.forkChoice.getBlockHex(root); if (block) { - const data = await this.db.block.get(fromHexString(root)); + const data = await this.db.block.get(fromHex(root)); if (data) { return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false}; } @@ -595,7 +595,7 @@ export class BeaconChain implements IBeaconChain { // TODO: Add a lock to the archiver to have deterministic behavior on where are blocks } - const data = await this.db.blockArchive.getByRoot(fromHexString(root)); + const data = await this.db.blockArchive.getByRoot(fromHex(root)); return data && {block: data, executionOptimistic: false, finalized: true}; } @@ -768,7 +768,7 @@ export class BeaconChain implements IBeaconChain { finalizedRoot: finalizedCheckpoint.epoch === GENESIS_EPOCH ? ZERO_HASH : finalizedCheckpoint.root, finalizedEpoch: finalizedCheckpoint.epoch, // TODO: PERFORMANCE: Memoize to prevent re-computing every time - headRoot: fromHexString(head.blockRoot), + headRoot: fromHex(head.blockRoot), headSlot: head.slot, }; } @@ -1120,7 +1120,7 @@ export class BeaconChain implements IBeaconChain { // TODO: Improve using regen here const {blockRoot, stateRoot, slot} = this.forkChoice.getHead(); const headState = this.regen.getStateSync(stateRoot); - const headBlock = await this.db.block.get(fromHexString(blockRoot)); + const headBlock = await this.db.block.get(fromHex(blockRoot)); if (headBlock == null) { throw Error(`Head block ${slot} ${headBlock} is not available in database`); } diff --git a/packages/beacon-node/src/chain/opPools/opPool.ts b/packages/beacon-node/src/chain/opPools/opPool.ts index dc3fee0ac8a8..ee66591e9aef 100644 --- a/packages/beacon-node/src/chain/opPools/opPool.ts +++ b/packages/beacon-node/src/chain/opPools/opPool.ts @@ -1,4 +1,3 @@ -import {fromHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAllForks, computeEpochAtSlot, @@ -16,7 +15,7 @@ import { ForkSeq, MAX_ATTESTER_SLASHINGS_ELECTRA, } from "@lodestar/params"; -import {toHex, toRootHex} from "@lodestar/utils"; +import {fromHex, toHex, toRootHex} from "@lodestar/utils"; import {Epoch, phase0, capella, ssz, ValidatorIndex, SignedBeaconBlock, AttesterSlashing} from "@lodestar/types"; import {IBeaconDb} from "../../db/index.js"; import {SignedBLSToExecutionChangeVersioned} from "../../util/types.js"; @@ -85,7 +84,7 @@ export class OpPool { persistDiff( db.attesterSlashing, Array.from(this.attesterSlashings.entries()).map(([key, value]) => ({ - key: fromHexString(key), + key: fromHex(key), value: value.attesterSlashing, })), toHex diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 04cf5b40b494..08fda70d9184 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -1,4 +1,3 @@ -import {fromHexString} from "@chainsafe/ssz"; import {phase0, Slot, RootHex, BeaconBlock, SignedBeaconBlock} from "@lodestar/types"; import { CachedBeaconStateAllForks, @@ -11,7 +10,7 @@ import { StateHashTreeRootSource, } from "@lodestar/state-transition"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {Logger, toRootHex} from "@lodestar/utils"; +import {Logger, fromHex, toRootHex} from "@lodestar/utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; import {Metrics} from "../../metrics/index.js"; @@ -216,7 +215,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { const protoBlocksAsc = blocksToReplay.reverse(); for (const [i, protoBlock] of protoBlocksAsc.entries()) { replaySlots[i] = protoBlock.slot; - blockPromises[i] = this.modules.db.block.get(fromHexString(protoBlock.blockRoot)); + blockPromises[i] = this.modules.db.block.get(fromHex(protoBlock.blockRoot)); } const logCtx = {stateRoot, replaySlots: replaySlots.join(",")}; diff --git a/packages/beacon-node/src/chain/stateCache/datastore/file.ts b/packages/beacon-node/src/chain/stateCache/datastore/file.ts index 8a07df90d168..f487079ae443 100644 --- a/packages/beacon-node/src/chain/stateCache/datastore/file.ts +++ b/packages/beacon-node/src/chain/stateCache/datastore/file.ts @@ -1,7 +1,6 @@ import path from "node:path"; -import {fromHexString} from "@chainsafe/ssz"; import {phase0, ssz} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; +import {fromHex, toHex} from "@lodestar/utils"; import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js"; import {CPStateDatastore, DatastoreKey} from "./types.js"; @@ -48,6 +47,6 @@ export class FileCPStateDatastore implements CPStateDatastore { const fileNames = await readFileNames(this.folderPath); return fileNames .filter((fileName) => fileName.startsWith("0x") && fileName.length === CHECKPOINT_FILE_NAME_LENGTH) - .map((fileName) => fromHexString(fileName)); + .map((fileName) => fromHex(fileName)); } } diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index c919c3c86233..d8769374a29c 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -1,7 +1,6 @@ -import {fromHexString} from "@chainsafe/ssz"; import {phase0, Epoch, RootHex} from "@lodestar/types"; import {CachedBeaconStateAllForks, computeStartSlotAtEpoch, getBlockRootAtSlot} from "@lodestar/state-transition"; -import {Logger, MapDef, sleep, toHex, toRootHex} from "@lodestar/utils"; +import {Logger, MapDef, fromHex, sleep, toHex, toRootHex} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {loadCachedBeaconState} from "@lodestar/state-transition"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; @@ -657,7 +656,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { let persistCount = 0; const epochBoundarySlot = computeStartSlotAtEpoch(epoch); const epochBoundaryRoot = - epochBoundarySlot === state.slot ? fromHexString(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot); + epochBoundarySlot === state.slot ? fromHex(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot); const epochBoundaryHex = toRootHex(epochBoundaryRoot); const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1)); @@ -698,7 +697,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } else { // persist and do not update epochIndex this.metrics?.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0); - const cpPersist = {epoch: epoch, root: fromHexString(rootHex)}; + const cpPersist = {epoch: epoch, root: fromHex(rootHex)}; // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. // As monitored on holesky as of Jan 2024: // - This does not increase heap allocation while gc time is the same diff --git a/packages/beacon-node/src/eth1/index.ts b/packages/beacon-node/src/eth1/index.ts index a8ba55c54141..7b2ec17496d3 100644 --- a/packages/beacon-node/src/eth1/index.ts +++ b/packages/beacon-node/src/eth1/index.ts @@ -1,6 +1,6 @@ -import {fromHexString} from "@chainsafe/ssz"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Root} from "@lodestar/types"; +import {fromHex} from "@lodestar/utils"; import {IEth1ForBlockProduction, Eth1DataAndDeposits, IEth1Provider, PowMergeBlock, TDProgress} from "./interface.js"; import {Eth1DepositDataTracker, Eth1DepositDataTrackerModules} from "./eth1DepositDataTracker.js"; import {Eth1MergeBlockTracker, Eth1MergeBlockTrackerModules} from "./eth1MergeBlockTracker.js"; @@ -92,7 +92,7 @@ export class Eth1ForBlockProduction implements IEth1ForBlockProduction { async getTerminalPowBlock(): Promise { const block = await this.eth1MergeBlockTracker.getTerminalPowBlock(); - return block && fromHexString(block.blockHash); + return block && fromHex(block.blockHash); } getPowBlock(powBlockHash: string): Promise { diff --git a/packages/beacon-node/src/eth1/provider/utils.ts b/packages/beacon-node/src/eth1/provider/utils.ts index cddf48c40d5a..096f1d7233c8 100644 --- a/packages/beacon-node/src/eth1/provider/utils.ts +++ b/packages/beacon-node/src/eth1/provider/utils.ts @@ -1,6 +1,5 @@ -import {fromHexString} from "@chainsafe/ssz"; import {RootHex} from "@lodestar/types"; -import {bytesToBigInt, bigIntToBytes, toHex} from "@lodestar/utils"; +import {bytesToBigInt, bigIntToBytes, toHex, fromHex} from "@lodestar/utils"; import {ErrorParseJson} from "./jsonRpcHttpClient.js"; /** QUANTITY as defined in ethereum execution layer JSON RPC https://eth.wiki/json-rpc/API */ @@ -108,7 +107,7 @@ export function bytesToData(bytes: Uint8Array): DATA { */ export function dataToBytes(hex: DATA, fixedLength: number | null): Uint8Array { try { - const bytes = fromHexString(hex); + const bytes = fromHex(hex); if (fixedLength != null && bytes.length !== fixedLength) { throw Error(`Wrong data length ${bytes.length} expected ${fixedLength}`); } diff --git a/packages/beacon-node/src/eth1/utils/depositContract.ts b/packages/beacon-node/src/eth1/utils/depositContract.ts index 7247fe8cebaf..b576a3d5f61c 100644 --- a/packages/beacon-node/src/eth1/utils/depositContract.ts +++ b/packages/beacon-node/src/eth1/utils/depositContract.ts @@ -1,6 +1,6 @@ import {Interface} from "@ethersproject/abi"; -import {fromHexString} from "@chainsafe/ssz"; import {phase0, ssz} from "@lodestar/types"; +import {fromHex} from "@lodestar/utils"; const depositEventFragment = "event DepositEvent(bytes pubkey, bytes withdrawal_credentials, bytes amount, bytes signature, bytes index)"; @@ -23,15 +23,15 @@ export function parseDepositLog(log: {blockNumber: number; data: string; topics: blockNumber: log.blockNumber, index: parseHexNumLittleEndian(values.index), depositData: { - pubkey: fromHexString(values.pubkey), - withdrawalCredentials: fromHexString(values.withdrawal_credentials), + pubkey: fromHex(values.pubkey), + withdrawalCredentials: fromHex(values.withdrawal_credentials), amount: parseHexNumLittleEndian(values.amount), - signature: fromHexString(values.signature), + signature: fromHex(values.signature), }, }; } function parseHexNumLittleEndian(hex: string): number { // Can't use parseInt() because amount is a hex string in little endian - return ssz.UintNum64.deserialize(fromHexString(hex)); + return ssz.UintNum64.deserialize(fromHex(hex)); } diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index 07b346bc29e4..83ef4c4fb063 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -2,13 +2,12 @@ import {Connection, PeerId} from "@libp2p/interface"; import {multiaddr} from "@multiformats/multiaddr"; import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types"; import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/dist/src/score/peer-score.js"; -import {fromHexString} from "@chainsafe/ssz"; import {ENR} from "@chainsafe/enr"; import {routes} from "@lodestar/api"; import {BeaconConfig} from "@lodestar/config"; import type {LoggerNode} from "@lodestar/logger/node"; import {Epoch, phase0} from "@lodestar/types"; -import {withTimeout} from "@lodestar/utils"; +import {fromHex, withTimeout} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; import {ResponseIncoming} from "@lodestar/reqresp"; import {Libp2p} from "../interface.js"; @@ -195,7 +194,7 @@ export class NetworkCore implements INetworkCore { await gossip.start(); const enr = opts.discv5?.enr; - const nodeId = enr ? fromHexString(ENR.decodeTxt(enr).nodeId) : null; + const nodeId = enr ? fromHex(ENR.decodeTxt(enr).nodeId) : null; const attnetsService = new AttnetsService(config, clock, gossip, metadata, logger, metrics, nodeId, opts); const syncnetsService = new SyncnetsService(config, clock, gossip, metadata, logger, metrics, opts); diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts index 2b802ab1edd9..b53addab9b43 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts @@ -1,7 +1,7 @@ -import {fromHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {phase0, deneb} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; +import {fromHex} from "@lodestar/utils"; import { BlockInput, BlockInputType, @@ -66,7 +66,7 @@ export async function unavailableBeaconBlobsByRoot( // resolve the block if thats unavailable let block, blobsCache, blockBytes, resolveAvailability, cachedData; if (unavailableBlockInput.block === null) { - const allBlocks = await network.sendBeaconBlocksByRoot(peerId, [fromHexString(unavailableBlockInput.blockRootHex)]); + const allBlocks = await network.sendBeaconBlocksByRoot(peerId, [fromHex(unavailableBlockInput.blockRootHex)]); block = allBlocks[0].data; blockBytes = allBlocks[0].bytes; cachedData = unavailableBlockInput.cachedData; diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index d985847d450f..a172b5d723db 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -1,6 +1,5 @@ -import {fromHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; -import {Logger, pruneSetToMax, toRootHex} from "@lodestar/utils"; +import {Logger, fromHex, pruneSetToMax, toRootHex} from "@lodestar/utils"; import {Root, RootHex, deneb} from "@lodestar/types"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; @@ -288,7 +287,7 @@ export class UnknownBlockSync { let res; if (block.blockInput === null) { - res = await wrapError(this.fetchUnknownBlockRoot(fromHexString(block.blockRootHex), connectedPeers)); + res = await wrapError(this.fetchUnknownBlockRoot(fromHex(block.blockRootHex), connectedPeers)); } else { res = await wrapError(this.fetchUnavailableBlockInput(block.blockInput, connectedPeers)); } @@ -519,7 +518,7 @@ export class UnknownBlockSync { if (unavailableBlockInput.block === null) { blockRootHex = unavailableBlockInput.blockRootHex; - blockRoot = fromHexString(blockRootHex); + blockRoot = fromHex(blockRootHex); } else { const unavailableBlock = unavailableBlockInput.block; blockRoot = this.config diff --git a/packages/cli/src/cmds/lightclient/handler.ts b/packages/cli/src/cmds/lightclient/handler.ts index a8e3d7333f98..5af8f84d8959 100644 --- a/packages/cli/src/cmds/lightclient/handler.ts +++ b/packages/cli/src/cmds/lightclient/handler.ts @@ -1,9 +1,9 @@ import path from "node:path"; -import {fromHexString} from "@chainsafe/ssz"; import {getClient} from "@lodestar/api"; import {Lightclient} from "@lodestar/light-client"; import {LightClientRestTransport} from "@lodestar/light-client/transport"; import {getNodeLogger} from "@lodestar/logger/node"; +import {fromHex} from "@lodestar/utils"; import {getBeaconConfigFromArgs} from "../../config/beaconParams.js"; import {getGlobalPaths} from "../../paths/global.js"; import {parseLoggerArgs} from "../../util/logger.js"; @@ -28,7 +28,7 @@ export async function lightclientHandler(args: ILightClientArgs & GlobalArgs): P genesisTime, genesisValidatorsRoot, }, - checkpointRoot: fromHexString(args.checkpointRoot), + checkpointRoot: fromHex(args.checkpointRoot), transport: new LightClientRestTransport(api), }); diff --git a/packages/cli/src/cmds/validator/blsToExecutionChange.ts b/packages/cli/src/cmds/validator/blsToExecutionChange.ts index 3a2fabcc37eb..dc55647fda80 100644 --- a/packages/cli/src/cmds/validator/blsToExecutionChange.ts +++ b/packages/cli/src/cmds/validator/blsToExecutionChange.ts @@ -1,11 +1,10 @@ -import {fromHexString} from "@chainsafe/ssz"; import {SecretKey} from "@chainsafe/blst"; import {computeSigningRoot} from "@lodestar/state-transition"; import {DOMAIN_BLS_TO_EXECUTION_CHANGE, ForkName} from "@lodestar/params"; import {createBeaconConfig} from "@lodestar/config"; import {ssz, capella} from "@lodestar/types"; import {getClient} from "@lodestar/api"; -import {CliCommand} from "@lodestar/utils"; +import {CliCommand, fromHex} from "@lodestar/utils"; import {GlobalArgs} from "../../options/index.js"; import {getBeaconConfigFromArgs} from "../../config/index.js"; @@ -67,13 +66,13 @@ like to choose for BLS To Execution Change.", throw new Error(`Validator pubkey ${publicKey} not found in state`); } - const blsPrivkey = SecretKey.fromBytes(fromHexString(args.fromBlsPrivkey)); + const blsPrivkey = SecretKey.fromBytes(fromHex(args.fromBlsPrivkey)); const fromBlsPubkey = blsPrivkey.toPublicKey().toBytes(); const blsToExecutionChange: capella.BLSToExecutionChange = { validatorIndex: validator.index, fromBlsPubkey, - toExecutionAddress: fromHexString(args.toExecutionAddress), + toExecutionAddress: fromHex(args.toExecutionAddress), }; const signatureFork = ForkName.phase0; diff --git a/packages/cli/src/cmds/validator/keymanager/impl.ts b/packages/cli/src/cmds/validator/keymanager/impl.ts index 2915c125eac2..d965b2af5075 100644 --- a/packages/cli/src/cmds/validator/keymanager/impl.ts +++ b/packages/cli/src/cmds/validator/keymanager/impl.ts @@ -1,5 +1,4 @@ import {Keystore} from "@chainsafe/bls-keystore"; -import {fromHexString} from "@chainsafe/ssz"; import {SecretKey} from "@chainsafe/blst"; import { DeleteRemoteKeyStatus, @@ -21,7 +20,7 @@ import {KeymanagerApiMethods as Api} from "@lodestar/api/keymanager/server"; import {Interchange, SignerType, Validator} from "@lodestar/validator"; import {ApiError} from "@lodestar/api/server"; import {Epoch} from "@lodestar/types"; -import {isValidHttpUrl} from "@lodestar/utils"; +import {fromHex, isValidHttpUrl} from "@lodestar/utils"; import {getPubkeyHexFromKeystore, isValidatePubkeyHex} from "../../../util/format.js"; import {parseFeeRecipient} from "../../../util/index.js"; import {DecryptKeystoresThreadPool} from "./decryptKeystores/index.js"; @@ -224,7 +223,7 @@ export class KeymanagerApi implements Api { } } - const pubkeysBytes = pubkeys.map((pubkeyHex) => fromHexString(pubkeyHex)); + const pubkeysBytes = pubkeys.map((pubkeyHex) => fromHex(pubkeyHex)); const interchangeV5 = await this.validator.exportInterchange(pubkeysBytes, { version: "5", diff --git a/packages/cli/src/util/format.ts b/packages/cli/src/util/format.ts index 01c2753193a4..a86ca0662a9f 100644 --- a/packages/cli/src/util/format.ts +++ b/packages/cli/src/util/format.ts @@ -1,5 +1,5 @@ import {PublicKey} from "@chainsafe/blst"; -import {fromHexString} from "@chainsafe/ssz"; +import {fromHex} from "@lodestar/utils"; /** * 0x prefix a string if not prefixed already @@ -50,7 +50,7 @@ export function parseRange(range: string): number[] { export function assertValidPubkeysHex(pubkeysHex: string[]): void { for (const pubkeyHex of pubkeysHex) { - const pubkeyBytes = fromHexString(pubkeyHex); + const pubkeyBytes = fromHex(pubkeyHex); PublicKey.fromBytes(pubkeyBytes, true); } } diff --git a/packages/cli/test/utils/crucible/assertions/blobsAssertion.ts b/packages/cli/test/utils/crucible/assertions/blobsAssertion.ts index dece5bc58ce8..50c9ed13f972 100644 --- a/packages/cli/test/utils/crucible/assertions/blobsAssertion.ts +++ b/packages/cli/test/utils/crucible/assertions/blobsAssertion.ts @@ -35,7 +35,7 @@ export function createBlobsAssertion( gasLimit: "0xc350", maxPriorityFeePerGas: "0x3b9aca00", maxFeePerGas: "0x3ba26b20", - maxFeePerBlobGas: "0x3e8", + maxFeePerBlobGas: "0x3e", value: "0x10000", nonce: `0x${(nonce ?? 0).toString(16)}`, blobVersionedHashes, diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 741ddc99f8cd..e5792a87ac70 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {PresetName} from "@lodestar/params"; import {ChainConfig} from "../types.js"; diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index 26f49cc3e47d..53229d283511 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {PresetName} from "@lodestar/params"; import {ChainConfig} from "../types.js"; diff --git a/packages/config/src/chainConfig/json.ts b/packages/config/src/chainConfig/json.ts index 887409cd4c5b..78db9230c836 100644 --- a/packages/config/src/chainConfig/json.ts +++ b/packages/config/src/chainConfig/json.ts @@ -1,5 +1,4 @@ -import {fromHexString} from "@chainsafe/ssz"; -import {toHex} from "@lodestar/utils"; +import {fromHex, toHex} from "@lodestar/utils"; import {ChainConfig, chainConfigTypes, SpecValue, SpecValueTypeName} from "./types.js"; const MAX_UINT64_JSON = "18446744073709551615"; @@ -96,7 +95,7 @@ export function deserializeSpecValue(valueStr: unknown, typeName: SpecValueTypeN return BigInt(valueStr); case "bytes": - return fromHexString(valueStr); + return fromHex(valueStr); case "string": return valueStr; diff --git a/packages/config/src/chainConfig/networks/chiado.ts b/packages/config/src/chainConfig/networks/chiado.ts index 43b13a210dac..d96bd7510c65 100644 --- a/packages/config/src/chainConfig/networks/chiado.ts +++ b/packages/config/src/chainConfig/networks/chiado.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; import {gnosisChainConfig as gnosis} from "./gnosis.js"; diff --git a/packages/config/src/chainConfig/networks/ephemery.ts b/packages/config/src/chainConfig/networks/ephemery.ts index 29e3f7b92d01..6fefc1800bfc 100644 --- a/packages/config/src/chainConfig/networks/ephemery.ts +++ b/packages/config/src/chainConfig/networks/ephemery.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; import {chainConfig as mainnet} from "../configs/mainnet.js"; diff --git a/packages/config/src/chainConfig/networks/gnosis.ts b/packages/config/src/chainConfig/networks/gnosis.ts index 8034e478fd28..2f58ffd4d045 100644 --- a/packages/config/src/chainConfig/networks/gnosis.ts +++ b/packages/config/src/chainConfig/networks/gnosis.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {PresetName} from "@lodestar/params"; import {ChainConfig} from "../types.js"; import {chainConfig as mainnet} from "../configs/mainnet.js"; diff --git a/packages/config/src/chainConfig/networks/holesky.ts b/packages/config/src/chainConfig/networks/holesky.ts index 187543b871f2..16c462a9468e 100644 --- a/packages/config/src/chainConfig/networks/holesky.ts +++ b/packages/config/src/chainConfig/networks/holesky.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; import {chainConfig as mainnet} from "../configs/mainnet.js"; diff --git a/packages/config/src/chainConfig/networks/mainnet.ts b/packages/config/src/chainConfig/networks/mainnet.ts index 24584ad8442b..c137c578fc0e 100644 --- a/packages/config/src/chainConfig/networks/mainnet.ts +++ b/packages/config/src/chainConfig/networks/mainnet.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; import {chainConfig as mainnet} from "../configs/mainnet.js"; diff --git a/packages/config/src/chainConfig/networks/sepolia.ts b/packages/config/src/chainConfig/networks/sepolia.ts index 51102cfafa7d..39e72a24f3f6 100644 --- a/packages/config/src/chainConfig/networks/sepolia.ts +++ b/packages/config/src/chainConfig/networks/sepolia.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString as b} from "@chainsafe/ssz"; +import {fromHex as b} from "@lodestar/utils"; import {ChainConfig} from "../types.js"; import {chainConfig as mainnet} from "../configs/mainnet.js"; diff --git a/packages/light-client/src/index.ts b/packages/light-client/src/index.ts index 03be60eca05d..5ec0dd73b469 100644 --- a/packages/light-client/src/index.ts +++ b/packages/light-client/src/index.ts @@ -1,5 +1,4 @@ import mitt from "mitt"; -import {fromHexString} from "@chainsafe/ssz"; import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD} from "@lodestar/params"; import { LightClientBootstrap, @@ -13,7 +12,7 @@ import { SyncPeriod, } from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainForkConfig} from "@lodestar/config"; -import {isErrorAborted, sleep, toRootHex} from "@lodestar/utils"; +import {fromHex, isErrorAborted, sleep, toRootHex} from "@lodestar/utils"; import {getCurrentSlot, slotWithFutureTolerance, timeUntilNextEpoch} from "./utils/clock.js"; import {chunkifyInclusiveRange} from "./utils/chunkify.js"; import {LightclientEmitter, LightclientEvent} from "./events.js"; @@ -120,7 +119,7 @@ export class Lightclient { this.genesisTime = genesisData.genesisTime; this.genesisValidatorsRoot = typeof genesisData.genesisValidatorsRoot === "string" - ? fromHexString(genesisData.genesisValidatorsRoot) + ? fromHex(genesisData.genesisValidatorsRoot) : genesisData.genesisValidatorsRoot; this.config = createBeaconConfig(config, this.genesisValidatorsRoot); diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index af6e976e9089..ba8ec9b05ab5 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,6 +1,5 @@ import {PublicKey} from "@chainsafe/blst"; import * as immutable from "immutable"; -import {fromHexString} from "@chainsafe/ssz"; import { BLSSignature, CommitteeIndex, @@ -25,7 +24,7 @@ import { SLOTS_PER_EPOCH, WEIGHT_DENOMINATOR, } from "@lodestar/params"; -import {LodestarError} from "@lodestar/utils"; +import {LodestarError, fromHex} from "@lodestar/utils"; import { computeActivationExitEpoch, computeEpochAtSlot, @@ -964,7 +963,7 @@ export class EpochCache { } this.pubkey2index.set(pubkey, index); - const pubkeyBytes = pubkey instanceof Uint8Array ? pubkey : fromHexString(pubkey); + const pubkeyBytes = pubkey instanceof Uint8Array ? pubkey : fromHex(pubkey); this.index2pubkey[index] = PublicKey.fromBytes(pubkeyBytes); // Optimize for aggregation } diff --git a/packages/utils/src/bytes/nodejs.ts b/packages/utils/src/bytes/nodejs.ts index 636f49bd8e76..4cf8e78c67c6 100644 --- a/packages/utils/src/bytes/nodejs.ts +++ b/packages/utils/src/bytes/nodejs.ts @@ -44,6 +44,18 @@ export function toPubkeyHex(pubkey: Uint8Array): string { } export function fromHex(hex: string): Uint8Array { - const b = Buffer.from(hex.replace("0x", ""), "hex"); + if (typeof hex !== "string") { + throw new Error(`hex argument type ${typeof hex} must be of type string`); + } + + if (hex.startsWith("0x")) { + hex = hex.slice(2); + } + + if (hex.length % 2 !== 0) { + throw new Error(`hex string length ${hex.length} must be multiple of 2`); + } + + const b = Buffer.from(hex, "hex"); return new Uint8Array(b.buffer, b.byteOffset, b.length); } diff --git a/packages/validator/src/services/chainHeaderTracker.ts b/packages/validator/src/services/chainHeaderTracker.ts index 845743264d0b..1c0b0d9a56d8 100644 --- a/packages/validator/src/services/chainHeaderTracker.ts +++ b/packages/validator/src/services/chainHeaderTracker.ts @@ -1,6 +1,5 @@ -import {fromHexString} from "@chainsafe/ssz"; import {ApiClient, routes} from "@lodestar/api"; -import {Logger} from "@lodestar/utils"; +import {Logger, fromHex} from "@lodestar/utils"; import {Slot, Root, RootHex} from "@lodestar/types"; import {GENESIS_SLOT} from "@lodestar/params"; import {ValidatorEvent, ValidatorEventEmitter} from "./emitter.js"; @@ -64,7 +63,7 @@ export class ChainHeaderTracker { const {message} = event; const {slot, block, previousDutyDependentRoot, currentDutyDependentRoot} = message; this.headBlockSlot = slot; - this.headBlockRoot = fromHexString(block); + this.headBlockRoot = fromHex(block); const headEventData = { slot: this.headBlockSlot, diff --git a/packages/validator/src/services/doppelgangerService.ts b/packages/validator/src/services/doppelgangerService.ts index 5435c5aed37f..e167ee1d5028 100644 --- a/packages/validator/src/services/doppelgangerService.ts +++ b/packages/validator/src/services/doppelgangerService.ts @@ -1,7 +1,6 @@ -import {fromHexString} from "@chainsafe/ssz"; import {Epoch, ValidatorIndex} from "@lodestar/types"; import {ApiClient, routes} from "@lodestar/api"; -import {Logger, sleep, truncBytes} from "@lodestar/utils"; +import {Logger, fromHex, sleep, truncBytes} from "@lodestar/utils"; import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ISlashingProtection} from "../slashingProtection/index.js"; import {ProcessShutdownCallback, PubkeyHex} from "../types.js"; @@ -69,7 +68,7 @@ export class DoppelgangerService { if (remainingEpochs > 0) { const previousEpoch = currentEpoch - 1; const attestedInPreviousEpoch = await this.slashingProtection.hasAttestedInEpoch( - fromHexString(pubkeyHex), + fromHex(pubkeyHex), previousEpoch ); diff --git a/packages/validator/src/services/externalSignerSync.ts b/packages/validator/src/services/externalSignerSync.ts index 2f1880dda1bb..2f6828d9e09b 100644 --- a/packages/validator/src/services/externalSignerSync.ts +++ b/packages/validator/src/services/externalSignerSync.ts @@ -1,8 +1,7 @@ -import {fromHexString} from "@chainsafe/ssz"; import {PublicKey} from "@chainsafe/blst"; import {ChainForkConfig} from "@lodestar/config"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {toPrintableUrl} from "@lodestar/utils"; +import {fromHex, toPrintableUrl} from "@lodestar/utils"; import {LoggerVc} from "../util/index.js"; import {externalSignerGetKeys} from "../util/externalSignerClient.js"; @@ -77,7 +76,7 @@ export function pollExternalSignerPubkeys( function assertValidPubkeysHex(pubkeysHex: string[]): void { for (const pubkeyHex of pubkeysHex) { - const pubkeyBytes = fromHexString(pubkeyHex); + const pubkeyBytes = fromHex(pubkeyHex); PublicKey.fromBytes(pubkeyBytes, true); } } diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index b64917ec03ae..6da4f597d87c 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -1,4 +1,4 @@ -import {BitArray, fromHexString} from "@chainsafe/ssz"; +import {BitArray} from "@chainsafe/ssz"; import {SecretKey} from "@chainsafe/blst"; import { computeEpochAtSlot, @@ -42,7 +42,7 @@ import { SignedAggregateAndProof, } from "@lodestar/types"; import {routes} from "@lodestar/api"; -import {toPubkeyHex, toRootHex} from "@lodestar/utils"; +import {fromHex, toPubkeyHex, toRootHex} from "@lodestar/utils"; import {ISlashingProtection} from "../slashingProtection/index.js"; import {PubkeyHex} from "../types.js"; import {externalSignerPostSignature, SignableMessageType, SignableMessage} from "../util/externalSignerClient.js"; @@ -693,8 +693,8 @@ export class ValidatorStore { regAttributes: {feeRecipient: Eth1Address; gasLimit: number}, _slot: Slot ): Promise { - const pubkey = typeof pubkeyMaybeHex === "string" ? fromHexString(pubkeyMaybeHex) : pubkeyMaybeHex; - const feeRecipient = fromHexString(regAttributes.feeRecipient); + const pubkey = typeof pubkeyMaybeHex === "string" ? fromHex(pubkeyMaybeHex) : pubkeyMaybeHex; + const feeRecipient = fromHex(regAttributes.feeRecipient); const {gasLimit} = regAttributes; const validatorRegistration: bellatrix.ValidatorRegistrationV1 = { @@ -775,7 +775,7 @@ export class ValidatorStore { signingSlot, signableMessage ); - return fromHexString(signatureHex); + return fromHex(signatureHex); } catch (e) { this.metrics?.remoteSignErrors.inc(); throw e; diff --git a/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts b/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts index d1e9475daa0e..b43ced6c80d3 100644 --- a/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts +++ b/packages/validator/src/slashingProtection/interchange/formats/completeV4.ts @@ -1,6 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString} from "@chainsafe/ssz"; -import {toPubkeyHex, toRootHex} from "@lodestar/utils"; +import {fromHex, toPubkeyHex, toRootHex} from "@lodestar/utils"; import {InterchangeLodestar} from "../types.js"; import {fromOptionalHexString, numToString, toOptionalHexString} from "../../utils.js"; @@ -110,9 +109,9 @@ export function serializeInterchangeCompleteV4({ export function parseInterchangeCompleteV4(interchange: InterchangeCompleteV4): InterchangeLodestar { return { - genesisValidatorsRoot: fromHexString(interchange.metadata.genesis_validators_root), + genesisValidatorsRoot: fromHex(interchange.metadata.genesis_validators_root), data: interchange.data.map((validator) => ({ - pubkey: fromHexString(validator.pubkey), + pubkey: fromHex(validator.pubkey), signedBlocks: validator.signed_blocks.map((block) => ({ slot: parseInt(block.slot, 10), signingRoot: fromOptionalHexString(block.signing_root), diff --git a/packages/validator/src/slashingProtection/interchange/formats/v5.ts b/packages/validator/src/slashingProtection/interchange/formats/v5.ts index 20d11aefe91a..838ba76c1a57 100644 --- a/packages/validator/src/slashingProtection/interchange/formats/v5.ts +++ b/packages/validator/src/slashingProtection/interchange/formats/v5.ts @@ -1,6 +1,5 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import {fromHexString} from "@chainsafe/ssz"; -import {toPubkeyHex, toRootHex} from "@lodestar/utils"; +import {fromHex, toPubkeyHex, toRootHex} from "@lodestar/utils"; import {InterchangeLodestar} from "../types.js"; import {fromOptionalHexString, numToString, toOptionalHexString} from "../../utils.js"; @@ -105,9 +104,9 @@ export function serializeInterchangeV5({data, genesisValidatorsRoot}: Interchang export function parseInterchangeV5(interchange: InterchangeV5): InterchangeLodestar { return { - genesisValidatorsRoot: fromHexString(interchange.metadata.genesis_validators_root), + genesisValidatorsRoot: fromHex(interchange.metadata.genesis_validators_root), data: interchange.data.map((validator) => ({ - pubkey: fromHexString(validator.pubkey), + pubkey: fromHex(validator.pubkey), signedBlocks: validator.signed_blocks.map((block) => ({ slot: parseInt(block.slot, 10), signingRoot: fromOptionalHexString(block.signing_root), diff --git a/packages/validator/src/slashingProtection/utils.ts b/packages/validator/src/slashingProtection/utils.ts index 294036981b14..adc93bdafee5 100644 --- a/packages/validator/src/slashingProtection/utils.ts +++ b/packages/validator/src/slashingProtection/utils.ts @@ -1,6 +1,5 @@ -import {fromHexString} from "@chainsafe/ssz"; import {Epoch, Root, ssz} from "@lodestar/types"; -import {toHex, toRootHex} from "@lodestar/utils"; +import {fromHex, toHex, toRootHex} from "@lodestar/utils"; export const blsPubkeyLen = 48; export const ZERO_ROOT = ssz.Root.defaultValue(); @@ -14,7 +13,7 @@ export function isEqualNonZeroRoot(root1: Root, root2: Root): boolean { } export function fromOptionalHexString(hex: string | undefined): Root { - return hex ? fromHexString(hex) : ZERO_ROOT; + return hex ? fromHex(hex) : ZERO_ROOT; } export function toOptionalHexString(root: Root): string | undefined { From 295690b89611da4e9bdc551afcd47dda49642462 Mon Sep 17 00:00:00 2001 From: g11tech Date: Sat, 14 Sep 2024 14:44:11 +0530 Subject: [PATCH 110/145] fix: use bigint gwei type for amount in requests instead of num 64 (#7085) * fix: use bigint gwei type for amount in requests instead of num 64 * revert deposit amount to uintnum64 as unlikely to get a high amount * fix --- packages/beacon-node/src/execution/engine/types.ts | 2 +- packages/types/src/electra/sszTypes.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 63cb4da88b6c..139fc6822780 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -461,7 +461,7 @@ export function deserializeWithdrawalRequest(withdrawalRequest: WithdrawalReques return { sourceAddress: dataToBytes(withdrawalRequest.sourceAddress, 20), validatorPubkey: dataToBytes(withdrawalRequest.validatorPubkey, 48), - amount: quantityToNum(withdrawalRequest.amount), + amount: quantityToBigint(withdrawalRequest.amount), }; } diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 31844aac86cc..c0485597686a 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -116,6 +116,8 @@ export const DepositRequest = new ContainerType( { pubkey: BLSPubkey, withdrawalCredentials: Bytes32, + // this is actually gwei uintbn64 type, but super unlikely to get a high amount here + // to warrant a bn type amount: UintNum64, signature: BLSSignature, index: DepositIndex, @@ -129,7 +131,7 @@ export const WithdrawalRequest = new ContainerType( { sourceAddress: ExecutionAddress, validatorPubkey: BLSPubkey, - amount: UintNum64, + amount: Gwei, }, {typeName: "WithdrawalRequest", jsonCase: "eth2"} ); From 43b41afd341b14777fa4ed352ae1460bea843553 Mon Sep 17 00:00:00 2001 From: twoeths Date: Mon, 16 Sep 2024 17:58:55 +0700 Subject: [PATCH 111/145] chore: metrics for regen getState() (#7086) --- dashboards/lodestar_state_cache_regen.json | 491 ++++++++++++++++++++- 1 file changed, 480 insertions(+), 11 deletions(-) diff --git a/dashboards/lodestar_state_cache_regen.json b/dashboards/lodestar_state_cache_regen.json index e7f6336eaeb9..426cde226022 100644 --- a/dashboards/lodestar_state_cache_regen.json +++ b/dashboards/lodestar_state_cache_regen.json @@ -1333,7 +1333,7 @@ "options": { "mode": "exclude", "names": [ - "count_per_epoch" + "sec_from_slot" ], "prefix": "All except:", "readOnly": true @@ -1610,7 +1610,32 @@ }, "mappings": [] }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "from_persistent" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 8, @@ -1705,7 +1730,32 @@ }, "mappings": [] }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "state_reload_validator_serialization" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 8, @@ -1827,7 +1877,32 @@ }, "mappings": [] }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "getState" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 8, @@ -2445,6 +2520,400 @@ "title": "Error rate (grouped by caller)", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 60, + "panels": [], + "title": "Regen - getState", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_regen_get_state_get_seed_state_seconds_sum[$rate_interval])\n/\nrate(lodestar_regen_get_state_get_seed_state_seconds_count[$rate_interval])", + "instant": false, + "legendFormat": "{{caller}}", + "range": true, + "refId": "A" + } + ], + "title": "Get seed state duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_regen_get_state_load_blocks_seconds_sum[$rate_interval])\n/\nrate(lodestar_regen_get_state_load_blocks_seconds_count[$rate_interval])", + "instant": false, + "legendFormat": "{{caller}}", + "range": true, + "refId": "A" + } + ], + "title": "Load blocks duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "validateGossipBlock" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_regen_get_state_state_transition_seconds_sum[$rate_interval])\n/\nrate(lodestar_regen_get_state_state_transition_seconds_count[$rate_interval])", + "instant": false, + "legendFormat": "{{caller}}", + "range": true, + "refId": "A" + } + ], + "title": "State transition duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "validateGossipBlock" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_regen_get_state_block_count_sum[$rate_interval])\n/\nrate(lodestar_regen_get_state_block_count_count[$rate_interval])", + "instant": false, + "legendFormat": "{{caller}}", + "range": true, + "refId": "A" + } + ], + "title": "Reprocessed block count", + "type": "timeseries" + }, { "collapsed": false, "datasource": { @@ -2455,7 +2924,7 @@ "h": 1, "w": 24, "x": 0, - "y": 82 + "y": 99 }, "id": 54, "panels": [], @@ -2523,7 +2992,7 @@ "h": 7, "w": 12, "x": 0, - "y": 83 + "y": 100 }, "id": 42, "options": { @@ -2608,7 +3077,7 @@ "h": 7, "w": 6, "x": 12, - "y": 83 + "y": 100 }, "id": 44, "options": { @@ -2692,7 +3161,7 @@ "h": 7, "w": 6, "x": 18, - "y": 83 + "y": 100 }, "id": 48, "options": { @@ -2776,7 +3245,7 @@ "h": 7, "w": 12, "x": 0, - "y": 90 + "y": 107 }, "id": 46, "options": { @@ -2860,7 +3329,7 @@ "h": 7, "w": 6, "x": 12, - "y": 90 + "y": 107 }, "id": 50, "options": { @@ -2944,7 +3413,7 @@ "h": 7, "w": 6, "x": 18, - "y": 90 + "y": 107 }, "id": 52, "options": { From d3d61af6bd33b92ce7d767953ea80cdeecd686cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:47:05 +0100 Subject: [PATCH 112/145] chore(deps): bump path-to-regexp from 6.2.2 to 6.3.0 (#7082) Bumps [path-to-regexp](https://github.com/pillarjs/path-to-regexp) from 6.2.2 to 6.3.0. - [Release notes](https://github.com/pillarjs/path-to-regexp/releases) - [Changelog](https://github.com/pillarjs/path-to-regexp/blob/master/History.md) - [Commits](https://github.com/pillarjs/path-to-regexp/compare/v6.2.2...v6.3.0) --- updated-dependencies: - dependency-name: path-to-regexp dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 60e41c276e53..7eb7a89c33dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10647,9 +10647,9 @@ path-scurry@^1.11.1: minipass "^5.0.0 || ^6.0.2 || ^7.0.0" path-to-regexp@^6.2.0: - version "6.2.2" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.2.tgz#324377a83e5049cbecadc5554d6a63a9a4866b36" - integrity sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw== + version "6.3.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz#2b6a26a337737a8e1416f9272ed0766b1c0389f4" + integrity sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ== path-type@^3.0.0: version "3.0.0" From 67a35f8e76642661287aeec86115e5b19b51fdbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:47:51 +0100 Subject: [PATCH 113/145] chore(deps): bump express from 4.19.2 to 4.21.0 in /docs (#7078) Bumps [express](https://github.com/expressjs/express) from 4.19.2 to 4.21.0. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/4.21.0/History.md) - [Commits](https://github.com/expressjs/express/compare/4.19.2...4.21.0) --- updated-dependencies: - dependency-name: express dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/yarn.lock | 95 ++++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/docs/yarn.lock b/docs/yarn.lock index 41966c013183..c39bab969363 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -3136,10 +3136,10 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== -body-parser@1.20.2: - version "1.20.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" - integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== +body-parser@1.20.3: + version "1.20.3" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6" + integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g== dependencies: bytes "3.1.2" content-type "~1.0.5" @@ -3149,7 +3149,7 @@ body-parser@1.20.2: http-errors "2.0.0" iconv-lite "0.4.24" on-finished "2.4.1" - qs "6.11.0" + qs "6.13.0" raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" @@ -4454,6 +4454,11 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== +encodeurl@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" + integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== + enhanced-resolve@^5.15.0: version "5.15.1" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.1.tgz#384391e025f099e67b4b00bfd7f0906a408214e1" @@ -4659,36 +4664,36 @@ execa@^5.0.0: strip-final-newline "^2.0.0" express@^4.17.3: - version "4.19.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" - integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== + version "4.21.0" + resolved "https://registry.yarnpkg.com/express/-/express-4.21.0.tgz#d57cb706d49623d4ac27833f1cbc466b668eb915" + integrity sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.2" + body-parser "1.20.3" content-disposition "0.5.4" content-type "~1.0.4" cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "1.2.0" + finalhandler "1.3.1" fresh "0.5.2" http-errors "2.0.0" - merge-descriptors "1.0.1" + merge-descriptors "1.0.3" methods "~1.1.2" on-finished "2.4.1" parseurl "~1.3.3" - path-to-regexp "0.1.7" + path-to-regexp "0.1.10" proxy-addr "~2.0.7" - qs "6.11.0" + qs "6.13.0" range-parser "~1.2.1" safe-buffer "5.2.1" - send "0.18.0" - serve-static "1.15.0" + send "0.19.0" + serve-static "1.16.2" setprototypeof "1.2.0" statuses "2.0.1" type-is "~1.6.18" @@ -4783,13 +4788,13 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -finalhandler@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" - integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== +finalhandler@1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.3.1.tgz#0c575f1d1d324ddd1da35ad7ece3df7d19088019" + integrity sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ== dependencies: debug "2.6.9" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" on-finished "2.4.1" parseurl "~1.3.3" @@ -6345,10 +6350,10 @@ memfs@^3.1.2, memfs@^3.4.3: dependencies: fs-monkey "^1.0.4" -merge-descriptors@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" - integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w== +merge-descriptors@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz#d80319a65f3c7935351e5cfdac8f9318504dbed5" + integrity sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ== merge-stream@^2.0.0: version "2.0.0" @@ -7426,10 +7431,10 @@ path-parse@^1.0.7: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== -path-to-regexp@0.1.7: - version "0.1.7" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" - integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== +path-to-regexp@0.1.10: + version "0.1.10" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.10.tgz#67e9108c5c0551b9e5326064387de4763c4d5f8b" + integrity sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w== path-to-regexp@2.2.1: version "2.2.1" @@ -7854,12 +7859,12 @@ pupa@^3.1.0: dependencies: escape-goat "^4.0.0" -qs@6.11.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" - integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== +qs@6.13.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" + integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg== dependencies: - side-channel "^1.0.4" + side-channel "^1.0.6" queue-microtask@^1.2.2: version "1.2.3" @@ -8458,10 +8463,10 @@ semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.5.4: dependencies: lru-cache "^6.0.0" -send@0.18.0: - version "0.18.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" - integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== +send@0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8" + integrity sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw== dependencies: debug "2.6.9" depd "2.0.0" @@ -8511,15 +8516,15 @@ serve-index@^1.9.1: mime-types "~2.1.17" parseurl "~1.3.2" -serve-static@1.15.0: - version "1.15.0" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" - integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== +serve-static@1.16.2: + version "1.16.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.2.tgz#b6a5343da47f6bdd2673848bf45754941e803296" + integrity sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw== dependencies: - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.18.0" + send "0.19.0" set-function-length@^1.2.1: version "1.2.1" @@ -8581,7 +8586,7 @@ shelljs@^0.8.5: interpret "^1.0.0" rechoir "^0.6.2" -side-channel@^1.0.4: +side-channel@^1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== From 9fe72391bea93f7f88b7b739006844352d929cad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:51:12 +0100 Subject: [PATCH 114/145] chore(deps): bump micromatch from 4.0.5 to 4.0.8 (#7058) Bumps [micromatch](https://github.com/micromatch/micromatch) from 4.0.5 to 4.0.8. - [Release notes](https://github.com/micromatch/micromatch/releases) - [Changelog](https://github.com/micromatch/micromatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/micromatch/compare/4.0.5...4.0.8) --- updated-dependencies: - dependency-name: micromatch dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 7eb7a89c33dd..f9fa36847af9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4506,7 +4506,14 @@ brace-expansion@^2.0.1: dependencies: balanced-match "^1.0.0" -braces@^3.0.2, braces@~3.0.2: +braces@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== + dependencies: + fill-range "^7.1.1" + +braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -6768,6 +6775,13 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== + dependencies: + to-regex-range "^5.0.1" + find-my-way@^8.0.0: version "8.1.0" resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-8.1.0.tgz#cc05e8e4b145322299d0de0a839b5be528c2083e" @@ -9274,11 +9288,11 @@ micro-ftch@^0.3.1: integrity sha512-/0LLxhzP0tfiR5hcQebtudP56gUurs2CLkGarnCiB/OqEyUFQ6U3paQi/tgLv0hBJYt2rnr9MNpxz4fiiugstg== micromatch@^4.0.4: - version "4.0.5" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" - integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== dependencies: - braces "^3.0.2" + braces "^3.0.3" picomatch "^2.3.1" miller-rabin@^4.0.0: From 1bb98ed6db16e2d1ec2fb9f1f9a0ea8fafba7aac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:38:42 +0000 Subject: [PATCH 115/145] chore(deps): bump webpack from 5.90.3 to 5.94.0 in /docs (#7059) Bumps [webpack](https://github.com/webpack/webpack) from 5.90.3 to 5.94.0. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.90.3...v5.94.0) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/yarn.lock | 167 ++++++++++++++++++++++--------------------------- 1 file changed, 75 insertions(+), 92 deletions(-) diff --git a/docs/yarn.lock b/docs/yarn.lock index c39bab969363..06e7cf388568 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -2430,22 +2430,6 @@ dependencies: "@types/ms" "*" -"@types/eslint-scope@^3.7.3": - version "3.7.7" - resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.7.tgz#3108bd5f18b0cdb277c867b3dd449c9ed7079ac5" - integrity sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg== - dependencies: - "@types/eslint" "*" - "@types/estree" "*" - -"@types/eslint@*": - version "8.56.5" - resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-8.56.5.tgz#94b88cab77588fcecdd0771a6d576fa1c0af9d02" - integrity sha512-u5/YPJHo1tvkSF2CE0USEkxon82Z5DBy2xR+qfyYNszpX9qcs4sT6uq2kBbj4BXY1+DBGDPnrhMZV3pKWGNukw== - dependencies: - "@types/estree" "*" - "@types/json-schema" "*" - "@types/estree-jsx@^1.0.0": version "1.0.5" resolved "https://registry.yarnpkg.com/@types/estree-jsx/-/estree-jsx-1.0.5.tgz#858a88ea20f34fe65111f005a689fa1ebf70dc18" @@ -2536,7 +2520,7 @@ dependencies: "@types/istanbul-lib-report" "*" -"@types/json-schema@*", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": +"@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== @@ -2736,10 +2720,10 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@webassemblyjs/ast@1.11.6", "@webassemblyjs/ast@^1.11.5": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.6.tgz#db046555d3c413f8966ca50a95176a0e2c642e24" - integrity sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q== +"@webassemblyjs/ast@1.12.1", "@webassemblyjs/ast@^1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.12.1.tgz#bb16a0e8b1914f979f45864c23819cc3e3f0d4bb" + integrity sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg== dependencies: "@webassemblyjs/helper-numbers" "1.11.6" "@webassemblyjs/helper-wasm-bytecode" "1.11.6" @@ -2754,10 +2738,10 @@ resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz#6132f68c4acd59dcd141c44b18cbebbd9f2fa768" integrity sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q== -"@webassemblyjs/helper-buffer@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.6.tgz#b66d73c43e296fd5e88006f18524feb0f2c7c093" - integrity sha512-z3nFzdcp1mb8nEOFFk8DrYLpHvhKC3grJD2ardfKOzmbmJvEf/tPIqCY+sNcwZIY8ZD7IkB2l7/pqhUhqm7hLA== +"@webassemblyjs/helper-buffer@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz#6df20d272ea5439bf20ab3492b7fb70e9bfcb3f6" + integrity sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw== "@webassemblyjs/helper-numbers@1.11.6": version "1.11.6" @@ -2773,15 +2757,15 @@ resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz#bb2ebdb3b83aa26d9baad4c46d4315283acd51e9" integrity sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA== -"@webassemblyjs/helper-wasm-section@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.6.tgz#ff97f3863c55ee7f580fd5c41a381e9def4aa577" - integrity sha512-LPpZbSOwTpEC2cgn4hTydySy1Ke+XEu+ETXuoyvuyezHO3Kjdu90KK95Sh9xTbmjrCsUwvWwCOQQNta37VrS9g== +"@webassemblyjs/helper-wasm-section@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz#3da623233ae1a60409b509a52ade9bc22a37f7bf" + integrity sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g== dependencies: - "@webassemblyjs/ast" "1.11.6" - "@webassemblyjs/helper-buffer" "1.11.6" + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-buffer" "1.12.1" "@webassemblyjs/helper-wasm-bytecode" "1.11.6" - "@webassemblyjs/wasm-gen" "1.11.6" + "@webassemblyjs/wasm-gen" "1.12.1" "@webassemblyjs/ieee754@1.11.6": version "1.11.6" @@ -2802,59 +2786,59 @@ resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.6.tgz#90f8bc34c561595fe156603be7253cdbcd0fab5a" integrity sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA== -"@webassemblyjs/wasm-edit@^1.11.5": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.6.tgz#c72fa8220524c9b416249f3d94c2958dfe70ceab" - integrity sha512-Ybn2I6fnfIGuCR+Faaz7YcvtBKxvoLV3Lebn1tM4o/IAJzmi9AWYIPWpyBfU8cC+JxAO57bk4+zdsTjJR+VTOw== +"@webassemblyjs/wasm-edit@^1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz#9f9f3ff52a14c980939be0ef9d5df9ebc678ae3b" + integrity sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g== dependencies: - "@webassemblyjs/ast" "1.11.6" - "@webassemblyjs/helper-buffer" "1.11.6" + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-buffer" "1.12.1" "@webassemblyjs/helper-wasm-bytecode" "1.11.6" - "@webassemblyjs/helper-wasm-section" "1.11.6" - "@webassemblyjs/wasm-gen" "1.11.6" - "@webassemblyjs/wasm-opt" "1.11.6" - "@webassemblyjs/wasm-parser" "1.11.6" - "@webassemblyjs/wast-printer" "1.11.6" - -"@webassemblyjs/wasm-gen@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.6.tgz#fb5283e0e8b4551cc4e9c3c0d7184a65faf7c268" - integrity sha512-3XOqkZP/y6B4F0PBAXvI1/bky7GryoogUtfwExeP/v7Nzwo1QLcq5oQmpKlftZLbT+ERUOAZVQjuNVak6UXjPA== - dependencies: - "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/helper-wasm-section" "1.12.1" + "@webassemblyjs/wasm-gen" "1.12.1" + "@webassemblyjs/wasm-opt" "1.12.1" + "@webassemblyjs/wasm-parser" "1.12.1" + "@webassemblyjs/wast-printer" "1.12.1" + +"@webassemblyjs/wasm-gen@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz#a6520601da1b5700448273666a71ad0a45d78547" + integrity sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w== + dependencies: + "@webassemblyjs/ast" "1.12.1" "@webassemblyjs/helper-wasm-bytecode" "1.11.6" "@webassemblyjs/ieee754" "1.11.6" "@webassemblyjs/leb128" "1.11.6" "@webassemblyjs/utf8" "1.11.6" -"@webassemblyjs/wasm-opt@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.6.tgz#d9a22d651248422ca498b09aa3232a81041487c2" - integrity sha512-cOrKuLRE7PCe6AsOVl7WasYf3wbSo4CeOk6PkrjS7g57MFfVUF9u6ysQBBODX0LdgSvQqRiGz3CXvIDKcPNy4g== +"@webassemblyjs/wasm-opt@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz#9e6e81475dfcfb62dab574ac2dda38226c232bc5" + integrity sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg== dependencies: - "@webassemblyjs/ast" "1.11.6" - "@webassemblyjs/helper-buffer" "1.11.6" - "@webassemblyjs/wasm-gen" "1.11.6" - "@webassemblyjs/wasm-parser" "1.11.6" + "@webassemblyjs/ast" "1.12.1" + "@webassemblyjs/helper-buffer" "1.12.1" + "@webassemblyjs/wasm-gen" "1.12.1" + "@webassemblyjs/wasm-parser" "1.12.1" -"@webassemblyjs/wasm-parser@1.11.6", "@webassemblyjs/wasm-parser@^1.11.5": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.6.tgz#bb85378c527df824004812bbdb784eea539174a1" - integrity sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ== +"@webassemblyjs/wasm-parser@1.12.1", "@webassemblyjs/wasm-parser@^1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz#c47acb90e6f083391e3fa61d113650eea1e95937" + integrity sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ== dependencies: - "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/ast" "1.12.1" "@webassemblyjs/helper-api-error" "1.11.6" "@webassemblyjs/helper-wasm-bytecode" "1.11.6" "@webassemblyjs/ieee754" "1.11.6" "@webassemblyjs/leb128" "1.11.6" "@webassemblyjs/utf8" "1.11.6" -"@webassemblyjs/wast-printer@1.11.6": - version "1.11.6" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.11.6.tgz#a7bf8dd7e362aeb1668ff43f35cb849f188eff20" - integrity sha512-JM7AhRcE+yW2GWYaKeHL5vt4xqee5N2WcezptmgyhNS+ScggqcT1OtXykhAb13Sn5Yas0j2uv9tHgrjwvzAP4A== +"@webassemblyjs/wast-printer@1.12.1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz#bcecf661d7d1abdaf989d8341a4833e33e2b31ac" + integrity sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA== dependencies: - "@webassemblyjs/ast" "1.11.6" + "@webassemblyjs/ast" "1.12.1" "@xtuc/long" "4.2.2" "@xtuc/ieee754@^1.2.0": @@ -2875,10 +2859,10 @@ accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" -acorn-import-assertions@^1.9.0: - version "1.9.0" - resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac" - integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA== +acorn-import-attributes@^1.9.5: + version "1.9.5" + resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" + integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== acorn-jsx@^5.0.0: version "5.3.2" @@ -4459,10 +4443,10 @@ encodeurl@~2.0.0: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== -enhanced-resolve@^5.15.0: - version "5.15.1" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.15.1.tgz#384391e025f099e67b4b00bfd7f0906a408214e1" - integrity sha512-3d3JRbwsCLJsYgvb6NuWEG44jjPSOMuS73L/6+7BZuoKm3W+qXnSoIYVHi8dG7Qcg4inAY4jbzkZ7MnskePeDg== +enhanced-resolve@^5.17.1: + version "5.17.1" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz#67bfbbcc2f81d511be77d686a90267ef7f898a15" + integrity sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg== dependencies: graceful-fs "^4.2.4" tapable "^2.2.0" @@ -5076,7 +5060,7 @@ graceful-fs@4.2.10: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: +graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -9251,10 +9235,10 @@ vfile@^6.0.0, vfile@^6.0.1: unist-util-stringify-position "^4.0.0" vfile-message "^4.0.0" -watchpack@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.0.tgz#fa33032374962c78113f93c7f2fb4c54c9862a5d" - integrity sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg== +watchpack@^2.4.1: + version "2.4.2" + resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.2.tgz#2feeaed67412e7c33184e5a79ca738fbd38564da" + integrity sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw== dependencies: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2" @@ -9357,25 +9341,24 @@ webpack-sources@^3.2.2, webpack-sources@^3.2.3: integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== webpack@^5.88.1: - version "5.90.3" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.90.3.tgz#37b8f74d3ded061ba789bb22b31e82eed75bd9ac" - integrity sha512-h6uDYlWCctQRuXBs1oYpVe6sFcWedl0dpcVaTf/YF67J9bKvwJajFulMVSYKHrksMB3I/pIagRzDxwxkebuzKA== + version "5.94.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.94.0.tgz#77a6089c716e7ab90c1c67574a28da518a20970f" + integrity sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg== dependencies: - "@types/eslint-scope" "^3.7.3" "@types/estree" "^1.0.5" - "@webassemblyjs/ast" "^1.11.5" - "@webassemblyjs/wasm-edit" "^1.11.5" - "@webassemblyjs/wasm-parser" "^1.11.5" + "@webassemblyjs/ast" "^1.12.1" + "@webassemblyjs/wasm-edit" "^1.12.1" + "@webassemblyjs/wasm-parser" "^1.12.1" acorn "^8.7.1" - acorn-import-assertions "^1.9.0" + acorn-import-attributes "^1.9.5" browserslist "^4.21.10" chrome-trace-event "^1.0.2" - enhanced-resolve "^5.15.0" + enhanced-resolve "^5.17.1" es-module-lexer "^1.2.1" eslint-scope "5.1.1" events "^3.2.0" glob-to-regexp "^0.4.1" - graceful-fs "^4.2.9" + graceful-fs "^4.2.11" json-parse-even-better-errors "^2.3.1" loader-runner "^4.2.0" mime-types "^2.1.27" @@ -9383,7 +9366,7 @@ webpack@^5.88.1: schema-utils "^3.2.0" tapable "^2.1.1" terser-webpack-plugin "^5.3.10" - watchpack "^2.4.0" + watchpack "^2.4.1" webpack-sources "^3.2.3" webpackbar@^5.0.2: From 2fcecd61f51d076769d4146611e898ec83690948 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 22:28:24 +0100 Subject: [PATCH 116/145] chore(deps): bump dompurify from 3.0.9 to 3.1.6 in /docs (#7088) Bumps [dompurify](https://github.com/cure53/DOMPurify) from 3.0.9 to 3.1.6. - [Release notes](https://github.com/cure53/DOMPurify/releases) - [Commits](https://github.com/cure53/DOMPurify/compare/3.0.9...3.1.6) --- updated-dependencies: - dependency-name: dompurify dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/yarn.lock b/docs/yarn.lock index 06e7cf388568..27123932fe26 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -4346,9 +4346,9 @@ domhandler@^5.0.2, domhandler@^5.0.3: domelementtype "^2.3.0" dompurify@^3.0.5: - version "3.0.9" - resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.0.9.tgz#b3f362f24b99f53498c75d43ecbd784b0b3ad65e" - integrity sha512-uyb4NDIvQ3hRn6NiC+SIFaP4mJ/MdXlvtunaqK9Bn6dD3RuB/1S/gasEjDHD8eiaqdSael2vBv+hOs7Y+jhYOQ== + version "3.1.6" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.1.6.tgz#43c714a94c6a7b8801850f82e756685300a027e2" + integrity sha512-cTOAhc36AalkjtBpfG6O8JimdTMWNXjiePT2xQH/ppBGi/4uIpmj8eKyIkMJErXWARyINV/sB38yf8JCLF5pbQ== domutils@^2.5.2, domutils@^2.8.0: version "2.8.0" From 25837f61fed8089ebb8e06e505fea8e7a48b614e Mon Sep 17 00:00:00 2001 From: Phil Ngo Date: Tue, 17 Sep 2024 14:45:18 -0400 Subject: [PATCH 117/145] Merge branch 'stable' into unstable --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 8 ++++---- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 +++++++------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 12 ++++++------ packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 6 +++--- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 100 insertions(+), 100 deletions(-) diff --git a/lerna.json b/lerna.json index 043a5c48569a..0ffc65fe5402 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.21.0", + "version": "1.22.0", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 138964ed946a..d2f8a621f02e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -72,10 +72,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index b41450f12d34..ade89ebc6cc1 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -119,18 +119,18 @@ "@libp2p/peer-id-factory": "^4.1.0", "@libp2p/prometheus-metrics": "^3.0.21", "@libp2p/tcp": "9.0.23", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/db": "^1.21.0", - "@lodestar/fork-choice": "^1.21.0", - "@lodestar/light-client": "^1.21.0", - "@lodestar/logger": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/reqresp": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", - "@lodestar/validator": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/db": "^1.22.0", + "@lodestar/fork-choice": "^1.22.0", + "@lodestar/light-client": "^1.22.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/reqresp": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", + "@lodestar/validator": "^1.22.0", "@multiformats/multiaddr": "^12.1.3", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index bf8621344dc5..579725eba04e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.21.0", + "version": "1.22.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -62,17 +62,17 @@ "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", "@libp2p/peer-id-factory": "^4.1.0", - "@lodestar/api": "^1.21.0", - "@lodestar/beacon-node": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/db": "^1.21.0", - "@lodestar/light-client": "^1.21.0", - "@lodestar/logger": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", - "@lodestar/validator": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/beacon-node": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/db": "^1.22.0", + "@lodestar/light-client": "^1.22.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", + "@lodestar/validator": "^1.22.0", "@multiformats/multiaddr": "^12.1.3", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -88,7 +88,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "@types/debug": "^4.1.7", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", diff --git a/packages/config/package.json b/packages/config/package.json index d13713d376c6..f257db65cf6e 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.21.0", + "version": "1.22.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,8 +65,8 @@ ], "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/params": "^1.21.0", - "@lodestar/utils": "^1.21.0", - "@lodestar/types": "^1.21.0" + "@lodestar/params": "^1.22.0", + "@lodestar/utils": "^1.22.0", + "@lodestar/types": "^1.22.0" } } diff --git a/packages/db/package.json b/packages/db/package.json index 1a31603b6791..0e3a2c847d4b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.21.0", + "version": "1.22.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -36,12 +36,12 @@ }, "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/utils": "^1.22.0", "classic-level": "^1.4.1", "it-all": "^3.0.4" }, "devDependencies": { - "@lodestar/logger": "^1.21.0" + "@lodestar/logger": "^1.22.0" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index 461147274591..3a11e9b49ebf 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.21.0", + "version": "1.22.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/blst": "^2.0.3", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index dfe4c942832d..728036b365d3 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -37,11 +37,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0" + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 1188732c8870..940a78ae66d5 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -77,11 +77,11 @@ "@chainsafe/blst": "^0.2.0", "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/ssz": "^0.17.1", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "mitt": "^3.0.0" }, "devDependencies": { diff --git a/packages/logger/package.json b/packages/logger/package.json index 1ab176dbfd93..196fb306a8b0 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.21.0", + "@lodestar/utils": "^1.22.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "@types/triple-beam": "^1.3.2", "triple-beam": "^1.3.0" }, diff --git a/packages/params/package.json b/packages/params/package.json index c246918959e4..c281f97a41ec 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.21.0", + "version": "1.22.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index 974125e2a17f..6a31b5b1baa0 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/light-client": "^1.21.0", - "@lodestar/logger": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/light-client": "^1.22.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "ethereum-cryptography": "^2.0.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 1b9d99b000a8..44dcb8048f99 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -54,9 +54,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.3.0", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/utils": "^1.22.0", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -65,8 +65,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.21.0", - "@lodestar/types": "^1.21.0", + "@lodestar/logger": "^1.22.0", + "@lodestar/types": "^1.22.0", "libp2p": "1.4.3" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 166df67a7409..b4aabf0140e8 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.21.0", + "version": "1.22.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,7 +62,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.21.0", + "@lodestar/utils": "^1.22.0", "axios": "^1.3.4", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 1dd43ef98f88..2df87a522a63 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -63,10 +63,10 @@ "@chainsafe/persistent-merkle-tree": "^0.8.0", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.17.1", - "@lodestar/config": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/config": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "bigint-buffer": "^1.1.5", "immutable": "^4.3.2" }, diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 6c8c9e62894f..5bf61891a9d5 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.21.0", + "version": "1.22.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -59,8 +59,8 @@ "dependencies": { "@chainsafe/bls-keystore": "^3.1.0", "@chainsafe/blst": "^2.0.3", - "@lodestar/params": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/params": "^1.22.0", + "@lodestar/utils": "^1.22.0", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index 861dbdbee0ef..f3e034ec1b35 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": { ".": { @@ -74,7 +74,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.17.1", - "@lodestar/params": "^1.21.0", + "@lodestar/params": "^1.22.0", "ethereum-cryptography": "^2.0.0" }, "keywords": [ diff --git a/packages/utils/package.json b/packages/utils/package.json index 492c65606dfb..0723f5f1815c 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.21.0", + "version": "1.22.0", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index 373e59817d2a..65ed656f010d 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.21.0", + "version": "1.22.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -47,17 +47,17 @@ "dependencies": { "@chainsafe/blst": "^2.0.3", "@chainsafe/ssz": "^0.17.1", - "@lodestar/api": "^1.21.0", - "@lodestar/config": "^1.21.0", - "@lodestar/db": "^1.21.0", - "@lodestar/params": "^1.21.0", - "@lodestar/state-transition": "^1.21.0", - "@lodestar/types": "^1.21.0", - "@lodestar/utils": "^1.21.0", + "@lodestar/api": "^1.22.0", + "@lodestar/config": "^1.22.0", + "@lodestar/db": "^1.22.0", + "@lodestar/params": "^1.22.0", + "@lodestar/state-transition": "^1.22.0", + "@lodestar/types": "^1.22.0", + "@lodestar/utils": "^1.22.0", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.21.0", + "@lodestar/test-utils": "^1.22.0", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From 8da003e0f18523a1e6093177a147815365bd37e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 22:02:41 +0100 Subject: [PATCH 118/145] chore(deps-dev): bump vite from 5.3.4 to 5.3.6 (#7090) Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 5.3.4 to 5.3.6. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v5.3.6/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v5.3.6/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index f9fa36847af9..5ed1b898f00c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13136,9 +13136,9 @@ vite-plugin-top-level-await@^1.4.2: uuid "^10.0.0" vite@^5.0.0, vite@^5.3.4: - version "5.3.4" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.4.tgz#b36ebd47c8a5e3a8727046375d5f10bf9fdf8715" - integrity sha512-Cw+7zL3ZG9/NZBB8C+8QbQZmR54GwqIz+WMI4b3JgdYJvX+ny9AjJXqkGQlDXSXRP9rP0B4tbciRMOVEKulVOA== + version "5.3.6" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.3.6.tgz#e097c0a7b79adb2e60bec9ef7907354f09d027bd" + integrity sha512-es78AlrylO8mTVBygC0gTC0FENv0C6T496vvd33ydbjF/mIi9q3XQ9A3NWo5qLGFKywvz10J26813OkLvcQleA== dependencies: esbuild "^0.21.3" postcss "^8.4.39" From fd6e23edab7cf3ca70aa596aaeb440908fb28bc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 18:18:54 +0100 Subject: [PATCH 119/145] chore(deps): bump find-my-way from 8.1.0 to 8.2.2 (#7093) Bumps [find-my-way](https://github.com/delvedor/find-my-way) from 8.1.0 to 8.2.2. - [Release notes](https://github.com/delvedor/find-my-way/releases) - [Commits](https://github.com/delvedor/find-my-way/compare/v8.1.0...v8.2.2) --- updated-dependencies: - dependency-name: find-my-way dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/yarn.lock b/yarn.lock index 5ed1b898f00c..38e6bae97d60 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6783,13 +6783,13 @@ fill-range@^7.1.1: to-regex-range "^5.0.1" find-my-way@^8.0.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-8.1.0.tgz#cc05e8e4b145322299d0de0a839b5be528c2083e" - integrity sha512-41QwjCGcVTODUmLLqTMeoHeiozbMXYMAE1CKFiDyi9zVZ2Vjh0yz3MF0WQZoIb+cmzP/XlbFjlF2NtJmvZHznA== + version "8.2.2" + resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-8.2.2.tgz#f3e78bc6ead2da4fdaa201335da3228600ed0285" + integrity sha512-Dobi7gcTEq8yszimcfp/R7+owiT4WncAJ7VTTgFH1jYJ5GaG1FbhjwDG820hptN0QDFvzVY3RfCzdInvGPGzjA== dependencies: fast-deep-equal "^3.1.3" fast-querystring "^1.0.0" - safe-regex2 "^2.0.0" + safe-regex2 "^3.1.0" find-up@5.0.0, find-up@^5.0.0: version "5.0.0" @@ -11358,10 +11358,10 @@ restore-cursor@^4.0.0: onetime "^5.1.0" signal-exit "^3.0.2" -ret@~0.2.0: - version "0.2.2" - resolved "https://registry.npmjs.org/ret/-/ret-0.2.2.tgz" - integrity sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ== +ret@~0.4.0: + version "0.4.3" + resolved "https://registry.yarnpkg.com/ret/-/ret-0.4.3.tgz#5243fa30e704a2e78a9b9b1e86079e15891aa85c" + integrity sha512-0f4Memo5QP7WQyUEAYUO3esD/XjOc3Zjjg5CPsAq1p8sIu0XPeMbHJemKA0BO7tV0X7+A0FoEpbmHXWxPyD3wQ== retry@^0.12.0: version "0.12.0" @@ -11550,12 +11550,12 @@ safe-regex-test@^1.0.0: get-intrinsic "^1.1.3" is-regex "^1.1.4" -safe-regex2@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/safe-regex2/-/safe-regex2-2.0.0.tgz" - integrity sha512-PaUSFsUaNNuKwkBijoAPHAK6/eM6VirvyPWlZ7BAQy4D+hCvh4B6lIG+nPdhbFfIbP+gTGBcrdsOaUs0F+ZBOQ== +safe-regex2@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/safe-regex2/-/safe-regex2-3.1.0.tgz#fd7ec23908e2c730e1ce7359a5b72883a87d2763" + integrity sha512-RAAZAGbap2kBfbVhvmnTFv73NWLMvDGOITFYTZBAaY8eR+Ir4ef7Up/e7amo+y1+AH+3PtLkrt9mvcTsG9LXug== dependencies: - ret "~0.2.0" + ret "~0.4.0" safe-stable-stringify@^2.3.1: version "2.4.2" From bb40ef7eb77f9441ec66ab8f1fdf6664240bb1a3 Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Wed, 18 Sep 2024 15:09:07 -0400 Subject: [PATCH 120/145] feat: add electra support for light-client (#7063) * Add constant * Use constants * Remove ZERO_NEXT_SYNC_COMMITTEE_BRANCH * Add normalizeMerkleBranch * add getId to CheckpointHeaderRepository * fix: light-client unit tests * chore: lint * Fix normalizeMerkleBranch * Enable LC spec test * fix spec test --------- Co-authored-by: NC <17676176+ensi321@users.noreply.github.com> --- .../src/chain/lightClient/proofs.ts | 14 +++++--- .../lightclientCheckpointHeader.ts | 4 +++ .../beacon-node/src/network/reqresp/types.ts | 23 ++++++++++--- .../test/spec/utils/specTestIterator.ts | 5 ++- packages/light-client/src/spec/index.ts | 6 ++-- packages/light-client/src/spec/utils.ts | 32 +++++++++++++++++-- .../src/spec/validateLightClientBootstrap.ts | 8 +++-- .../src/spec/validateLightClientUpdate.ts | 16 ++++++---- .../src/utils/normalizeMerkleBranch.ts | 15 +++++++++ packages/light-client/src/validation.ts | 23 ++++++++++--- packages/light-client/test/unit/utils.test.ts | 16 ++++++++++ packages/params/src/index.ts | 6 ++-- packages/types/src/electra/sszTypes.ts | 20 ++++++------ packages/types/src/utils/typeguards.ts | 9 ++++++ 14 files changed, 155 insertions(+), 42 deletions(-) create mode 100644 packages/light-client/src/utils/normalizeMerkleBranch.ts diff --git a/packages/beacon-node/src/chain/lightClient/proofs.ts b/packages/beacon-node/src/chain/lightClient/proofs.ts index 87ad4544ec69..8d273e30ae5c 100644 --- a/packages/beacon-node/src/chain/lightClient/proofs.ts +++ b/packages/beacon-node/src/chain/lightClient/proofs.ts @@ -1,6 +1,11 @@ import {Tree} from "@chainsafe/persistent-merkle-tree"; -import {BeaconStateAllForks} from "@lodestar/state-transition"; -import {FINALIZED_ROOT_GINDEX, BLOCK_BODY_EXECUTION_PAYLOAD_GINDEX, ForkExecution} from "@lodestar/params"; +import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import { + FINALIZED_ROOT_GINDEX, + BLOCK_BODY_EXECUTION_PAYLOAD_GINDEX, + ForkExecution, + FINALIZED_ROOT_GINDEX_ELECTRA, +} from "@lodestar/params"; import {BeaconBlockBody, SSZTypesFor, ssz} from "@lodestar/types"; import {SyncCommitteeWitness} from "./types.js"; @@ -40,9 +45,10 @@ export function getCurrentSyncCommitteeBranch(syncCommitteesWitness: SyncCommitt return [syncCommitteesWitness.nextSyncCommitteeRoot, ...syncCommitteesWitness.witness]; } -export function getFinalizedRootProof(state: BeaconStateAllForks): Uint8Array[] { +export function getFinalizedRootProof(state: CachedBeaconStateAllForks): Uint8Array[] { state.commit(); - return new Tree(state.node).getSingleProof(BigInt(FINALIZED_ROOT_GINDEX)); + const finalizedRootGindex = state.epochCtx.isPostElectra() ? FINALIZED_ROOT_GINDEX_ELECTRA : FINALIZED_ROOT_GINDEX; + return new Tree(state.node).getSingleProof(BigInt(finalizedRootGindex)); } export function getBlockBodyExecutionHeaderProof( diff --git a/packages/beacon-node/src/db/repositories/lightclientCheckpointHeader.ts b/packages/beacon-node/src/db/repositories/lightclientCheckpointHeader.ts index 78f165bb975c..22d6559792eb 100644 --- a/packages/beacon-node/src/db/repositories/lightclientCheckpointHeader.ts +++ b/packages/beacon-node/src/db/repositories/lightclientCheckpointHeader.ts @@ -25,4 +25,8 @@ export class CheckpointHeaderRepository extends Repository; @@ -48,10 +61,10 @@ type ResponseBodyByMethod = { [ReqRespMethod.BeaconBlocksByRoot]: SignedBeaconBlock; [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecar; [ReqRespMethod.BlobSidecarsByRoot]: deneb.BlobSidecar; - [ReqRespMethod.LightClientBootstrap]: altair.LightClientBootstrap; - [ReqRespMethod.LightClientUpdatesByRange]: altair.LightClientUpdate; - [ReqRespMethod.LightClientFinalityUpdate]: altair.LightClientFinalityUpdate; - [ReqRespMethod.LightClientOptimisticUpdate]: altair.LightClientOptimisticUpdate; + [ReqRespMethod.LightClientBootstrap]: LightClientBootstrap; + [ReqRespMethod.LightClientUpdatesByRange]: LightClientUpdate; + [ReqRespMethod.LightClientFinalityUpdate]: LightClientFinalityUpdate; + [ReqRespMethod.LightClientOptimisticUpdate]: LightClientOptimisticUpdate; }; /** Request SSZ type for each method and ForkName */ diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 48a002580043..d8b4f9c0574c 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -65,10 +65,9 @@ export const defaultSkipOpts: SkipOpts = { skippedTestSuites: [ /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, - /^electra\/light_client\/.*/, + /^electra\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, ], - // TODO Electra: Review this test in the next spec test release - skippedTests: [/^deneb\/light_client\/sync\/.*electra_fork.*/], + skippedTests: [], skippedRunners: ["merkle_proof", "networking"], }; diff --git a/packages/light-client/src/spec/index.ts b/packages/light-client/src/spec/index.ts index fc1a431129e8..0934e15b1c17 100644 --- a/packages/light-client/src/spec/index.ts +++ b/packages/light-client/src/spec/index.ts @@ -10,7 +10,7 @@ import { import {computeSyncPeriodAtSlot} from "../utils/index.js"; import {getSyncCommitteeAtPeriod, processLightClientUpdate, ProcessUpdateOpts} from "./processLightClientUpdate.js"; import {ILightClientStore, LightClientStore, LightClientStoreEvents} from "./store.js"; -import {ZERO_FINALITY_BRANCH, ZERO_HEADER, ZERO_NEXT_SYNC_COMMITTEE_BRANCH, ZERO_SYNC_COMMITTEE} from "./utils.js"; +import {ZERO_FINALITY_BRANCH, ZERO_HEADER, ZERO_SYNC_COMMITTEE, getZeroSyncCommitteeBranch} from "./utils.js"; export {isBetterUpdate, toLightClientUpdateSummary} from "./isBetterUpdate.js"; export type {LightClientUpdateSummary} from "./isBetterUpdate.js"; @@ -37,7 +37,7 @@ export class LightclientSpec { this.onUpdate(currentSlot, { attestedHeader: finalityUpdate.attestedHeader, nextSyncCommittee: ZERO_SYNC_COMMITTEE, - nextSyncCommitteeBranch: ZERO_NEXT_SYNC_COMMITTEE_BRANCH, + nextSyncCommitteeBranch: getZeroSyncCommitteeBranch(this.config.getForkName(finalityUpdate.signatureSlot)), finalizedHeader: finalityUpdate.finalizedHeader, finalityBranch: finalityUpdate.finalityBranch, syncAggregate: finalityUpdate.syncAggregate, @@ -49,7 +49,7 @@ export class LightclientSpec { this.onUpdate(currentSlot, { attestedHeader: optimisticUpdate.attestedHeader, nextSyncCommittee: ZERO_SYNC_COMMITTEE, - nextSyncCommitteeBranch: ZERO_NEXT_SYNC_COMMITTEE_BRANCH, + nextSyncCommitteeBranch: getZeroSyncCommitteeBranch(this.config.getForkName(optimisticUpdate.signatureSlot)), finalizedHeader: {beacon: ZERO_HEADER}, finalityBranch: ZERO_FINALITY_BRANCH, syncAggregate: optimisticUpdate.syncAggregate, diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 3e59cb14cfa0..872aa5d9f910 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -7,6 +7,9 @@ import { ForkName, BLOCK_BODY_EXECUTION_PAYLOAD_DEPTH as EXECUTION_PAYLOAD_DEPTH, BLOCK_BODY_EXECUTION_PAYLOAD_INDEX as EXECUTION_PAYLOAD_INDEX, + NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA, + isForkPostElectra, + FINALIZED_ROOT_DEPTH_ELECTRA, } from "@lodestar/params"; import { ssz, @@ -17,17 +20,18 @@ import { LightClientUpdate, BeaconBlockHeader, SyncCommittee, + isElectraLightClientUpdate, } from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {isValidMerkleBranch, computeEpochAtSlot, computeSyncPeriodAtSlot} from "../utils/index.js"; +import {normalizeMerkleBranch} from "../utils/normalizeMerkleBranch.js"; import {LightClientStore} from "./store.js"; export const GENESIS_SLOT = 0; export const ZERO_HASH = new Uint8Array(32); export const ZERO_PUBKEY = new Uint8Array(48); export const ZERO_SYNC_COMMITTEE = ssz.altair.SyncCommittee.defaultValue(); -export const ZERO_NEXT_SYNC_COMMITTEE_BRANCH = Array.from({length: NEXT_SYNC_COMMITTEE_DEPTH}, () => ZERO_HASH); export const ZERO_HEADER = ssz.phase0.BeaconBlockHeader.defaultValue(); export const ZERO_FINALITY_BRANCH = Array.from({length: FINALIZED_ROOT_DEPTH}, () => ZERO_HASH); /** From https://notes.ethereum.org/@vbuterin/extended_light_client_protocol#Optimistic-head-determining-function */ @@ -41,10 +45,19 @@ export function getSafetyThreshold(maxActiveParticipants: number): number { return Math.floor(maxActiveParticipants / SAFETY_THRESHOLD_FACTOR); } +export function getZeroSyncCommitteeBranch(fork: ForkName): Uint8Array[] { + const nextSyncCommitteeDepth = isForkPostElectra(fork) + ? NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA + : NEXT_SYNC_COMMITTEE_DEPTH; + + return Array.from({length: nextSyncCommitteeDepth}, () => ZERO_HASH); +} + export function isSyncCommitteeUpdate(update: LightClientUpdate): boolean { return ( // Fast return for when constructing full LightClientUpdate from partial updates - update.nextSyncCommitteeBranch !== ZERO_NEXT_SYNC_COMMITTEE_BRANCH && + update.nextSyncCommitteeBranch !== + getZeroSyncCommitteeBranch(isElectraLightClientUpdate(update) ? ForkName.electra : ForkName.altair) && update.nextSyncCommitteeBranch.some((branch) => !byteArrayEquals(branch, ZERO_HASH)) ); } @@ -160,7 +173,8 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC if (epoch < config.ELECTRA_FORK_EPOCH) { if ( (header as LightClientHeader).execution.depositRequestsRoot !== undefined || - (header as LightClientHeader).execution.withdrawalRequestsRoot !== undefined + (header as LightClientHeader).execution.withdrawalRequestsRoot !== undefined || + (header as LightClientHeader).execution.consolidationRequestsRoot !== undefined ) { return false; } @@ -184,6 +198,14 @@ export function upgradeLightClientUpdate( ): LightClientUpdate { update.attestedHeader = upgradeLightClientHeader(config, targetFork, update.attestedHeader); update.finalizedHeader = upgradeLightClientHeader(config, targetFork, update.finalizedHeader); + update.nextSyncCommitteeBranch = normalizeMerkleBranch( + update.nextSyncCommitteeBranch, + isForkPostElectra(targetFork) ? NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA : NEXT_SYNC_COMMITTEE_DEPTH + ); + update.finalityBranch = normalizeMerkleBranch( + update.finalityBranch, + isForkPostElectra(targetFork) ? FINALIZED_ROOT_DEPTH_ELECTRA : FINALIZED_ROOT_DEPTH + ); return update; } @@ -195,6 +217,10 @@ export function upgradeLightClientFinalityUpdate( ): LightClientFinalityUpdate { finalityUpdate.attestedHeader = upgradeLightClientHeader(config, targetFork, finalityUpdate.attestedHeader); finalityUpdate.finalizedHeader = upgradeLightClientHeader(config, targetFork, finalityUpdate.finalizedHeader); + finalityUpdate.finalityBranch = normalizeMerkleBranch( + finalityUpdate.finalityBranch, + isForkPostElectra(targetFork) ? FINALIZED_ROOT_DEPTH_ELECTRA : FINALIZED_ROOT_DEPTH + ); return finalityUpdate; } diff --git a/packages/light-client/src/spec/validateLightClientBootstrap.ts b/packages/light-client/src/spec/validateLightClientBootstrap.ts index 30540da24bd1..2eafea0791f0 100644 --- a/packages/light-client/src/spec/validateLightClientBootstrap.ts +++ b/packages/light-client/src/spec/validateLightClientBootstrap.ts @@ -2,11 +2,14 @@ import {byteArrayEquals} from "@chainsafe/ssz"; import {LightClientBootstrap, Root, ssz} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {toHex} from "@lodestar/utils"; +import {isForkPostElectra} from "@lodestar/params"; import {isValidMerkleBranch} from "../utils/verifyMerkleBranch.js"; import {isValidLightClientHeader} from "./utils.js"; const CURRENT_SYNC_COMMITTEE_INDEX = 22; const CURRENT_SYNC_COMMITTEE_DEPTH = 5; +const CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA = 22; +const CURRENT_SYNC_COMMITTEE_DEPTH_ELECTRA = 6; export function validateLightClientBootstrap( config: ChainForkConfig, @@ -14,6 +17,7 @@ export function validateLightClientBootstrap( bootstrap: LightClientBootstrap ): void { const headerRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(bootstrap.header.beacon); + const fork = config.getForkName(bootstrap.header.beacon.slot); if (!isValidLightClientHeader(config, bootstrap.header)) { throw Error("Bootstrap Header is not Valid Light Client Header"); @@ -27,8 +31,8 @@ export function validateLightClientBootstrap( !isValidMerkleBranch( ssz.altair.SyncCommittee.hashTreeRoot(bootstrap.currentSyncCommittee), bootstrap.currentSyncCommitteeBranch, - CURRENT_SYNC_COMMITTEE_DEPTH, - CURRENT_SYNC_COMMITTEE_INDEX, + isForkPostElectra(fork) ? CURRENT_SYNC_COMMITTEE_DEPTH_ELECTRA : CURRENT_SYNC_COMMITTEE_DEPTH, + isForkPostElectra(fork) ? CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA : CURRENT_SYNC_COMMITTEE_INDEX, bootstrap.header.beacon.stateRoot ) ) { diff --git a/packages/light-client/src/spec/validateLightClientUpdate.ts b/packages/light-client/src/spec/validateLightClientUpdate.ts index fde760da3b05..9a5ea1985f16 100644 --- a/packages/light-client/src/spec/validateLightClientUpdate.ts +++ b/packages/light-client/src/spec/validateLightClientUpdate.ts @@ -1,15 +1,19 @@ import bls from "@chainsafe/bls"; import type {PublicKey, Signature} from "@chainsafe/bls/types"; -import {LightClientUpdate, Root, ssz} from "@lodestar/types"; +import {LightClientUpdate, Root, isElectraLightClientUpdate, ssz} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import { FINALIZED_ROOT_INDEX, FINALIZED_ROOT_DEPTH, - NEXT_SYNC_COMMITTEE_INDEX, NEXT_SYNC_COMMITTEE_DEPTH, MIN_SYNC_COMMITTEE_PARTICIPANTS, DOMAIN_SYNC_COMMITTEE, GENESIS_SLOT, + NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA, + NEXT_SYNC_COMMITTEE_INDEX_ELECTRA, + NEXT_SYNC_COMMITTEE_INDEX, + FINALIZED_ROOT_DEPTH_ELECTRA, + FINALIZED_ROOT_INDEX_ELECTRA, } from "@lodestar/params"; import {getParticipantPubkeys, sumBits} from "../utils/utils.js"; import {isValidMerkleBranch} from "../utils/index.js"; @@ -78,8 +82,8 @@ export function validateLightClientUpdate( !isValidMerkleBranch( finalizedRoot, update.finalityBranch, - FINALIZED_ROOT_DEPTH, - FINALIZED_ROOT_INDEX, + isElectraLightClientUpdate(update) ? FINALIZED_ROOT_DEPTH_ELECTRA : FINALIZED_ROOT_DEPTH, + isElectraLightClientUpdate(update) ? FINALIZED_ROOT_INDEX_ELECTRA : FINALIZED_ROOT_INDEX, update.attestedHeader.beacon.stateRoot ) ) { @@ -98,8 +102,8 @@ export function validateLightClientUpdate( !isValidMerkleBranch( ssz.altair.SyncCommittee.hashTreeRoot(update.nextSyncCommittee), update.nextSyncCommitteeBranch, - NEXT_SYNC_COMMITTEE_DEPTH, - NEXT_SYNC_COMMITTEE_INDEX, + isElectraLightClientUpdate(update) ? NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA : NEXT_SYNC_COMMITTEE_DEPTH, + isElectraLightClientUpdate(update) ? NEXT_SYNC_COMMITTEE_INDEX_ELECTRA : NEXT_SYNC_COMMITTEE_INDEX, update.attestedHeader.beacon.stateRoot ) ) { diff --git a/packages/light-client/src/utils/normalizeMerkleBranch.ts b/packages/light-client/src/utils/normalizeMerkleBranch.ts new file mode 100644 index 000000000000..ae3309f8ff2e --- /dev/null +++ b/packages/light-client/src/utils/normalizeMerkleBranch.ts @@ -0,0 +1,15 @@ +import {ZERO_HASH} from "../spec/utils.js"; + +export const SYNC_COMMITTEES_DEPTH = 4; +export const SYNC_COMMITTEES_INDEX = 11; + +/** + * Given merkle branch ``branch``, extend its depth according to ``depth`` + * If given ``depth`` is less than the depth of ``branch``, it will return + * unmodified ``branch`` + */ +export function normalizeMerkleBranch(branch: Uint8Array[], depth: number): Uint8Array[] { + const numExtraDepth = depth - branch.length; + + return [...Array.from({length: numExtraDepth}, () => ZERO_HASH), ...branch]; +} diff --git a/packages/light-client/src/validation.ts b/packages/light-client/src/validation.ts index e7839f115153..c756d612f3e7 100644 --- a/packages/light-client/src/validation.ts +++ b/packages/light-client/src/validation.ts @@ -1,6 +1,14 @@ import bls from "@chainsafe/bls"; import type {PublicKey, Signature} from "@chainsafe/bls/types"; -import {altair, LightClientFinalityUpdate, LightClientUpdate, Root, Slot, ssz} from "@lodestar/types"; +import { + altair, + isElectraLightClientUpdate, + LightClientFinalityUpdate, + LightClientUpdate, + Root, + Slot, + ssz, +} from "@lodestar/types"; import { FINALIZED_ROOT_INDEX, FINALIZED_ROOT_DEPTH, @@ -8,6 +16,9 @@ import { NEXT_SYNC_COMMITTEE_DEPTH, MIN_SYNC_COMMITTEE_PARTICIPANTS, DOMAIN_SYNC_COMMITTEE, + NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA, + FINALIZED_ROOT_DEPTH_ELECTRA, + NEXT_SYNC_COMMITTEE_INDEX_ELECTRA, } from "@lodestar/params"; import {BeaconConfig} from "@lodestar/config"; import {isValidMerkleBranch} from "./utils/verifyMerkleBranch.js"; @@ -39,7 +50,11 @@ export function assertValidLightClientUpdate( if (isFinalized) { assertValidFinalityProof(update); } else { - assertZeroHashes(update.finalityBranch, FINALIZED_ROOT_DEPTH, "finalityBranches"); + assertZeroHashes( + update.finalityBranch, + isElectraLightClientUpdate(update) ? FINALIZED_ROOT_DEPTH_ELECTRA : FINALIZED_ROOT_DEPTH, + "finalityBranches" + ); } // DIFF FROM SPEC: @@ -99,8 +114,8 @@ export function assertValidSyncCommitteeProof(update: LightClientUpdate): void { !isValidMerkleBranch( ssz.altair.SyncCommittee.hashTreeRoot(update.nextSyncCommittee), update.nextSyncCommitteeBranch, - NEXT_SYNC_COMMITTEE_DEPTH, - NEXT_SYNC_COMMITTEE_INDEX, + isElectraLightClientUpdate(update) ? NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA : NEXT_SYNC_COMMITTEE_DEPTH, + isElectraLightClientUpdate(update) ? NEXT_SYNC_COMMITTEE_INDEX_ELECTRA : NEXT_SYNC_COMMITTEE_INDEX, update.attestedHeader.beacon.stateRoot ) ) { diff --git a/packages/light-client/test/unit/utils.test.ts b/packages/light-client/test/unit/utils.test.ts index 91bfab113431..9913c6c462a4 100644 --- a/packages/light-client/test/unit/utils.test.ts +++ b/packages/light-client/test/unit/utils.test.ts @@ -1,6 +1,8 @@ import {describe, it, expect} from "vitest"; import {isValidMerkleBranch} from "../../src/utils/verifyMerkleBranch.js"; import {computeMerkleBranch} from "../utils/utils.js"; +import {normalizeMerkleBranch} from "../../src/utils/normalizeMerkleBranch.js"; +import {ZERO_HASH} from "../../src/spec/utils.js"; describe("utils", () => { it("constructMerkleBranch", () => { @@ -11,4 +13,18 @@ describe("utils", () => { expect(isValidMerkleBranch(leaf, proof, depth, index, root)).toBe(true); }); + it("normalizeMerkleBranch", () => { + const branch: Uint8Array[] = []; + const branchDepth = 5; + const newDepth = 7; + + for (let i = 0; i < branchDepth; i++) { + branch.push(new Uint8Array(Array.from({length: 32}, () => i))); + } + + const normalizedBranch = normalizeMerkleBranch(branch, newDepth); + const expectedNormalizedBranch = [ZERO_HASH, ZERO_HASH, ...branch]; + + expect(normalizedBranch).toEqual(expectedNormalizedBranch); + }); }); diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index e7fd5b976336..a44fd5d0e0cd 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -264,7 +264,9 @@ export const BLOBSIDECAR_FIXED_SIZE = ACTIVE_PRESET === PresetName.minimal ? 131 // Electra Misc export const UNSET_DEPOSIT_REQUESTS_START_INDEX = 2n ** 64n - 1n; export const FULL_EXIT_REQUEST_AMOUNT = 0; +export const FINALIZED_ROOT_GINDEX_ELECTRA = 169; +export const FINALIZED_ROOT_DEPTH_ELECTRA = 7; +export const FINALIZED_ROOT_INDEX_ELECTRA = 41; export const NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA = 87; export const NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA = 6; -export const FINALIZED_ROOT_DEPTH_ELECTRA = 7; -export const FINALIZED_ROOT_INDEX_ELECTRA = 169; +export const NEXT_SYNC_COMMITTEE_INDEX_ELECTRA = 23; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index c0485597686a..522f7245cc1b 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -358,20 +358,20 @@ export const LightClientHeader = new ContainerType( export const LightClientBootstrap = new ContainerType( { - header: LightClientHeader, + header: LightClientHeader, // Modified in ELECTRA currentSyncCommittee: altairSsz.SyncCommittee, - currentSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), + currentSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), // Modified in ELECTRA }, {typeName: "LightClientBootstrap", jsonCase: "eth2"} ); export const LightClientUpdate = new ContainerType( { - attestedHeader: LightClientHeader, + attestedHeader: LightClientHeader, // Modified in ELECTRA nextSyncCommittee: altairSsz.SyncCommittee, - nextSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), - finalizedHeader: LightClientHeader, - finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), + nextSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), // Modified in ELECTRA + finalizedHeader: LightClientHeader, // Modified in ELECTRA + finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), // Modified in ELECTRA syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, }, @@ -382,7 +382,7 @@ export const LightClientFinalityUpdate = new ContainerType( { attestedHeader: LightClientHeader, finalizedHeader: LightClientHeader, - finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), + finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), // Modified in ELECTRA syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, }, @@ -391,7 +391,7 @@ export const LightClientFinalityUpdate = new ContainerType( export const LightClientOptimisticUpdate = new ContainerType( { - attestedHeader: LightClientHeader, + attestedHeader: LightClientHeader, // Modified in ELECTRA syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, }, @@ -400,8 +400,8 @@ export const LightClientOptimisticUpdate = new ContainerType( export const LightClientStore = new ContainerType( { - snapshot: LightClientBootstrap, - validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), + snapshot: LightClientBootstrap, // Modified in ELECTRA + validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), // Modified in ELECTRA }, {typeName: "LightClientStore", jsonCase: "eth2"} ); diff --git a/packages/types/src/utils/typeguards.ts b/packages/types/src/utils/typeguards.ts index 72910645f6e1..a5f5b7808ae5 100644 --- a/packages/types/src/utils/typeguards.ts +++ b/packages/types/src/utils/typeguards.ts @@ -14,6 +14,7 @@ import { SignedBlockContents, BeaconBlock, Attestation, + LightClientUpdate, } from "../types.js"; export function isExecutionPayload( @@ -71,3 +72,11 @@ export function isSignedBlockContents( export function isElectraAttestation(attestation: Attestation): attestation is Attestation { return (attestation as Attestation).committeeBits !== undefined; } + +export function isElectraLightClientUpdate(update: LightClientUpdate): update is LightClientUpdate { + const updatePostElectra = update as LightClientUpdate; + return ( + updatePostElectra.attestedHeader.execution !== undefined && + updatePostElectra.attestedHeader.execution.depositRequestsRoot !== undefined + ); +} From 404f13abfd56a859a35c69f198178d5669d3867a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Sat, 21 Sep 2024 18:53:39 +0100 Subject: [PATCH 121/145] chore: update Teku holesky bootnode (#7099) --- packages/cli/src/networks/holesky.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/networks/holesky.ts b/packages/cli/src/networks/holesky.ts index b86f6b543582..63bc6e07f8f2 100644 --- a/packages/cli/src/networks/holesky.ts +++ b/packages/cli/src/networks/holesky.ts @@ -10,7 +10,7 @@ export const bootEnrs = [ "enr:-Ku4QPG7F72mbKx3gEQEx07wpYYusGDh-ni6SNkLvOS-hhN-BxIggN7tKlmalb0L5JPoAfqD-akTZ-gX06hFeBEz4WoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpAhnTT-AQFwAP__________gmlkgnY0gmlwhJK-DYCJc2VjcDI1NmsxoQKLVXFOhp2uX6jeT0DvvDpPcU8FWMjQdR4wMuORMhpX24N1ZHCCIyk", "enr:-LK4QPxe-mDiSOtEB_Y82ozvxn9aQM07Ui8A-vQHNgYGMMthfsfOabaaTHhhJHFCBQQVRjBww_A5bM1rf8MlkJU_l68Eh2F0dG5ldHOIAADAAAAAAACEZXRoMpBpt9l0BAFwAAABAAAAAAAAgmlkgnY0gmlwhLKAiOmJc2VjcDI1NmsxoQJu6T9pclPObAzEVQ53DpVQqjadmVxdTLL-J3h9NFoCeIN0Y3CCIyiDdWRwgiMo", "enr:-Ly4QGbOw4xNel5EhmDsJJ-QhC9XycWtsetnWoZ0uRy381GHdHsNHJiCwDTOkb3S1Ade0SFQkWJX_pgb3g8Jfh93rvMBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBpt9l0BAFwAAABAAAAAAAAgmlkgnY0gmlwhJK-DYCJc2VjcDI1NmsxoQOxKv9sv3zKF8GDewgFGGHKP5HCZZpPpTrwl9eXKAWGxIhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA", - "enr:-LS4QG0uV4qvcpJ-HFDJRGBmnlD3TJo7yc4jwK8iP7iKaTlfQ5kZvIDspLMJhk7j9KapuL9yyHaZmwTEZqr10k9XumyCEcmHYXR0bmV0c4gAAAAABgAAAIRldGgykGm32XQEAXAAAAEAAAAAAACCaWSCdjSCaXCErK4j-YlzZWNwMjU2azGhAgfWRBEJlb7gAhXIB5ePmjj2b8io0UpEenq1Kl9cxStJg3RjcIIjKIN1ZHCCIyg", + "enr:-KO4QCi3ZY4TM5KL7bAG6laSYiYelDWu0crvUjCXlyc_cwEfUpMIuARuMJYGxWe-UYYpHEw_aBbZ1u-4tHQ8imyI5uaCAsGEZXRoMpBprg6ZBQFwAP__________gmlkgnY0gmlwhKyuI_mJc2VjcDI1NmsxoQLoFG5-vuNX6N49vnkTBaA3ZsBDF8B30DGqWOGtRGz5w4N0Y3CCIyiDdWRwgiMo", "enr:-Le4QLoE1wFHSlGcm48a9ZESb_MRLqPPu6G0vHqu4MaUcQNDHS69tsy-zkN0K6pglyzX8m24mkb-LtBcbjAYdP1uxm4BhGV0aDKQabfZdAQBcAAAAQAAAAAAAIJpZIJ2NIJpcIQ5gR6Wg2lwNpAgAUHQBwEQAAAAAAAAADR-iXNlY3AyNTZrMaEDPMSNdcL92uNIyCsS177Z6KTXlbZakQqxv3aQcWawNXeDdWRwgiMohHVkcDaCI4I", "enr:-KG4QC9Wm32mtzB5Fbj2ri2TEKglHmIWgvwTQCvNHBopuwpNAi1X6qOsBg_Z1-Bee-kfSrhzUQZSgDUyfH5outUprtoBgmlkgnY0gmlwhHEel3eDaXA2kP6AAAAAAAAAAlBW__4Srr-Jc2VjcDI1NmsxoQO7KE63Z4eSI55S1Yn7q9_xFkJ1Wt-a3LgiXuKGs19s0YN1ZHCCIyiEdWRwNoIjKA", ]; From cd98c237683456be7959d752bbd7d10f8b02b8ec Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Sun, 22 Sep 2024 21:41:47 -0400 Subject: [PATCH 122/145] feat: async shuffling refactor (#6938) * feat: add ShufflingCache to EpochCache * fix: implementation in state-transition for EpochCache with ShufflingCache * feat: remove shufflingCache.processState * feat: implement ShufflingCache changes in beacon-node * feat: pass shufflingCache when loading cached state from db * test: fix state-transition tests for EpochCache changes * feat: Pass shufflingCache to EpochCache at startup * test: fix slot off by one for decision root in perf test * chore: use ?. syntax * chore: refactoring * feat: add comments and clean up afterProcessEpoch * fix: perf test slot incrementing * fix: remove MockShufflingCache * Revert "chore: refactoring" This reverts commit 104aa56d84ce3ca33d045705591767ef578dec3a. * refactor: shufflingCache getters * refactor: shufflingCache setters * refactor: build and getOrBuild * docs: add comments to ShufflingCache methods * chore: lint issues * test: update tests in beacon-node * chore: lint * feat: get shufflings from cache for API * feat: minTimeDelayToBuildShuffling cli flag * test: fix shufflingCache promise insertion test * fix: rebase conflicts * fix: changes from debugging sim tests * refactor: minimize changes in afterProcessEpoch * chore: fix lint * chore: fix check-types * chore: fix check-types * feat: add diff utility * fix: bug in spec tests from invalid nextActiveIndices * refactor: add/remove comments * refactor: remove this.activeIndicesLength from EpochCache * refactor: simplify shufflingCache.getSync * refactor: remove unnecessary undefined's * refactor: clean up ShufflingCache unit test * feat: add metrics for ShufflingCache * feat: add shufflingCache metrics to state-transition * chore: lint * fix: metric name clash * refactor: add comment about not having ShufflingCache in EpochCache * refactor: rename shuffling decision root functions * refactor: remove unused comment * feat: async add nextShuffling to EpochCache after its built * feat: make ShufflingCache.set private * feat: chance metrics to nextShufflingNotOnEpochCache instead of positive case * refactor: move diff to separate PR * chore: fix tests using shufflingCache.set method * feat: remove minTimeDelayToBuild * feat: return promise from insertPromise and then through build * fix: update metrics names and help field * feat: move build of shuffling to beforeProcessEpoch * feat: allow calc of pivot slot before slot increment * fix: calc of pivot slot before slot increment * Revert "fix: calc of pivot slot before slot increment" This reverts commit 5e65f7efa85fad169a6263d1bebc1b4f8d3701d4. * Revert "feat: allow calc of pivot slot before slot increment" This reverts commit ed850ee0ced72029b643db4e86d82600cfb7cf43. * feat: allow getting current block root for shuffling calculation * fix: get nextShufflingDecisionRoot directly from state.blockRoots * fix: convert toRootHex * docs: add comment about pulling decisionRoot directly from state * feat: add back metrics for regen attestation cache hit/miss * docs: fix docstring on shufflingCache.build * refactor: change validatorIndices to Uint32Array * refactor: remove comment and change variable name * fix: use toRootHex instead of toHexString * refactor: deduplicate moved function computeAnchorCheckpoint * fix: touch up metrics per PR comments * fix: merge conflict * chore: lint * refactor: add scope around activeIndices to GC arrays * feat: directly use Uint32Array instead of transcribing number array to Uint32Array * refactor: activeIndices per tuyen comment * refactor: rename to epochAfterNext * chore: review PR * feat: update no shuffling ApiError to 500 status * fix: add back unnecessary eslint directive. to be remove under separate PR * feat: update no shuffling ApiError to 500 status * docs: add comment about upcomingEpoch --------- Co-authored-by: Cayman Co-authored-by: Tuyen Nguyen --- .../src/api/impl/beacon/state/index.ts | 9 +- .../src/api/impl/validator/index.ts | 11 +- .../src/chain/blocks/importBlock.ts | 7 - packages/beacon-node/src/chain/chain.ts | 25 +- .../beacon-node/src/chain/forkChoice/index.ts | 2 +- packages/beacon-node/src/chain/initState.ts | 38 +- .../beacon-node/src/chain/shufflingCache.ts | 183 ++++++---- .../stateCache/persistentCheckpointsCache.ts | 15 +- .../src/metrics/metrics/lodestar.ts | 47 ++- .../src/node/utils/interop/state.ts | 1 + packages/beacon-node/src/node/utils/state.ts | 3 + .../beacon-node/src/sync/backfill/backfill.ts | 3 +- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../unit/chain/forkChoice/forkChoice.test.ts | 3 +- .../test/unit/chain/shufflingCache.test.ts | 42 +-- packages/beacon-node/test/utils/state.ts | 1 + .../test/utils/validationData/attestation.ts | 27 +- packages/params/src/index.ts | 2 + .../state-transition/src/cache/epochCache.ts | 336 +++++++++++++----- .../src/cache/epochTransitionCache.ts | 40 ++- .../state-transition/src/cache/stateCache.ts | 3 +- .../src/epoch/processSyncCommitteeUpdates.ts | 2 +- .../src/slot/upgradeStateToAltair.ts | 2 +- .../state-transition/src/stateTransition.ts | 19 + .../src/util/calculateCommitteeAssignments.ts | 43 +++ .../src/util/computeAnchorCheckpoint.ts | 38 ++ .../src/util/epochShuffling.ts | 83 ++++- packages/state-transition/src/util/index.ts | 2 + .../test/perf/epoch/epochAltair.test.ts | 6 +- .../test/perf/epoch/epochCapella.test.ts | 6 +- .../test/perf/epoch/epochPhase0.test.ts | 6 +- .../perf/util/loadState/loadState.test.ts | 6 +- .../test/perf/util/shufflings.test.ts | 15 +- .../test/unit/cachedBeaconState.test.ts | 34 +- 34 files changed, 700 insertions(+), 362 deletions(-) create mode 100644 packages/state-transition/src/util/calculateCommitteeAssignments.ts create mode 100644 packages/state-transition/src/util/computeAnchorCheckpoint.ts diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index 9d9646ee8cf3..77e2fd5aab46 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -202,7 +202,14 @@ export function getBeaconStateApi({ const epoch = filters.epoch ?? computeEpochAtSlot(state.slot); const startSlot = computeStartSlotAtEpoch(epoch); - const shuffling = stateCached.epochCtx.getShufflingAtEpoch(epoch); + const decisionRoot = stateCached.epochCtx.getShufflingDecisionRoot(epoch); + const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); + if (!shuffling) { + throw new ApiError( + 500, + `No shuffling found to calculate committees for epoch: ${epoch} and decisionRoot: ${decisionRoot}` + ); + } const committees = shuffling.committees; const committeesFlat = committees.flatMap((slotCommittees, slotInEpoch) => { const slot = startSlot + slotInEpoch; diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index a17c1418809e..d2ce6672c89f 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -3,6 +3,7 @@ import {ApplicationMethods} from "@lodestar/api/server"; import { CachedBeaconStateAllForks, computeStartSlotAtEpoch, + calculateCommitteeAssignments, proposerShufflingDecisionRoot, attesterShufflingDecisionRoot, getBlockRootAtSlot, @@ -995,7 +996,15 @@ export function getValidatorApi( // Check that all validatorIndex belong to the state before calling getCommitteeAssignments() const pubkeys = getPubkeysForIndices(state.validators, indices); - const committeeAssignments = state.epochCtx.getCommitteeAssignments(epoch, indices); + const decisionRoot = state.epochCtx.getShufflingDecisionRoot(epoch); + const shuffling = await chain.shufflingCache.get(epoch, decisionRoot); + if (!shuffling) { + throw new ApiError( + 500, + `No shuffling found to calculate committee assignments for epoch: ${epoch} and decisionRoot: ${decisionRoot}` + ); + } + const committeeAssignments = calculateCommitteeAssignments(shuffling, indices); const duties: routes.validator.AttesterDuty[] = []; for (let i = 0, len = indices.length; i < len; i++) { const validatorIndex = indices[i]; diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 4c46bdae53ee..d19d6a60c564 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -64,7 +64,6 @@ export async function importBlock( const blockRootHex = toRootHex(blockRoot); const currentEpoch = computeEpochAtSlot(this.forkChoice.getTime()); const blockEpoch = computeEpochAtSlot(blockSlot); - const parentEpoch = computeEpochAtSlot(parentBlockSlot); const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch; const blockDelaySec = (fullyVerifiedBlock.seenTimestampSec - postState.genesisTime) % this.config.SECONDS_PER_SLOT; const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000); @@ -335,12 +334,6 @@ export async function importBlock( this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot}); } - if (parentEpoch < blockEpoch) { - // current epoch and previous epoch are likely cached in previous states - this.shufflingCache.processState(postState, postState.epochCtx.nextShuffling.epoch); - this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot}); - } - if (blockSlot % SLOTS_PER_EPOCH === 0) { // Cache state to preserve epoch transition work const checkpointState = postState; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index ee70231c7d40..0b0ac30edf40 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -13,6 +13,7 @@ import { PubkeyIndexMap, EpochShuffling, computeEndSlotAtEpoch, + computeAnchorCheckpoint, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import { @@ -60,7 +61,6 @@ import { import {IChainOptions} from "./options.js"; import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; -import {computeAnchorCheckpoint} from "./initState.js"; import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls/index.js"; import { SeenAttesters, @@ -246,7 +246,6 @@ export class BeaconChain implements IBeaconChain { this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); - this.shufflingCache = new ShufflingCache(metrics, this.opts); // Restore state caches // anchorState may already by a CachedBeaconState. If so, don't create the cache again, since deserializing all @@ -261,9 +260,21 @@ export class BeaconChain implements IBeaconChain { pubkey2index: new PubkeyIndexMap(), index2pubkey: [], }); - this.shufflingCache.processState(cachedState, cachedState.epochCtx.previousShuffling.epoch); - this.shufflingCache.processState(cachedState, cachedState.epochCtx.currentShuffling.epoch); - this.shufflingCache.processState(cachedState, cachedState.epochCtx.nextShuffling.epoch); + + this.shufflingCache = cachedState.epochCtx.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [ + { + shuffling: cachedState.epochCtx.previousShuffling, + decisionRoot: cachedState.epochCtx.previousDecisionRoot, + }, + { + shuffling: cachedState.epochCtx.currentShuffling, + decisionRoot: cachedState.epochCtx.currentDecisionRoot, + }, + { + shuffling: cachedState.epochCtx.nextShuffling, + decisionRoot: cachedState.epochCtx.nextDecisionRoot, + }, + ]); // Persist single global instance of state caches this.pubkey2index = cachedState.epochCtx.pubkey2index; @@ -902,8 +913,8 @@ export class BeaconChain implements IBeaconChain { state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller); } - // resolve the promise to unblock other calls of the same epoch and dependent root - return this.shufflingCache.processState(state, attEpoch); + // should always be the current epoch of the active context so no need to await a result from the ShufflingCache + return state.epochCtx.getShufflingAtEpoch(attEpoch); } /** diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index d57b7f86cb98..346a4afe1e7f 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -14,10 +14,10 @@ import { getEffectiveBalanceIncrementsZeroInactive, isExecutionStateType, isMergeTransitionComplete, + computeAnchorCheckpoint, } from "@lodestar/state-transition"; import {Logger, toRootHex} from "@lodestar/utils"; -import {computeAnchorCheckpoint} from "../initState.js"; import {ChainEventEmitter} from "../emitter.js"; import {ChainEvent} from "../emitter.js"; import {GENESIS_SLOT} from "../../constants/index.js"; diff --git a/packages/beacon-node/src/chain/initState.ts b/packages/beacon-node/src/chain/initState.ts index e413bdff0f7d..311806fb1be7 100644 --- a/packages/beacon-node/src/chain/initState.ts +++ b/packages/beacon-node/src/chain/initState.ts @@ -1,15 +1,13 @@ import { - blockToHeader, computeEpochAtSlot, BeaconStateAllForks, CachedBeaconStateAllForks, - computeCheckpointEpochAtStateSlot, computeStartSlotAtEpoch, } from "@lodestar/state-transition"; -import {SignedBeaconBlock, phase0, ssz} from "@lodestar/types"; +import {SignedBeaconBlock} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {Logger, toHex, toRootHex} from "@lodestar/utils"; -import {GENESIS_SLOT, ZERO_HASH} from "../constants/index.js"; +import {GENESIS_SLOT} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; import {Eth1Provider} from "../eth1/index.js"; import {Metrics} from "../metrics/index.js"; @@ -204,35 +202,3 @@ export function initBeaconMetrics(metrics: Metrics, state: BeaconStateAllForks): metrics.currentJustifiedEpoch.set(state.currentJustifiedCheckpoint.epoch); metrics.finalizedEpoch.set(state.finalizedCheckpoint.epoch); } - -export function computeAnchorCheckpoint( - config: ChainForkConfig, - anchorState: BeaconStateAllForks -): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader} { - let blockHeader; - let root; - const blockTypes = config.getForkTypes(anchorState.latestBlockHeader.slot); - - if (anchorState.latestBlockHeader.slot === GENESIS_SLOT) { - const block = blockTypes.BeaconBlock.defaultValue(); - block.stateRoot = anchorState.hashTreeRoot(); - blockHeader = blockToHeader(config, block); - root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); - } else { - blockHeader = ssz.phase0.BeaconBlockHeader.clone(anchorState.latestBlockHeader); - if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { - blockHeader.stateRoot = anchorState.hashTreeRoot(); - } - root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); - } - - return { - checkpoint: { - root, - // the checkpoint epoch = computeEpochAtSlot(anchorState.slot) + 1 if slot is not at epoch boundary - // this is similar to a process_slots() call - epoch: computeCheckpointEpochAtStateSlot(anchorState.slot), - }, - blockHeader, - }; -} diff --git a/packages/beacon-node/src/chain/shufflingCache.ts b/packages/beacon-node/src/chain/shufflingCache.ts index 12dd1bf3e9ae..6c42228b5356 100644 --- a/packages/beacon-node/src/chain/shufflingCache.ts +++ b/packages/beacon-node/src/chain/shufflingCache.ts @@ -1,9 +1,14 @@ -import {CachedBeaconStateAllForks, EpochShuffling, getShufflingDecisionBlock} from "@lodestar/state-transition"; -import {Epoch, RootHex, ssz} from "@lodestar/types"; -import {MapDef, pruneSetToMax, toRootHex} from "@lodestar/utils"; -import {GENESIS_SLOT} from "@lodestar/params"; +import { + BeaconStateAllForks, + EpochShuffling, + IShufflingCache, + ShufflingBuildProps, + computeEpochShuffling, +} from "@lodestar/state-transition"; +import {Epoch, RootHex} from "@lodestar/types"; +import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; -import {computeAnchorCheckpoint} from "./initState.js"; +import {callInNextEventLoop} from "../util/eventLoop.js"; /** * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case: @@ -31,6 +36,7 @@ type ShufflingCacheItem = { type PromiseCacheItem = { type: CacheItemType.promise; + timeInsertedMs: number; promise: Promise; resolveFn: (shuffling: EpochShuffling) => void; }; @@ -47,7 +53,7 @@ export type ShufflingCacheOpts = { * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice * - skip computing shuffling when loading state bytes from disk */ -export class ShufflingCache { +export class ShufflingCache implements IShufflingCache { /** LRU cache implemented as a map, pruned every time we add an item */ private readonly itemsByDecisionRootByEpoch: MapDef> = new MapDef( () => new Map() @@ -56,8 +62,10 @@ export class ShufflingCache { private readonly maxEpochs: number; constructor( - private readonly metrics: Metrics | null = null, - opts: ShufflingCacheOpts = {} + readonly metrics: Metrics | null = null, + readonly logger: Logger | null = null, + opts: ShufflingCacheOpts = {}, + precalculatedShufflings?: {shuffling: EpochShuffling | null; decisionRoot: RootHex}[] ) { if (metrics) { metrics.shufflingCache.size.addCollect(() => @@ -68,66 +76,25 @@ export class ShufflingCache { } this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS; - } - - /** - * Extract shuffling from state and add to cache - */ - processState(state: CachedBeaconStateAllForks, shufflingEpoch: Epoch): EpochShuffling { - const decisionBlockHex = getDecisionBlock(state, shufflingEpoch); - let shuffling: EpochShuffling; - switch (shufflingEpoch) { - case state.epochCtx.nextShuffling.epoch: - shuffling = state.epochCtx.nextShuffling; - break; - case state.epochCtx.currentShuffling.epoch: - shuffling = state.epochCtx.currentShuffling; - break; - case state.epochCtx.previousShuffling.epoch: - shuffling = state.epochCtx.previousShuffling; - break; - default: - throw new Error(`Shuffling not found from state ${state.slot} for epoch ${shufflingEpoch}`); - } - let cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionBlockHex); - if (cacheItem !== undefined) { - // update existing promise - if (isPromiseCacheItem(cacheItem)) { - // unblock consumers of this promise - cacheItem.resolveFn(shuffling); - // then update item type to shuffling - cacheItem = { - type: CacheItemType.shuffling, - shuffling, - }; - this.add(shufflingEpoch, decisionBlockHex, cacheItem); - // we updated type to CacheItemType.shuffling so the above fields are not used anyway - this.metrics?.shufflingCache.processStateUpdatePromise.inc(); - } else { - // ShufflingCacheItem, do nothing - this.metrics?.shufflingCache.processStateNoOp.inc(); + precalculatedShufflings?.map(({shuffling, decisionRoot}) => { + if (shuffling !== null) { + this.set(shuffling, decisionRoot); } - } else { - // not found, new shuffling - this.add(shufflingEpoch, decisionBlockHex, {type: CacheItemType.shuffling, shuffling}); - this.metrics?.shufflingCache.processStateInsertNew.inc(); - } - - return shuffling; + }); } /** * Insert a promise to make sure we don't regen state for the same shuffling. * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up. */ - insertPromise(shufflingEpoch: Epoch, decisionRootHex: RootHex): void { + insertPromise(epoch: Epoch, decisionRoot: RootHex): void { const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values()) .flatMap((innerMap) => Array.from(innerMap.values())) .filter((item) => isPromiseCacheItem(item)).length; if (promiseCount >= MAX_PROMISES) { throw new Error( - `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRootHex}` + `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${epoch}, decisionRootHex: ${decisionRoot}` ); } let resolveFn: ((shuffling: EpochShuffling) => void) | null = null; @@ -140,10 +107,11 @@ export class ShufflingCache { const cacheItem: PromiseCacheItem = { type: CacheItemType.promise, + timeInsertedMs: Date.now(), promise, resolveFn, }; - this.add(shufflingEpoch, decisionRootHex, cacheItem); + this.itemsByDecisionRootByEpoch.getOrDefault(epoch).set(decisionRoot, cacheItem); this.metrics?.shufflingCache.insertPromiseCount.inc(); } @@ -152,39 +120,95 @@ export class ShufflingCache { * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve. * Return null if we don't have a shuffling for this epoch and dependentRootHex. */ - async get(shufflingEpoch: Epoch, decisionRootHex: RootHex): Promise { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); + async get(epoch: Epoch, decisionRoot: RootHex): Promise { + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); if (cacheItem === undefined) { + this.metrics?.shufflingCache.miss.inc(); return null; } if (isShufflingCacheItem(cacheItem)) { + this.metrics?.shufflingCache.hit.inc(); return cacheItem.shuffling; } else { - // promise + this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc(); return cacheItem.promise; } } /** - * Same to get() function but synchronous. + * Gets a cached shuffling via the epoch and decision root. If the shuffling is not + * available it will build it synchronously and return the shuffling. + * + * NOTE: If a shuffling is already queued and not calculated it will build and resolve + * the promise but the already queued build will happen at some later time */ - getSync(shufflingEpoch: Epoch, decisionRootHex: RootHex): EpochShuffling | null { - const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex); - if (cacheItem === undefined) { - return null; + getSync( + epoch: Epoch, + decisionRoot: RootHex, + buildProps?: T + ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null { + const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot); + if (!cacheItem) { + this.metrics?.shufflingCache.miss.inc(); + } else if (isShufflingCacheItem(cacheItem)) { + this.metrics?.shufflingCache.hit.inc(); + return cacheItem.shuffling; + } else if (buildProps) { + // TODO: (@matthewkeil) This should possible log a warning?? + this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway.inc(); + } else { + this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc(); } - if (isShufflingCacheItem(cacheItem)) { - return cacheItem.shuffling; + let shuffling: EpochShuffling | null = null; + if (buildProps) { + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "getSync"}); + shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch); + timer?.(); + this.set(shuffling, decisionRoot); } + return shuffling as T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; + } - // ignore promise - return null; + /** + * Queue asynchronous build for an EpochShuffling, triggered from state-transition + */ + build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: Uint32Array): void { + this.insertPromise(epoch, decisionRoot); + /** + * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations + * on a NICE thread with a rust implementation + */ + callInNextEventLoop(() => { + const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "build"}); + const shuffling = computeEpochShuffling(state, activeIndices, epoch); + timer?.(); + this.set(shuffling, decisionRoot); + }); } - private add(shufflingEpoch: Epoch, decisionBlock: RootHex, cacheItem: CacheItem): void { - this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionBlock, cacheItem); + /** + * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will + * resolve the promise with the built shuffling + */ + private set(shuffling: EpochShuffling, decisionRoot: string): void { + const shufflingAtEpoch = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch); + // if a pending shuffling promise exists, resolve it + const cacheItem = shufflingAtEpoch.get(decisionRoot); + if (cacheItem) { + if (isPromiseCacheItem(cacheItem)) { + cacheItem.resolveFn(shuffling); + this.metrics?.shufflingCache.shufflingPromiseResolutionTime.observe( + (Date.now() - cacheItem.timeInsertedMs) / 1000 + ); + } else { + this.metrics?.shufflingCache.shufflingBuiltMultipleTimes.inc(); + } + } + // set the shuffling + shufflingAtEpoch.set(decisionRoot, {type: CacheItemType.shuffling, shuffling}); + // prune the cache pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs); } } @@ -197,13 +221,14 @@ function isPromiseCacheItem(item: CacheItem): item is PromiseCacheItem { return item.type === CacheItemType.promise; } -/** - * Get the shuffling decision block root for the given epoch of given state - * - Special case close to genesis block, return the genesis block root - * - This is similar to forkchoice.getDependentRoot() function, otherwise we cannot get cached shuffing in attestation verification when syncing from genesis. - */ -function getDecisionBlock(state: CachedBeaconStateAllForks, epoch: Epoch): RootHex { - return state.slot > GENESIS_SLOT - ? getShufflingDecisionBlock(state, epoch) - : toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(state.config, state).blockHeader)); +export enum ShufflingCacheErrorCode { + NO_SHUFFLING_FOUND = "SHUFFLING_CACHE_ERROR_NO_SHUFFLING_FOUND", } + +type ShufflingCacheErrorType = { + code: ShufflingCacheErrorCode.NO_SHUFFLING_FOUND; + epoch: Epoch; + decisionRoot: RootHex; +}; + +export class ShufflingCacheError extends LodestarError {} diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index d8769374a29c..74e85e6fa574 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -212,20 +212,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } sszTimer?.(); const timer = this.metrics?.stateReloadDuration.startTimer(); - const newCachedState = loadCachedBeaconState( - seedState, - stateBytes, - { - shufflingGetter: (shufflingEpoch, decisionRootHex) => { - const shuffling = this.shufflingCache.getSync(shufflingEpoch, decisionRootHex); - if (shuffling == null) { - this.metrics?.stateReloadShufflingCacheMiss.inc(); - } - return shuffling; - }, - }, - validatorsBytes - ); + const newCachedState = loadCachedBeaconState(seedState, stateBytes, {}, validatorsBytes); newCachedState.commit(); const stateRoot = toRootHex(newCachedState.hashTreeRoot()); timer?.(); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 737a900e5f64..4b076471dcab 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1297,22 +1297,45 @@ export function createLodestarMetrics( name: "lodestar_shuffling_cache_size", help: "Shuffling cache size", }), - processStateInsertNew: register.gauge({ - name: "lodestar_shuffling_cache_process_state_insert_new_total", - help: "Total number of times processState is called resulting a new shuffling", - }), - processStateUpdatePromise: register.gauge({ - name: "lodestar_shuffling_cache_process_state_update_promise_total", - help: "Total number of times processState is called resulting a promise being updated with shuffling", - }), - processStateNoOp: register.gauge({ - name: "lodestar_shuffling_cache_process_state_no_op_total", - help: "Total number of times processState is called resulting no changes", - }), insertPromiseCount: register.gauge({ name: "lodestar_shuffling_cache_insert_promise_count", help: "Total number of times insertPromise is called", }), + hit: register.gauge({ + name: "lodestar_shuffling_cache_hit_count", + help: "Count of shuffling cache hit", + }), + miss: register.gauge({ + name: "lodestar_shuffling_cache_miss_count", + help: "Count of shuffling cache miss", + }), + shufflingBuiltMultipleTimes: register.gauge({ + name: "lodestar_shuffling_cache_recalculated_shuffling_count", + help: "Count of shuffling that were build multiple times", + }), + shufflingPromiseNotResolvedAndThrownAway: register.gauge({ + name: "lodestar_shuffling_cache_promise_not_resolved_and_thrown_away_count", + help: "Count of shuffling cache promises that were discarded and the shuffling was built synchronously", + }), + shufflingPromiseNotResolved: register.gauge({ + name: "lodestar_shuffling_cache_promise_not_resolved_count", + help: "Count of shuffling cache promises that were requested before the promise was resolved", + }), + nextShufflingNotOnEpochCache: register.gauge({ + name: "lodestar_shuffling_cache_next_shuffling_not_on_epoch_cache", + help: "The next shuffling was not on the epoch cache before the epoch transition", + }), + shufflingPromiseResolutionTime: register.histogram({ + name: "lodestar_shuffling_cache_promise_resolution_time_seconds", + help: "Time from promise insertion until promise resolution when shuffling was ready in seconds", + buckets: [0.5, 1, 1.5, 2], + }), + shufflingCalculationTime: register.histogram<{source: "build" | "getSync"}>({ + name: "lodestar_shuffling_cache_shuffling_calculation_time_seconds", + help: "Run time of shuffling calculation", + buckets: [0.5, 0.75, 1, 1.25, 1.5], + labelNames: ["source"], + }), }, seenCache: { diff --git a/packages/beacon-node/src/node/utils/interop/state.ts b/packages/beacon-node/src/node/utils/interop/state.ts index 6528bd392bc7..fe26afef2013 100644 --- a/packages/beacon-node/src/node/utils/interop/state.ts +++ b/packages/beacon-node/src/node/utils/interop/state.ts @@ -20,6 +20,7 @@ export type InteropStateOpts = { withEth1Credentials?: boolean; }; +// TODO: (@matthewkeil) - Only used by initDevState. Consider combining into that function export function getInteropState( config: ChainForkConfig, { diff --git a/packages/beacon-node/src/node/utils/state.ts b/packages/beacon-node/src/node/utils/state.ts index 25bd77c82274..05da7042eef4 100644 --- a/packages/beacon-node/src/node/utils/state.ts +++ b/packages/beacon-node/src/node/utils/state.ts @@ -5,6 +5,9 @@ import {IBeaconDb} from "../../db/index.js"; import {interopDeposits} from "./interop/deposits.js"; import {getInteropState, InteropStateOpts} from "./interop/state.js"; +/** + * Builds state for `dev` command, for sim testing and some other tests + */ export function initDevState( config: ChainForkConfig, validatorCount: number, diff --git a/packages/beacon-node/src/sync/backfill/backfill.ts b/packages/beacon-node/src/sync/backfill/backfill.ts index 38258fde07fd..77d2836bdcc3 100644 --- a/packages/beacon-node/src/sync/backfill/backfill.ts +++ b/packages/beacon-node/src/sync/backfill/backfill.ts @@ -1,6 +1,6 @@ import {EventEmitter} from "events"; import {StrictEventEmitter} from "strict-event-emitter-types"; -import {BeaconStateAllForks, blockToHeader} from "@lodestar/state-transition"; +import {BeaconStateAllForks, blockToHeader, computeAnchorCheckpoint} from "@lodestar/state-transition"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {phase0, Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types"; import {ErrorAborted, Logger, sleep, toRootHex} from "@lodestar/utils"; @@ -15,7 +15,6 @@ import {PeerIdStr} from "../../util/peerId.js"; import {shuffleOne} from "../../util/shuffle.js"; import {Metrics} from "../../metrics/metrics"; import {byteArrayEquals} from "../../util/bytes.js"; -import {computeAnchorCheckpoint} from "../../chain/initState.js"; import {verifyBlockProposerSignature, verifyBlockSequence, BackfillBlockHeader, BackfillBlock} from "./verify.js"; import {BackfillSyncError, BackfillSyncErrorCode} from "./errors.js"; /** diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 63fc4ee2e12c..45fc07281c3b 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -2,6 +2,7 @@ import {itBench} from "@dapplion/benchmark"; import {BitArray, toHexString} from "@chainsafe/ssz"; import { CachedBeaconStateAltair, + computeAnchorCheckpoint, computeEpochAtSlot, computeStartSlotAtEpoch, getBlockRootAtSlot, @@ -15,7 +16,6 @@ import {createChainForkConfig, defaultChainConfig} from "@lodestar/config"; // eslint-disable-next-line import/no-relative-packages import {generatePerfTestCachedStateAltair} from "../../../../../state-transition/test/perf/util.js"; import {AggregatedAttestationPool} from "../../../../src/chain/opPools/aggregatedAttestationPool.js"; -import {computeAnchorCheckpoint} from "../../../../src/chain/initState.js"; const vc = 1_500_000; diff --git a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts index 6b96a0d1172f..611673086ce5 100644 --- a/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts +++ b/packages/beacon-node/test/unit/chain/forkChoice/forkChoice.test.ts @@ -5,12 +5,13 @@ import {CheckpointWithHex, ExecutionStatus, ForkChoice, DataAvailabilityStatus} import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import { CachedBeaconStateAllForks, + computeAnchorCheckpoint, computeEpochAtSlot, getEffectiveBalanceIncrementsZeroed, } from "@lodestar/state-transition"; import {phase0, Slot, ssz, ValidatorIndex} from "@lodestar/types"; import {getTemporaryBlockHeader, processSlots} from "@lodestar/state-transition"; -import {ChainEventEmitter, computeAnchorCheckpoint, initializeForkChoice} from "../../../../src/chain/index.js"; +import {ChainEventEmitter, initializeForkChoice} from "../../../../src/chain/index.js"; import {generateSignedBlockAtSlot} from "../../../utils/typeGenerator.js"; import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState.js"; import {generateState} from "../../../utils/state.js"; diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 6295a993c072..62b02cbf2b12 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,6 +1,4 @@ import {describe, it, expect, beforeEach} from "vitest"; - -import {getShufflingDecisionBlock} from "@lodestar/state-transition"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; @@ -9,39 +7,43 @@ describe("ShufflingCache", function () { const vc = 64; const stateSlot = 100; const state = generateTestCachedBeaconStateOnlyValidators({vc, slot: stateSlot}); - const currentEpoch = state.epochCtx.currentShuffling.epoch; + const currentEpoch = state.epochCtx.epoch; + const currentDecisionRoot = state.epochCtx.currentDecisionRoot; let shufflingCache: ShufflingCache; beforeEach(() => { - shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.processState(state, currentEpoch); + shufflingCache = new ShufflingCache(null, null, {maxShufflingCacheEpochs: 1}, [ + { + shuffling: state.epochCtx.currentShuffling, + decisionRoot: currentDecisionRoot, + }, + ]); }); it("should get shuffling from cache", async function () { - const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toEqual(state.epochCtx.currentShuffling); }); it("should bound by maxSize(=1)", async function () { - const decisionRoot = getShufflingDecisionBlock(state, currentEpoch); - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert promises at the same epoch does not prune the cache shufflingCache.insertPromise(currentEpoch, "0x00"); - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); - // insert shufflings at other epochs does prune the cache - shufflingCache.processState(state, currentEpoch + 1); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toEqual(state.epochCtx.currentShuffling); + // insert shuffling at other epochs does prune the cache + shufflingCache["set"](state.epochCtx.previousShuffling, state.epochCtx.previousDecisionRoot); // the current shuffling is not available anymore - expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); + expect(await shufflingCache.get(currentEpoch, currentDecisionRoot)).toBeNull(); }); it("should return shuffling from promise", async function () { - const nextDecisionRoot = getShufflingDecisionBlock(state, currentEpoch + 1); - shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); - const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - shufflingCache.processState(state, currentEpoch + 1); - expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); - expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); + const previousEpoch = state.epochCtx.epoch - 1; + const previousDecisionRoot = state.epochCtx.previousDecisionRoot; + shufflingCache.insertPromise(previousEpoch, previousDecisionRoot); + const shufflingRequest0 = shufflingCache.get(previousEpoch, previousDecisionRoot); + const shufflingRequest1 = shufflingCache.get(previousEpoch, previousDecisionRoot); + shufflingCache["set"](state.epochCtx.previousShuffling, previousDecisionRoot); + expect(await shufflingRequest0).toEqual(state.epochCtx.previousShuffling); + expect(await shufflingRequest1).toEqual(state.epochCtx.previousShuffling); }); it("should support up to 2 promises at a time", async function () { diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index bb55adca1eb0..49a27435bdd7 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -106,6 +106,7 @@ export function generateState( /** * This generates state with default pubkey + * TODO: (@matthewkeil) - this is duplicated and exists in state-transition as well */ export function generateCachedState(opts?: TestBeaconState): CachedBeaconStateAllForks { const config = getConfig(ForkName.phase0); diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index c33d942dabc5..22f551cbb663 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -1,10 +1,5 @@ import {BitArray, toHexString} from "@chainsafe/ssz"; -import { - computeEpochAtSlot, - computeSigningRoot, - computeStartSlotAtEpoch, - getShufflingDecisionBlock, -} from "@lodestar/state-transition"; +import {computeEpochAtSlot, computeSigningRoot, computeStartSlotAtEpoch} from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; import {phase0, Slot, ssz} from "@lodestar/types"; @@ -81,10 +76,20 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { dataAvailabilityStatus: DataAvailabilityStatus.PreData, }; - const shufflingCache = new ShufflingCache(); - shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); - shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); - const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); + const shufflingCache = new ShufflingCache(null, null, {}, [ + { + shuffling: state.epochCtx.previousShuffling, + decisionRoot: state.epochCtx.previousDecisionRoot, + }, + { + shuffling: state.epochCtx.currentShuffling, + decisionRoot: state.epochCtx.currentDecisionRoot, + }, + { + shuffling: state.epochCtx.nextShuffling, + decisionRoot: state.epochCtx.nextDecisionRoot, + }, + ]); const forkChoice = { getBlock: (root) => { @@ -95,7 +100,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { if (rootHex !== toHexString(beaconBlockRoot)) return null; return headBlock; }, - getDependentRoot: () => dependentRoot, + getDependentRoot: () => state.epochCtx.currentDecisionRoot, } as Partial as IForkChoice; const committeeIndices = state.epochCtx.getBeaconCommittee(attSlot, attIndex); diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index a44fd5d0e0cd..dd5c24ac3a54 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -124,6 +124,8 @@ export const FAR_FUTURE_EPOCH = Infinity; export const BASE_REWARDS_PER_EPOCH = 4; export const DEPOSIT_CONTRACT_TREE_DEPTH = 2 ** 5; // 32 export const JUSTIFICATION_BITS_LENGTH = 4; +export const ZERO_HASH = Buffer.alloc(32, 0); +export const ZERO_HASH_HEX = "0x" + "00".repeat(32); // Withdrawal prefixes // Since the prefixes are just 1 byte, we define and use them as number diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index ba8ec9b05ab5..3783c7f91f0d 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -7,6 +7,7 @@ import { Slot, ValidatorIndex, phase0, + RootHex, SyncPeriod, Attestation, IndexedAttestation, @@ -37,12 +38,19 @@ import { computeProposers, getActivationChurnLimit, } from "../util/index.js"; -import {computeEpochShuffling, EpochShuffling, getShufflingDecisionBlock} from "../util/epochShuffling.js"; +import { + computeEpochShuffling, + EpochShuffling, + calculateShufflingDecisionRoot, + IShufflingCache, +} from "../util/epochShuffling.js"; import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js"; import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js"; import {getTotalSlashingsByIncrement} from "../epoch/processSlashings.js"; +import {AttesterDuty, calculateCommitteeAssignments} from "../util/calculateCommitteeAssignments.js"; import {EpochCacheMetrics} from "../metrics.js"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js"; +import {BeaconStateAllForks, BeaconStateAltair} from "./types.js"; import { Index2PubkeyCache, PubkeyIndexMap, @@ -52,13 +60,13 @@ import { PubkeyHex, newUnfinalizedPubkeyIndexMap, } from "./pubkeyCache.js"; -import {BeaconStateAllForks, BeaconStateAltair, ShufflingGetter} from "./types.js"; import { computeSyncCommitteeCache, getSyncCommitteeCache, SyncCommitteeCache, SyncCommitteeCacheEmpty, } from "./syncCommitteeCache.js"; +import {CachedBeaconStateAllForks} from "./stateCache.js"; /** `= PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)` */ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT); @@ -67,12 +75,12 @@ export type EpochCacheImmutableData = { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; + shufflingCache?: IShufflingCache; }; export type EpochCacheOpts = { skipSyncCommitteeCache?: boolean; skipSyncPubkeys?: boolean; - shufflingGetter?: ShufflingGetter; }; /** Defers computing proposers by persisting only the seed, and dropping it once indexes are computed */ @@ -127,6 +135,11 @@ export class EpochCache { * Unique pubkey registry shared in the same fork. There should only exist one for the fork. */ unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + /** + * ShufflingCache is passed in from `beacon-node` so should be available at runtime but may not be + * present during testing. + */ + shufflingCache?: IShufflingCache; /** * Indexes of the block proposers for the current epoch. @@ -145,6 +158,12 @@ export class EpochCache { */ proposersNextEpoch: ProposersDeferred; + /** + * Epoch decision roots to look up correct shuffling from the Shuffling Cache + */ + previousDecisionRoot: RootHex; + currentDecisionRoot: RootHex; + nextDecisionRoot: RootHex; /** * Shuffling of validator indexes. Immutable through the epoch, then it's replaced entirely. * Note: Per spec definition, shuffling will always be defined. They are never called before loadState() @@ -155,7 +174,12 @@ export class EpochCache { /** Same as previousShuffling */ currentShuffling: EpochShuffling; /** Same as previousShuffling */ - nextShuffling: EpochShuffling; + nextShuffling: EpochShuffling | null; + /** + * Cache nextActiveIndices so that in afterProcessEpoch the next shuffling can be build synchronously + * in case it is not built or the ShufflingCache is not available + */ + nextActiveIndices: Uint32Array; /** * Effective balances, for altair processAttestations() */ @@ -227,7 +251,6 @@ export class EpochCache { nextSyncCommitteeIndexed: SyncCommitteeCache; // TODO: Helper stats - epoch: Epoch; syncPeriod: SyncPeriod; /** * state.validators.length of every state at epoch boundary @@ -239,17 +262,28 @@ export class EpochCache { */ historicalValidatorLengths: immutable.List; + epoch: Epoch; + + get nextEpoch(): Epoch { + return this.epoch + 1; + } + constructor(data: { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + shufflingCache?: IShufflingCache; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; + previousDecisionRoot: RootHex; + currentDecisionRoot: RootHex; + nextDecisionRoot: RootHex; previousShuffling: EpochShuffling; currentShuffling: EpochShuffling; - nextShuffling: EpochShuffling; + nextShuffling: EpochShuffling | null; + nextActiveIndices: Uint32Array; effectiveBalanceIncrements: EffectiveBalanceIncrements; totalSlashingsByIncrement: number; syncParticipantReward: number; @@ -272,12 +306,17 @@ export class EpochCache { this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; this.unfinalizedPubkey2index = data.unfinalizedPubkey2index; + this.shufflingCache = data.shufflingCache; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; + this.previousDecisionRoot = data.previousDecisionRoot; + this.currentDecisionRoot = data.currentDecisionRoot; + this.nextDecisionRoot = data.nextDecisionRoot; this.previousShuffling = data.previousShuffling; this.currentShuffling = data.currentShuffling; this.nextShuffling = data.nextShuffling; + this.nextActiveIndices = data.nextActiveIndices; this.effectiveBalanceIncrements = data.effectiveBalanceIncrements; this.totalSlashingsByIncrement = data.totalSlashingsByIncrement; this.syncParticipantReward = data.syncParticipantReward; @@ -305,7 +344,7 @@ export class EpochCache { */ static createFromState( state: BeaconStateAllForks, - {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, + {config, pubkey2index, index2pubkey, shufflingCache}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { const currentEpoch = computeEpochAtSlot(state.slot); @@ -328,18 +367,18 @@ export class EpochCache { const effectiveBalanceIncrements = getEffectiveBalanceIncrementsWithLen(validatorCount); const totalSlashingsByIncrement = getTotalSlashingsByIncrement(state); - const previousActiveIndices: ValidatorIndex[] = []; - const currentActiveIndices: ValidatorIndex[] = []; - const nextActiveIndices: ValidatorIndex[] = []; + const previousActiveIndicesAsNumberArray: ValidatorIndex[] = []; + const currentActiveIndicesAsNumberArray: ValidatorIndex[] = []; + const nextActiveIndicesAsNumberArray: ValidatorIndex[] = []; // BeaconChain could provide a shuffling cache to avoid re-computing shuffling every epoch // in that case, we don't need to compute shufflings again - const previousShufflingDecisionBlock = getShufflingDecisionBlock(state, previousEpoch); - const cachedPreviousShuffling = opts?.shufflingGetter?.(previousEpoch, previousShufflingDecisionBlock); - const currentShufflingDecisionBlock = getShufflingDecisionBlock(state, currentEpoch); - const cachedCurrentShuffling = opts?.shufflingGetter?.(currentEpoch, currentShufflingDecisionBlock); - const nextShufflingDecisionBlock = getShufflingDecisionBlock(state, nextEpoch); - const cachedNextShuffling = opts?.shufflingGetter?.(nextEpoch, nextShufflingDecisionBlock); + const previousDecisionRoot = calculateShufflingDecisionRoot(config, state, previousEpoch); + const cachedPreviousShuffling = shufflingCache?.getSync(previousEpoch, previousDecisionRoot); + const currentDecisionRoot = calculateShufflingDecisionRoot(config, state, currentEpoch); + const cachedCurrentShuffling = shufflingCache?.getSync(currentEpoch, currentDecisionRoot); + const nextDecisionRoot = calculateShufflingDecisionRoot(config, state, nextEpoch); + const cachedNextShuffling = shufflingCache?.getSync(nextEpoch, nextDecisionRoot); for (let i = 0; i < validatorCount; i++) { const validator = validators[i]; @@ -350,17 +389,17 @@ export class EpochCache { // we only need to track active indices for previous, current and next epoch if we have to compute shufflings // skip doing that if we already have cached shufflings if (cachedPreviousShuffling == null && isActiveValidator(validator, previousEpoch)) { - previousActiveIndices.push(i); + previousActiveIndicesAsNumberArray.push(i); } if (isActiveValidator(validator, currentEpoch)) { if (cachedCurrentShuffling == null) { - currentActiveIndices.push(i); + currentActiveIndicesAsNumberArray.push(i); } // We track totalActiveBalanceIncrements as ETH to fit total network balance in a JS number (53 bits) totalActiveBalanceIncrements += effectiveBalanceIncrements[i]; } if (cachedNextShuffling == null && isActiveValidator(validator, nextEpoch)) { - nextActiveIndices.push(i); + nextActiveIndicesAsNumberArray.push(i); } const {exitEpoch} = validator; @@ -382,16 +421,48 @@ export class EpochCache { throw Error("totalActiveBalanceIncrements >= Number.MAX_SAFE_INTEGER. MAX_EFFECTIVE_BALANCE is too low."); } - const currentShuffling = - cachedCurrentShuffling ?? - computeEpochShuffling(state, currentActiveIndices, currentActiveIndices.length, currentEpoch); - const previousShuffling = - cachedPreviousShuffling ?? - (isGenesis - ? currentShuffling - : computeEpochShuffling(state, previousActiveIndices, previousActiveIndices.length, previousEpoch)); - const nextShuffling = - cachedNextShuffling ?? computeEpochShuffling(state, nextActiveIndices, nextActiveIndices.length, nextEpoch); + const nextActiveIndices = new Uint32Array(nextActiveIndicesAsNumberArray); + let previousShuffling: EpochShuffling; + let currentShuffling: EpochShuffling; + let nextShuffling: EpochShuffling; + + if (!shufflingCache) { + // Only for testing. shufflingCache should always be available in prod + previousShuffling = computeEpochShuffling( + state, + new Uint32Array(previousActiveIndicesAsNumberArray), + previousEpoch + ); + + currentShuffling = isGenesis + ? previousShuffling + : computeEpochShuffling(state, new Uint32Array(currentActiveIndicesAsNumberArray), currentEpoch); + + nextShuffling = computeEpochShuffling(state, nextActiveIndices, nextEpoch); + } else { + currentShuffling = cachedCurrentShuffling + ? cachedCurrentShuffling + : shufflingCache.getSync(currentEpoch, currentDecisionRoot, { + state, + activeIndices: new Uint32Array(currentActiveIndicesAsNumberArray), + }); + + previousShuffling = cachedPreviousShuffling + ? cachedPreviousShuffling + : isGenesis + ? currentShuffling + : shufflingCache.getSync(previousEpoch, previousDecisionRoot, { + state, + activeIndices: new Uint32Array(previousActiveIndicesAsNumberArray), + }); + + nextShuffling = cachedNextShuffling + ? cachedNextShuffling + : shufflingCache.getSync(nextEpoch, nextDecisionRoot, { + state, + activeIndices: nextActiveIndices, + }); + } const currentProposerSeed = getSeed(state, currentEpoch, DOMAIN_BEACON_PROPOSER); @@ -482,13 +553,18 @@ export class EpochCache { index2pubkey, // `createFromFinalizedState()` creates cache with empty unfinalizedPubkey2index. Be cautious to only pass in finalized state unfinalizedPubkey2index: newUnfinalizedPubkeyIndexMap(), + shufflingCache, proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, proposersNextEpoch, + previousDecisionRoot, + currentDecisionRoot, + nextDecisionRoot, previousShuffling, currentShuffling, nextShuffling, + nextActiveIndices, effectiveBalanceIncrements, totalSlashingsByIncrement, syncParticipantReward, @@ -523,13 +599,18 @@ export class EpochCache { index2pubkey: this.index2pubkey, // No need to clone this reference. On each mutation the `unfinalizedPubkey2index` reference is replaced, @see `addPubkey` unfinalizedPubkey2index: this.unfinalizedPubkey2index, + shufflingCache: this.shufflingCache, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, proposersNextEpoch: this.proposersNextEpoch, + previousDecisionRoot: this.previousDecisionRoot, + currentDecisionRoot: this.currentDecisionRoot, + nextDecisionRoot: this.nextDecisionRoot, previousShuffling: this.previousShuffling, currentShuffling: this.currentShuffling, nextShuffling: this.nextShuffling, + nextActiveIndices: this.nextActiveIndices, // Uint8Array, requires cloning, but it is cloned only when necessary before an epoch transition // See EpochCache.beforeEpochTransition() effectiveBalanceIncrements: this.effectiveBalanceIncrements, @@ -555,46 +636,98 @@ export class EpochCache { /** * Called to re-use information, such as the shuffling of the next epoch, after transitioning into a - * new epoch. + * new epoch. Also handles pre-computation of values that may change during the upcoming epoch and + * that get used in the following epoch transition. Often those pre-computations are not used by the + * chain but are courtesy values that are served via the API for epoch look ahead of duties. + * + * Steps for afterProcessEpoch + * 1) update previous/current/next values of cached items */ afterProcessEpoch( - state: BeaconStateAllForks, + state: CachedBeaconStateAllForks, epochTransitionCache: { - indicesEligibleForActivationQueue: ValidatorIndex[]; - nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; - nextEpochShufflingActiveIndicesLength: number; + nextShufflingDecisionRoot: RootHex; + nextShufflingActiveIndices: Uint32Array; nextEpochTotalActiveBalanceByIncrement: number; } ): void { + // Because the slot was incremented before entering this function the "next epoch" is actually the "current epoch" + // in this context but that is not actually true because the state transition happens in the last 4 seconds of the + // epoch. For the context of this function "upcoming epoch" is used to denote the epoch that will begin after this + // function returns. The epoch that is "next" once the state transition is complete is referred to as the + // epochAfterUpcoming for the same reason to help minimize confusion. + const upcomingEpoch = this.nextEpoch; + const epochAfterUpcoming = upcomingEpoch + 1; + + // move current to previous this.previousShuffling = this.currentShuffling; - this.currentShuffling = this.nextShuffling; - const currEpoch = this.currentShuffling.epoch; - const nextEpoch = currEpoch + 1; - - this.nextShuffling = computeEpochShuffling( - state, - epochTransitionCache.nextEpochShufflingActiveValidatorIndices, - epochTransitionCache.nextEpochShufflingActiveIndicesLength, - nextEpoch - ); - - // Roll current proposers into previous proposers for metrics + this.previousDecisionRoot = this.currentDecisionRoot; this.proposersPrevEpoch = this.proposers; - const currentProposerSeed = getSeed(state, this.currentShuffling.epoch, DOMAIN_BEACON_PROPOSER); + // move next to current or calculate upcoming + this.currentDecisionRoot = this.nextDecisionRoot; + if (this.nextShuffling) { + // was already pulled from the ShufflingCache to the EpochCache (should be in most cases) + this.currentShuffling = this.nextShuffling; + } else { + this.shufflingCache?.metrics?.shufflingCache.nextShufflingNotOnEpochCache.inc(); + this.currentShuffling = + this.shufflingCache?.getSync(upcomingEpoch, this.currentDecisionRoot, { + state, + // have to use the "nextActiveIndices" that were saved in the last transition here to calculate + // the upcoming shuffling if it is not already built (similar condition to the below computation) + activeIndices: this.nextActiveIndices, + }) ?? + // allow for this case during testing where the ShufflingCache is not present, may affect perf testing + // so should be taken into account when structuring tests. Should not affect unit or other tests though + computeEpochShuffling(state, this.nextActiveIndices, upcomingEpoch); + } + const upcomingProposerSeed = getSeed(state, upcomingEpoch, DOMAIN_BEACON_PROPOSER); + // next epoch was moved to current epoch so use current here this.proposers = computeProposers( - this.config.getForkSeqAtEpoch(currEpoch), - currentProposerSeed, + this.config.getForkSeqAtEpoch(upcomingEpoch), + upcomingProposerSeed, this.currentShuffling, this.effectiveBalanceIncrements ); + // handle next values + this.nextDecisionRoot = epochTransitionCache.nextShufflingDecisionRoot; + this.nextActiveIndices = epochTransitionCache.nextShufflingActiveIndices; + if (this.shufflingCache) { + this.nextShuffling = null; + // This promise will resolve immediately after the synchronous code of the state-transition runs. Until + // the build is done on a worker thread it will be calculated immediately after the epoch transition + // completes. Once the work is done concurrently it should be ready by time this get runs so the promise + // will resolve directly on the next spin of the event loop because the epoch transition and shuffling take + // about the same time to calculate so theoretically its ready now. Do not await here though in case it + // is not ready yet as the transition must not be asynchronous. + this.shufflingCache + .get(epochAfterUpcoming, this.nextDecisionRoot) + .then((shuffling) => { + if (!shuffling) { + throw new Error("EpochShuffling not returned from get in afterProcessEpoch"); + } + this.nextShuffling = shuffling; + }) + .catch((err) => { + this.shufflingCache?.logger?.error( + "EPOCH_CONTEXT_SHUFFLING_BUILD_ERROR", + {epoch: epochAfterUpcoming, decisionRoot: epochTransitionCache.nextShufflingDecisionRoot}, + err + ); + }); + } else { + // Only for testing. shufflingCache should always be available in prod + this.nextShuffling = computeEpochShuffling(state, this.nextActiveIndices, epochAfterUpcoming); + } + // Only pre-compute the seed since it's very cheap. Do the expensive computeProposers() call only on demand. - this.proposersNextEpoch = {computed: false, seed: getSeed(state, this.nextShuffling.epoch, DOMAIN_BEACON_PROPOSER)}; + this.proposersNextEpoch = {computed: false, seed: getSeed(state, epochAfterUpcoming, DOMAIN_BEACON_PROPOSER)}; // TODO: DEDUPLICATE from createEpochCache // - // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute everytime the + // Precompute churnLimit for efficient initiateValidatorExit() during block proposing MUST be recompute every time the // active validator indices set changes in size. Validators change active status only when: // - validator.activation_epoch is set. Only changes in process_registry_updates() if validator can be activated. If // the value changes it will be set to `epoch + 1 + MAX_SEED_LOOKAHEAD`. @@ -616,14 +749,14 @@ export class EpochCache { ); // Maybe advance exitQueueEpoch at the end of the epoch if there haven't been any exists for a while - const exitQueueEpoch = computeActivationExitEpoch(currEpoch); + const exitQueueEpoch = computeActivationExitEpoch(upcomingEpoch); if (exitQueueEpoch > this.exitQueueEpoch) { this.exitQueueEpoch = exitQueueEpoch; this.exitQueueChurn = 0; } this.totalActiveBalanceIncrements = epochTransitionCache.nextEpochTotalActiveBalanceByIncrement; - if (currEpoch >= this.config.ALTAIR_FORK_EPOCH) { + if (upcomingEpoch >= this.config.ALTAIR_FORK_EPOCH) { this.syncParticipantReward = computeSyncParticipantReward(this.totalActiveBalanceIncrements); this.syncProposerReward = Math.floor(this.syncParticipantReward * PROPOSER_WEIGHT_FACTOR); this.baseRewardPerIncrement = computeBaseRewardPerIncrement(this.totalActiveBalanceIncrements); @@ -644,7 +777,7 @@ export class EpochCache { // Only keep validatorLength for epochs after finalized cpState.epoch // eg. [100(epoch 1), 102(epoch 2)].push(104(epoch 3)), this.epoch = 3, finalized cp epoch = 1 // We keep the last (3 - 1) items = [102, 104] - if (currEpoch >= this.config.ELECTRA_FORK_EPOCH) { + if (upcomingEpoch >= this.config.ELECTRA_FORK_EPOCH) { this.historicalValidatorLengths = this.historicalValidatorLengths.push(state.validators.length); // If number of validatorLengths we want to keep exceeds the current list size, it implies @@ -786,7 +919,7 @@ export class EpochCache { const indexes = computeProposers( this.config.getForkSeqAtEpoch(this.epoch + 1), this.proposersNextEpoch.seed, - this.nextShuffling, + this.getShufflingAtEpoch(this.nextEpoch), this.effectiveBalanceIncrements ); this.proposersNextEpoch = {computed: true, indexes}; @@ -840,30 +973,8 @@ export class EpochCache { epoch: Epoch, requestedValidatorIndices: ValidatorIndex[] ): Map { - const requestedValidatorIndicesSet = new Set(requestedValidatorIndices); - const duties = new Map(); - - const epochCommittees = this.getShufflingAtEpoch(epoch).committees; - for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) { - const slotCommittees = epochCommittees[epochSlot]; - for (let i = 0, committeesAtSlot = slotCommittees.length; i < committeesAtSlot; i++) { - for (let j = 0, committeeLength = slotCommittees[i].length; j < committeeLength; j++) { - const validatorIndex = slotCommittees[i][j]; - if (requestedValidatorIndicesSet.has(validatorIndex)) { - duties.set(validatorIndex, { - validatorIndex, - committeeLength, - committeesAtSlot, - validatorCommitteeIndex: j, - committeeIndex: i, - slot: epoch * SLOTS_PER_EPOCH + epochSlot, - }); - } - } - } - } - - return duties; + const shuffling = this.getShufflingAtEpoch(epoch); + return calculateCommitteeAssignments(shuffling, requestedValidatorIndices); } /** @@ -987,6 +1098,13 @@ export class EpochCache { getShufflingAtEpoch(epoch: Epoch): EpochShuffling { const shuffling = this.getShufflingAtEpochOrNull(epoch); if (shuffling === null) { + if (epoch === this.nextEpoch) { + throw new EpochCacheError({ + code: EpochCacheErrorCode.NEXT_SHUFFLING_NOT_AVAILABLE, + epoch: epoch, + decisionRoot: this.getShufflingDecisionRoot(this.nextEpoch), + }); + } throw new EpochCacheError({ code: EpochCacheErrorCode.COMMITTEE_EPOCH_OUT_OF_RANGE, currentEpoch: this.currentShuffling.epoch, @@ -997,15 +1115,37 @@ export class EpochCache { return shuffling; } + getShufflingDecisionRoot(epoch: Epoch): RootHex { + switch (epoch) { + case this.epoch - 1: + return this.previousDecisionRoot; + case this.epoch: + return this.currentDecisionRoot; + case this.nextEpoch: + return this.nextDecisionRoot; + default: + throw new EpochCacheError({ + code: EpochCacheErrorCode.DECISION_ROOT_EPOCH_OUT_OF_RANGE, + currentEpoch: this.epoch, + requestedEpoch: epoch, + }); + } + } + getShufflingAtEpochOrNull(epoch: Epoch): EpochShuffling | null { - if (epoch === this.previousShuffling.epoch) { - return this.previousShuffling; - } else if (epoch === this.currentShuffling.epoch) { - return this.currentShuffling; - } else if (epoch === this.nextShuffling.epoch) { - return this.nextShuffling; - } else { - return null; + switch (epoch) { + case this.epoch - 1: + return this.previousShuffling; + case this.epoch: + return this.currentShuffling; + case this.nextEpoch: + if (!this.nextShuffling) { + this.nextShuffling = + this.shufflingCache?.getSync(this.nextEpoch, this.getShufflingDecisionRoot(this.nextEpoch)) ?? null; + } + return this.nextShuffling; + default: + return null; } } @@ -1102,19 +1242,11 @@ function getEffectiveBalanceIncrementsByteLen(validatorCount: number): number { return 1024 * Math.ceil(validatorCount / 1024); } -// Copied from lodestar-api package to avoid depending on the package -type AttesterDuty = { - validatorIndex: ValidatorIndex; - committeeIndex: CommitteeIndex; - committeeLength: number; - committeesAtSlot: number; - validatorCommitteeIndex: number; - slot: Slot; -}; - export enum EpochCacheErrorCode { COMMITTEE_INDEX_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_COMMITTEE_INDEX_OUT_OF_RANGE", COMMITTEE_EPOCH_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_COMMITTEE_EPOCH_OUT_OF_RANGE", + DECISION_ROOT_EPOCH_OUT_OF_RANGE = "EPOCH_CONTEXT_ERROR_DECISION_ROOT_EPOCH_OUT_OF_RANGE", + NEXT_SHUFFLING_NOT_AVAILABLE = "EPOCH_CONTEXT_ERROR_NEXT_SHUFFLING_NOT_AVAILABLE", NO_SYNC_COMMITTEE = "EPOCH_CONTEXT_ERROR_NO_SYNC_COMMITTEE", PROPOSER_EPOCH_MISMATCH = "EPOCH_CONTEXT_ERROR_PROPOSER_EPOCH_MISMATCH", } @@ -1130,6 +1262,16 @@ type EpochCacheErrorType = requestedEpoch: Epoch; currentEpoch: Epoch; } + | { + code: EpochCacheErrorCode.DECISION_ROOT_EPOCH_OUT_OF_RANGE; + requestedEpoch: Epoch; + currentEpoch: Epoch; + } + | { + code: EpochCacheErrorCode.NEXT_SHUFFLING_NOT_AVAILABLE; + epoch: Epoch; + decisionRoot: RootHex; + } | { code: EpochCacheErrorCode.NO_SYNC_COMMITTEE; epoch: Epoch; diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 6f27ad96d1c8..27b781e8a6a1 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,6 +1,12 @@ -import {Epoch, ValidatorIndex, phase0} from "@lodestar/types"; -import {intDiv} from "@lodestar/utils"; -import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; +import {phase0, Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; +import {intDiv, toRootHex} from "@lodestar/utils"; +import { + EPOCHS_PER_SLASHINGS_VECTOR, + FAR_FUTURE_EPOCH, + ForkSeq, + SLOTS_PER_HISTORICAL_ROOT, + MIN_ACTIVATION_BALANCE, +} from "@lodestar/params"; import { hasMarkers, @@ -155,12 +161,12 @@ export interface EpochTransitionCache { * | beforeProcessEpoch | calculate during the validator loop| * | afterEpochTransitionCache | read it | */ - nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; + nextShufflingActiveIndices: Uint32Array; /** - * We do not use up to `nextEpochShufflingActiveValidatorIndices.length`, use this to control that + * Shuffling decision root that gets set on the EpochCache in afterProcessEpoch */ - nextEpochShufflingActiveIndicesLength: number; + nextShufflingDecisionRoot: RootHex; /** * Altair specific, this is total active balances for the next epoch. @@ -360,6 +366,24 @@ export function beforeProcessEpoch( } } + // Trigger async build of shuffling for epoch after next (nextShuffling post epoch transition) + const epochAfterNext = state.epochCtx.nextEpoch + 1; + // cannot call calculateShufflingDecisionRoot here because spec prevent getting current slot + // as a decision block. we are part way through the transition though and this was added in + // process slot beforeProcessEpoch happens so it available and valid + const nextShufflingDecisionRoot = toRootHex(state.blockRoots.get(state.slot % SLOTS_PER_HISTORICAL_ROOT)); + const nextShufflingActiveIndices = new Uint32Array(nextEpochShufflingActiveIndicesLength); + if (nextEpochShufflingActiveIndicesLength > nextEpochShufflingActiveValidatorIndices.length) { + throw new Error( + `Invalid activeValidatorCount: ${nextEpochShufflingActiveIndicesLength} > ${nextEpochShufflingActiveValidatorIndices.length}` + ); + } + // only the first `activeValidatorCount` elements are copied to `activeIndices` + for (let i = 0; i < nextEpochShufflingActiveIndicesLength; i++) { + nextShufflingActiveIndices[i] = nextEpochShufflingActiveValidatorIndices[i]; + } + state.epochCtx.shufflingCache?.build(epochAfterNext, nextShufflingDecisionRoot, state, nextShufflingActiveIndices); + if (totalActiveStakeByIncrement < 1) { totalActiveStakeByIncrement = 1; } else if (totalActiveStakeByIncrement >= Number.MAX_SAFE_INTEGER) { @@ -483,8 +507,8 @@ export function beforeProcessEpoch( indicesEligibleForActivationQueue, indicesEligibleForActivation, indicesToEject, - nextEpochShufflingActiveValidatorIndices, - nextEpochShufflingActiveIndicesLength, + nextShufflingDecisionRoot, + nextShufflingActiveIndices, // to be updated in processEffectiveBalanceUpdates nextEpochTotalActiveBalanceByIncrement: 0, isActivePrevEpoch, diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index 0435e8829d21..5412675352e9 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -175,7 +175,7 @@ export function loadCachedBeaconState { + const requestedValidatorIndicesSet = new Set(requestedValidatorIndices); + const duties = new Map(); + + const epochCommittees = epochShuffling.committees; + for (let epochSlot = 0; epochSlot < SLOTS_PER_EPOCH; epochSlot++) { + const slotCommittees = epochCommittees[epochSlot]; + for (let i = 0, committeesAtSlot = slotCommittees.length; i < committeesAtSlot; i++) { + for (let j = 0, committeeLength = slotCommittees[i].length; j < committeeLength; j++) { + const validatorIndex = slotCommittees[i][j]; + if (requestedValidatorIndicesSet.has(validatorIndex)) { + duties.set(validatorIndex, { + validatorIndex, + committeeLength, + committeesAtSlot, + validatorCommitteeIndex: j, + committeeIndex: i, + slot: epochShuffling.epoch * SLOTS_PER_EPOCH + epochSlot, + }); + } + } + } + } + + return duties; +} diff --git a/packages/state-transition/src/util/computeAnchorCheckpoint.ts b/packages/state-transition/src/util/computeAnchorCheckpoint.ts new file mode 100644 index 000000000000..e2efc18952c2 --- /dev/null +++ b/packages/state-transition/src/util/computeAnchorCheckpoint.ts @@ -0,0 +1,38 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {ssz, phase0} from "@lodestar/types"; +import {GENESIS_SLOT, ZERO_HASH} from "@lodestar/params"; +import {BeaconStateAllForks} from "../types.js"; +import {blockToHeader} from "./blockRoot.js"; +import {computeCheckpointEpochAtStateSlot} from "./epoch.js"; + +export function computeAnchorCheckpoint( + config: ChainForkConfig, + anchorState: BeaconStateAllForks +): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader} { + let blockHeader; + let root; + const blockTypes = config.getForkTypes(anchorState.latestBlockHeader.slot); + + if (anchorState.latestBlockHeader.slot === GENESIS_SLOT) { + const block = blockTypes.BeaconBlock.defaultValue(); + block.stateRoot = anchorState.hashTreeRoot(); + blockHeader = blockToHeader(config, block); + root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); + } else { + blockHeader = ssz.phase0.BeaconBlockHeader.clone(anchorState.latestBlockHeader); + if (ssz.Root.equals(blockHeader.stateRoot, ZERO_HASH)) { + blockHeader.stateRoot = anchorState.hashTreeRoot(); + } + root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader); + } + + return { + checkpoint: { + root, + // the checkpoint epoch = computeEpochAtSlot(anchorState.slot) + 1 if slot is not at epoch boundary + // this is similar to a process_slots() call + epoch: computeCheckpointEpochAtStateSlot(anchorState.slot), + }, + blockHeader, + }; +} diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index 856721d8d083..c26c62fd2079 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -1,16 +1,60 @@ -import {Epoch, RootHex, ValidatorIndex} from "@lodestar/types"; -import {intDiv, toRootHex} from "@lodestar/utils"; +import {Epoch, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; +import {GaugeExtra, intDiv, Logger, NoLabels, toRootHex} from "@lodestar/utils"; import { DOMAIN_BEACON_ATTESTER, + GENESIS_SLOT, MAX_COMMITTEES_PER_SLOT, SLOTS_PER_EPOCH, TARGET_COMMITTEE_SIZE, } from "@lodestar/params"; +import {BeaconConfig} from "@lodestar/config"; import {BeaconStateAllForks} from "../types.js"; import {getSeed} from "./seed.js"; import {unshuffleList} from "./shuffle.js"; import {computeStartSlotAtEpoch} from "./epoch.js"; import {getBlockRootAtSlot} from "./blockRoot.js"; +import {computeAnchorCheckpoint} from "./computeAnchorCheckpoint.js"; + +export interface ShufflingBuildProps { + state: BeaconStateAllForks; + activeIndices: Uint32Array; +} + +export interface PublicShufflingCacheMetrics { + shufflingCache: { + nextShufflingNotOnEpochCache: GaugeExtra; + }; +} +export interface IShufflingCache { + metrics: PublicShufflingCacheMetrics | null; + logger: Logger | null; + /** + * Gets a cached shuffling via the epoch and decision root. If the state and + * activeIndices are passed and a shuffling is not available it will be built + * synchronously. If the state is not passed and the shuffling is not available + * nothing will be returned. + * + * NOTE: If a shuffling is already queued and not calculated it will build and resolve + * the promise but the already queued build will happen at some later time + */ + getSync( + epoch: Epoch, + decisionRoot: RootHex, + buildProps?: T + ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null; + + /** + * Gets a cached shuffling via the epoch and decision root. Returns a promise + * for the shuffling if it hs not calculated yet. Returns null if a build has + * not been queued nor a shuffling was calculated. + */ + get(epoch: Epoch, decisionRoot: RootHex): Promise; + + /** + * Queue asynchronous build for an EpochShuffling + */ + build(epoch: Epoch, decisionRoot: RootHex, state: BeaconStateAllForks, activeIndices: Uint32Array): void; +} /** * Readonly interface for EpochShuffling. @@ -60,21 +104,13 @@ export function computeCommitteeCount(activeValidatorCount: number): number { export function computeEpochShuffling( state: BeaconStateAllForks, - activeIndices: ArrayLike, - activeValidatorCount: number, + activeIndices: Uint32Array, epoch: Epoch ): EpochShuffling { - const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); + const activeValidatorCount = activeIndices.length; - if (activeValidatorCount > activeIndices.length) { - throw new Error(`Invalid activeValidatorCount: ${activeValidatorCount} > ${activeIndices.length}`); - } - // only the first `activeValidatorCount` elements are copied to `activeIndices` - const _activeIndices = new Uint32Array(activeValidatorCount); - for (let i = 0; i < activeValidatorCount; i++) { - _activeIndices[i] = activeIndices[i]; - } - const shuffling = _activeIndices.slice(); + const shuffling = activeIndices.slice(); + const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); unshuffleList(shuffling, seed); const committeesPerSlot = computeCommitteeCount(activeValidatorCount); @@ -98,14 +134,29 @@ export function computeEpochShuffling( return { epoch, - activeIndices: _activeIndices, + activeIndices, shuffling, committees, committeesPerSlot, }; } -export function getShufflingDecisionBlock(state: BeaconStateAllForks, epoch: Epoch): RootHex { +function calculateDecisionRoot(state: BeaconStateAllForks, epoch: Epoch): RootHex { const pivotSlot = computeStartSlotAtEpoch(epoch - 1) - 1; return toRootHex(getBlockRootAtSlot(state, pivotSlot)); } + +/** + * Get the shuffling decision block root for the given epoch of given state + * - Special case close to genesis block, return the genesis block root + * - This is similar to forkchoice.getDependentRoot() function, otherwise we cannot get cached shuffing in attestation verification when syncing from genesis. + */ +export function calculateShufflingDecisionRoot( + config: BeaconConfig, + state: BeaconStateAllForks, + epoch: Epoch +): RootHex { + return state.slot > GENESIS_SLOT + ? calculateDecisionRoot(state, epoch) + : toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(config, state).blockHeader)); +} diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index 9b9916f1d49e..5f8d9e5cdcfc 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -4,7 +4,9 @@ export * from "./attestation.js"; export * from "./attesterStatus.js"; export * from "./balance.js"; export * from "./blindedBlock.js"; +export * from "./calculateCommitteeAssignments.js"; export * from "./capella.js"; +export * from "./computeAnchorCheckpoint.js"; export * from "./execution.js"; export * from "./blockRoot.js"; export * from "./domain.js"; diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 15cde849ce9f..5a10fd4d8bbd 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -46,6 +46,7 @@ describe(`altair processEpoch - ${stateId}`, () => { fn: (state) => { const cache = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStateAltair, cache); + state.slot++; state.epochCtx.afterProcessEpoch(state, cache); // Simulate root computation through the next block to account for changes // 74184 hash64 ops - 92.730 ms @@ -187,6 +188,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => state.epochCtx.afterProcessEpoch(state, cache), + fn: ({state, cache}) => { + state.slot++; + state.epochCtx.afterProcessEpoch(state, cache); + }, }); } diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index 61bfad20b1ee..a4daf308aaa0 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -46,6 +46,7 @@ describe(`capella processEpoch - ${stateId}`, () => { fn: (state) => { const cache = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStateCapella, cache); + state.slot++; state.epochCtx.afterProcessEpoch(state, cache); // Simulate root computation through the next block to account for changes // 74184 hash64 ops - 92.730 ms @@ -159,6 +160,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => state.epochCtx.afterProcessEpoch(state, cache), + fn: ({state, cache}) => { + state.slot++; + state.epochCtx.afterProcessEpoch(state, cache); + }, }); } diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index 3af3d4d4a832..5c19b347af62 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -43,6 +43,7 @@ describe(`phase0 processEpoch - ${stateId}`, () => { fn: (state) => { const cache = beforeProcessEpoch(state); processEpoch(fork, state as CachedBeaconStatePhase0, cache); + state.slot++; state.epochCtx.afterProcessEpoch(state, cache); // Simulate root computation through the next block to account for changes state.hashTreeRoot(); @@ -162,6 +163,9 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue return {state, cache: cacheAfter}; }, beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => state.epochCtx.afterProcessEpoch(state, cache), + fn: ({state, cache}) => { + state.slot++; + state.epochCtx.afterProcessEpoch(state, cache); + }, }); } diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index a8a1b1399dc5..25b43e242d02 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -79,17 +79,15 @@ describe("loadState", function () { pubkey2index.set(pubkey, validatorIndex); index2pubkey[validatorIndex] = PublicKey.fromBytes(pubkey); } - // skip computimg shuffling in performance test because in reality we have a ShufflingCache - // eslint-disable-next-line @typescript-eslint/explicit-function-return-type - const shufflingGetter = () => seedState.epochCtx.currentShuffling; createCachedBeaconState( migratedState, { config: seedState.config, pubkey2index, index2pubkey, + shufflingCache: seedState.epochCtx.shufflingCache, }, - {skipSyncPubkeys: true, skipSyncCommitteeCache: true, shufflingGetter} + {skipSyncPubkeys: true, skipSyncCommitteeCache: true} ); }, }); diff --git a/packages/state-transition/test/perf/util/shufflings.test.ts b/packages/state-transition/test/perf/util/shufflings.test.ts index 24be96c4676d..41767c184349 100644 --- a/packages/state-transition/test/perf/util/shufflings.test.ts +++ b/packages/state-transition/test/perf/util/shufflings.test.ts @@ -27,17 +27,17 @@ describe("epoch shufflings", () => { itBench({ id: `computeProposers - vc ${numValidators}`, fn: () => { - const epochSeed = getSeed(state, state.epochCtx.nextShuffling.epoch, DOMAIN_BEACON_PROPOSER); + const epochSeed = getSeed(state, state.epochCtx.epoch, DOMAIN_BEACON_PROPOSER); const fork = state.config.getForkSeq(state.slot); - computeProposers(fork, epochSeed, state.epochCtx.nextShuffling, state.epochCtx.effectiveBalanceIncrements); + computeProposers(fork, epochSeed, state.epochCtx.currentShuffling, state.epochCtx.effectiveBalanceIncrements); }, }); itBench({ id: `computeEpochShuffling - vc ${numValidators}`, fn: () => { - const {activeIndices} = state.epochCtx.nextShuffling; - computeEpochShuffling(state, activeIndices, activeIndices.length, nextEpoch); + const {nextActiveIndices} = state.epochCtx; + computeEpochShuffling(state, nextActiveIndices, nextEpoch); }, }); @@ -45,12 +45,7 @@ describe("epoch shufflings", () => { id: `getNextSyncCommittee - vc ${numValidators}`, fn: () => { const fork = state.config.getForkSeq(state.slot); - getNextSyncCommittee( - fork, - state, - state.epochCtx.nextShuffling.activeIndices, - state.epochCtx.effectiveBalanceIncrements - ); + getNextSyncCommittee(fork, state, state.epochCtx.nextActiveIndices, state.epochCtx.effectiveBalanceIncrements); }, }); }); diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 77c5da7a5f4a..092dda321610 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -1,6 +1,6 @@ import {fromHexString} from "@chainsafe/ssz"; import {describe, it, expect} from "vitest"; -import {Epoch, ssz, RootHex} from "@lodestar/types"; +import {ssz} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {config as defaultConfig} from "@lodestar/config/default"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; @@ -9,7 +9,6 @@ import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; import {interopPubkeysCached} from "../utils/interop.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; -import {EpochShuffling, getShufflingDecisionBlock} from "../../src/util/epochShuffling.js"; describe("CachedBeaconState", () => { it("Clone and mutate", () => { @@ -189,42 +188,21 @@ describe("CachedBeaconState", () => { // confirm loadState() result const stateBytes = state.serialize(); - const newCachedState = loadCachedBeaconState(seedState, stateBytes, {skipSyncCommitteeCache: true}); + const newCachedState = loadCachedBeaconState(seedState, stateBytes, { + skipSyncCommitteeCache: true, + }); const newStateBytes = newCachedState.serialize(); expect(newStateBytes).toEqual(stateBytes); expect(newCachedState.hashTreeRoot()).toEqual(state.hashTreeRoot()); - const shufflingGetter = (shufflingEpoch: Epoch, dependentRoot: RootHex): EpochShuffling | null => { - if ( - shufflingEpoch === seedState.epochCtx.epoch - 1 && - dependentRoot === getShufflingDecisionBlock(seedState, shufflingEpoch) - ) { - return seedState.epochCtx.previousShuffling; - } - - if ( - shufflingEpoch === seedState.epochCtx.epoch && - dependentRoot === getShufflingDecisionBlock(seedState, shufflingEpoch) - ) { - return seedState.epochCtx.currentShuffling; - } - - if ( - shufflingEpoch === seedState.epochCtx.epoch + 1 && - dependentRoot === getShufflingDecisionBlock(seedState, shufflingEpoch) - ) { - return seedState.epochCtx.nextShuffling; - } - - return null; - }; const cachedState = createCachedBeaconState( state, { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + shufflingCache: seedState.epochCtx.shufflingCache, }, - {skipSyncCommitteeCache: true, shufflingGetter} + {skipSyncCommitteeCache: true} ); // validatorCountDelta < 0 is unrealistic and shuffling computation results in a different result if (validatorCountDelta >= 0) { From 0dceb831ca7c44ee8ebfdb667a793a042f0b93c4 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 23 Sep 2024 17:35:45 +0100 Subject: [PATCH 123/145] chore: clean up from electra review (#7102) --- packages/beacon-node/src/api/impl/validator/index.ts | 2 +- packages/beacon-node/src/metrics/metrics/beacon.ts | 2 +- packages/beacon-node/src/metrics/metrics/lodestar.ts | 4 ++-- packages/beacon-node/test/unit/util/sszBytes.test.ts | 2 +- packages/params/src/presets/mainnet.ts | 1 - packages/params/src/presets/minimal.ts | 1 - 6 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index d2ce6672c89f..4876975c1bea 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1087,7 +1087,7 @@ export function getValidatorApi( await waitForSlot(slot); // Must never request for a future slot > currentSlot - const dataRootHex = toHex(attestationDataRoot); + const dataRootHex = toRootHex(attestationDataRoot); const aggregate = chain.attestationPool.getAggregate(slot, null, dataRootHex); const fork = chain.config.getForkName(slot); diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index 96bb6bea4174..949999dbb1f4 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -122,7 +122,7 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { headState: { unfinalizedPubkeyCacheSize: register.gauge({ - name: "head_state_unfinalized_pubkey_cache_size", + name: "beacon_head_state_unfinalized_pubkey_cache_size", help: "Current size of the unfinalizedPubkey2Index cache in the head state", }), }, diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 4b076471dcab..fb4ea61b6523 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -381,11 +381,11 @@ export function createLodestarMetrics( epochCache: { finalizedPubkeyDuplicateInsert: register.gauge({ - name: "lodestar_epoch_cache_finalized_pubkey_duplicate_insert", + name: "lodestar_epoch_cache_finalized_pubkey_duplicate_insert_total", help: "Total count of duplicate insert of finalized pubkeys", }), newUnFinalizedPubkey: register.gauge({ - name: "lodestar_epoch_cache_new_unfinalized_pubkey", + name: "lodestar_epoch_cache_new_unfinalized_pubkey_total", help: "Total count of unfinalized pubkeys added", }), }, diff --git a/packages/beacon-node/test/unit/util/sszBytes.test.ts b/packages/beacon-node/test/unit/util/sszBytes.test.ts index 612eea5a4388..8b72c31df6c8 100644 --- a/packages/beacon-node/test/unit/util/sszBytes.test.ts +++ b/packages/beacon-node/test/unit/util/sszBytes.test.ts @@ -6,7 +6,7 @@ import {ForkName, MAX_COMMITTEES_PER_SLOT} from "@lodestar/params"; import { getAttDataFromAttestationSerialized, getAttDataFromSignedAggregateAndProofPhase0, - getAggregationBitsFromAttestationSerialized as getAggregationBitsFromAttestationSerialized, + getAggregationBitsFromAttestationSerialized, getBlockRootFromAttestationSerialized, getBlockRootFromSignedAggregateAndProofSerialized, getSlotFromAttestationSerialized, diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index a4a88aac1f58..ca599e990df4 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -127,7 +127,6 @@ export const mainnetPreset: BeaconPreset = { MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 8, // 2**11 * 10**9 (= 2,048,000,000,000) Gwei MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000, - // 2**16 (= 65536) MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: 4096, MIN_ACTIVATION_BALANCE: 32000000000, PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index 97eff53cf013..5dc8fc10d803 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -128,7 +128,6 @@ export const minimalPreset: BeaconPreset = { MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 1, // 2**11 * 10**9 (= 2,048,000,000,000) Gwei MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000, - // 2**16 (= 65536) MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: 4096, MIN_ACTIVATION_BALANCE: 32000000000, PENDING_BALANCE_DEPOSITS_LIMIT: 134217728, From d0ba6bc3ccf203307bcbf0087d82749dff81699b Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 23 Sep 2024 21:54:20 +0100 Subject: [PATCH 124/145] chore: upgrade fastify to v5 (#7101) * chore: upgrade fastify to v5 * Upgrade fastify plugins * Clean up yarn lock --- packages/api/package.json | 2 +- packages/api/src/utils/schema.ts | 12 +- packages/api/src/utils/server/parser.ts | 14 +- packages/beacon-node/package.json | 10 +- packages/cli/package.json | 2 +- packages/light-client/package.json | 2 +- yarn.lock | 394 +++++++++++++----------- 7 files changed, 228 insertions(+), 208 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index d2f8a621f02e..7d851d9513ba 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -83,7 +83,7 @@ "@types/eventsource": "^1.1.11", "@types/qs": "^6.9.7", "ajv": "^8.12.0", - "fastify": "^4.27.0" + "fastify": "^5.0.0" }, "keywords": [ "ethereum", diff --git a/packages/api/src/utils/schema.ts b/packages/api/src/utils/schema.ts index 2d086fd8dfa9..3f297db6f665 100644 --- a/packages/api/src/utils/schema.ts +++ b/packages/api/src/utils/schema.ts @@ -1,3 +1,4 @@ +import {MediaType} from "./headers.js"; import {Endpoint, HeaderParams, PathParams, QueryParams} from "./types.js"; // Reasoning: Allows to declare JSON schemas for server routes in a succinct typesafe way. @@ -91,7 +92,16 @@ export function getFastifySchema(schemaDef: Schem const schema: {params?: JsonSchemaObj; querystring?: JsonSchemaObj; headers?: JsonSchemaObj; body?: JsonSchema} = {}; if (schemaDef.body != null) { - schema.body = getJsonSchemaItem(schemaDef.body); + schema.body = { + content: { + [MediaType.json]: { + schema: getJsonSchemaItem(schemaDef.body), + }, + [MediaType.ssz]: { + schema: {}, + }, + }, + }; } if (schemaDef.params) { diff --git a/packages/api/src/utils/server/parser.ts b/packages/api/src/utils/server/parser.ts index fd668b63757e..3300575a4845 100644 --- a/packages/api/src/utils/server/parser.ts +++ b/packages/api/src/utils/server/parser.ts @@ -2,22 +2,10 @@ import type * as fastify from "fastify"; import {MediaType} from "../headers.js"; export function addSszContentTypeParser(server: fastify.FastifyInstance): void { - // Cache body schema symbol, does not change per request - let bodySchemaSymbol: symbol | undefined; - server.addContentTypeParser( MediaType.ssz, {parseAs: "buffer"}, - async (request: fastify.FastifyRequest, payload: Buffer) => { - if (bodySchemaSymbol === undefined) { - // Get body schema symbol to be able to access validation function - // https://github.com/fastify/fastify/blob/af2ccb5ff681c1d0ac22eb7314c6fa803f73c873/lib/symbols.js#L25 - bodySchemaSymbol = Object.getOwnPropertySymbols(request.context).find((s) => s.description === "body-schema"); - } - // JSON schema validation will be applied to `Buffer` object, it is required to override validation function - // See https://github.com/fastify/help/issues/1012, it is not possible right now to define a schema per content type - (request.context as unknown as Record)[bodySchemaSymbol as symbol] = () => true; - + async (_request: fastify.FastifyRequest, payload: Buffer) => { // We could just return the `Buffer` here which is a subclass of `Uint8Array` but downstream code does not require it // and it's better to convert it here to avoid unexpected behavior such as `Buffer.prototype.slice` not copying memory // See https://github.com/nodejs/node/issues/41588#issuecomment-1016269584 diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index ade89ebc6cc1..f101206bfd9e 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -106,10 +106,10 @@ "@chainsafe/ssz": "^0.17.1", "@chainsafe/threads": "^1.11.1", "@ethersproject/abi": "^5.7.0", - "@fastify/bearer-auth": "^9.0.0", - "@fastify/cors": "^8.2.1", - "@fastify/swagger": "^8.10.0", - "@fastify/swagger-ui": "^1.9.3", + "@fastify/bearer-auth": "^10.0.1", + "@fastify/cors": "^10.0.1", + "@fastify/swagger": "^9.0.0", + "@fastify/swagger-ui": "^5.0.1", "@libp2p/bootstrap": "^10.0.21", "@libp2p/identify": "^1.0.20", "@libp2p/interface": "^1.3.0", @@ -136,7 +136,7 @@ "datastore-core": "^9.1.1", "datastore-level": "^10.1.1", "deepmerge": "^4.3.1", - "fastify": "^4.27.0", + "fastify": "^5.0.0", "interface-datastore": "^8.2.7", "it-all": "^3.0.4", "it-pipe": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 579725eba04e..a5527a8eee87 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -94,6 +94,6 @@ "@types/inquirer": "^9.0.3", "@types/proper-lockfile": "^4.1.4", "@types/yargs": "^17.0.24", - "fastify": "^4.27.0" + "fastify": "^5.0.0" } } diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 940a78ae66d5..2eda3a45d455 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -87,7 +87,7 @@ "devDependencies": { "@chainsafe/as-sha256": "^0.5.0", "@types/qs": "^6.9.7", - "fastify": "^4.27.0", + "fastify": "^5.0.0", "qs": "^6.11.1", "uint8arrays": "^5.0.1" }, diff --git a/yarn.lock b/yarn.lock index 38e6bae97d60..f546d770acd8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1285,108 +1285,104 @@ "@ethersproject/properties" "^5.7.0" "@ethersproject/strings" "^5.7.0" -"@fastify/accept-negotiator@^1.0.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@fastify/accept-negotiator/-/accept-negotiator-1.1.0.tgz#c1c66b3b771c09742a54dd5bc87c582f6b0630ff" - integrity sha512-OIHZrb2ImZ7XG85HXOONLcJWGosv7sIvM2ifAPQVhg9Lv7qdmMBNVaai4QTdyuaqbKM5eO6sLSQOYI7wEQeCJQ== +"@fastify/accept-negotiator@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@fastify/accept-negotiator/-/accept-negotiator-2.0.0.tgz#efce76b4d658e7ee669e681c2d79bffc9a654fdb" + integrity sha512-/Sce/kBzuTxIq5tJh85nVNOq9wKD8s+viIgX0fFMDBdw95gnpf53qmF1oBgJym3cPFliWUuSloVg/1w/rH0FcQ== -"@fastify/ajv-compiler@^3.5.0": - version "3.5.0" - resolved "https://registry.yarnpkg.com/@fastify/ajv-compiler/-/ajv-compiler-3.5.0.tgz#459bff00fefbf86c96ec30e62e933d2379e46670" - integrity sha512-ebbEtlI7dxXF5ziNdr05mOY8NnDiPB1XvAlLHctRt/Rc+C3LCOVW5imUVX+mhvUhnNzmPBHewUkOFgGlCxgdAA== +"@fastify/ajv-compiler@^4.0.0": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@fastify/ajv-compiler/-/ajv-compiler-4.0.1.tgz#9567b4c09149a0f342e931c7196a8ed9dc292954" + integrity sha512-DxrBdgsjNLP0YM6W5Hd6/Fmj43S8zMKiFJYgi+Ri3htTGAowPVG/tG1wpnWLMjufEnehRivUCKZ1pLDIoZdTuw== dependencies: - ajv "^8.11.0" - ajv-formats "^2.1.1" - fast-uri "^2.0.0" + ajv "^8.12.0" + ajv-formats "^3.0.1" + fast-uri "^3.0.0" -"@fastify/bearer-auth@^9.0.0": - version "9.0.0" - resolved "https://registry.yarnpkg.com/@fastify/bearer-auth/-/bearer-auth-9.0.0.tgz#9a75abcb1d54751dc04e97b37db5363e325c0f90" - integrity sha512-I1egwg1LRdIvhjL/P+3UEfyK7A3YTnN3goTyf8MJ+v7vVkwyyd8ieccFKI0SzEuxMmfQh/p4yNyLZWcMVwpInA== +"@fastify/bearer-auth@^10.0.1": + version "10.0.1" + resolved "https://registry.yarnpkg.com/@fastify/bearer-auth/-/bearer-auth-10.0.1.tgz#893466052fa566c24eb4f44a3c6aacb50db96ecd" + integrity sha512-i2snRkAJsMmfFcsRS/fFIovcLL3WeZtxJP9pprx2NvB8N/l+fjMNmKeWWyX0hDS2Q0zEPqLz/G0DK92nqJYAJQ== dependencies: - fastify-plugin "^4.0.0" + "@fastify/error" "^4.0.0" + fastify-plugin "^5.0.0" "@fastify/busboy@^2.0.0": version "2.1.1" resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== -"@fastify/cors@^8.2.1": - version "8.2.1" - resolved "https://registry.yarnpkg.com/@fastify/cors/-/cors-8.2.1.tgz#dd348162bcbfb87dff4b492e2bef32d41244006a" - integrity sha512-2H2MrDD3ea7g707g1CNNLWb9/tYbmw7HS+MK2SDcgjxwzbOFR93JortelTIO8DBFsZqFtEpKNxiZfSyrGgYcbw== +"@fastify/cors@^10.0.1": + version "10.0.1" + resolved "https://registry.yarnpkg.com/@fastify/cors/-/cors-10.0.1.tgz#c208fa5f672db31a8383400349e9852762903d64" + integrity sha512-O8JIf6448uQbOgzSkCqhClw6gFTAqrdfeA6R3fc/3gwTJGUp7gl8/3tbNB+6INuu4RmgVOq99BmvdGbtu5pgOA== dependencies: - fastify-plugin "^4.0.0" - mnemonist "0.39.5" + fastify-plugin "^5.0.0" + mnemonist "0.39.8" -"@fastify/deepmerge@^1.0.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@fastify/deepmerge/-/deepmerge-1.3.0.tgz#8116858108f0c7d9fd460d05a7d637a13fe3239a" - integrity sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A== - -"@fastify/error@^3.3.0", "@fastify/error@^3.4.0": - version "3.4.1" - resolved "https://registry.yarnpkg.com/@fastify/error/-/error-3.4.1.tgz#b14bb4cac3dd4ec614becbc643d1511331a6425c" - integrity sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ== +"@fastify/error@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@fastify/error/-/error-4.0.0.tgz#7842d6161fbce78953638318be99033a0c2d5070" + integrity sha512-OO/SA8As24JtT1usTUTKgGH7uLvhfwZPwlptRi2Dp5P4KKmJI3gvsZ8MIHnNwDs4sLf/aai5LzTyl66xr7qMxA== -"@fastify/fast-json-stringify-compiler@^4.3.0": - version "4.3.0" - resolved "https://registry.yarnpkg.com/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-4.3.0.tgz#5df89fa4d1592cbb8780f78998355feb471646d5" - integrity sha512-aZAXGYo6m22Fk1zZzEUKBvut/CIIQe/BapEORnxiD5Qr0kPHqqI69NtEMCme74h+at72sPhbkb4ZrLd1W3KRLA== +"@fastify/fast-json-stringify-compiler@^5.0.0": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-5.0.1.tgz#659c74f3181fb4f984fe27dcc95d14366ae85ca0" + integrity sha512-f2d3JExJgFE3UbdFcpPwqNUEoHWmt8pAKf8f+9YuLESdefA0WgqxeT6DrGL4Yrf/9ihXNSKOqpjEmurV405meA== dependencies: - fast-json-stringify "^5.7.0" + fast-json-stringify "^6.0.0" -"@fastify/merge-json-schemas@^0.1.0": +"@fastify/merge-json-schemas@^0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@fastify/merge-json-schemas/-/merge-json-schemas-0.1.1.tgz#3551857b8a17a24e8c799e9f51795edb07baa0bc" integrity sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA== dependencies: fast-deep-equal "^3.1.3" -"@fastify/send@^2.0.0": - version "2.1.0" - resolved "https://registry.yarnpkg.com/@fastify/send/-/send-2.1.0.tgz#1aa269ccb4b0940a2dadd1f844443b15d8224ea0" - integrity sha512-yNYiY6sDkexoJR0D8IDy3aRP3+L4wdqCpvx5WP+VtEU58sn7USmKynBzDQex5X42Zzvw2gNzzYgP90UfWShLFA== +"@fastify/send@^3.1.0": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@fastify/send/-/send-3.1.1.tgz#455a8fa56ae005c4c387ddf111364f346b848117" + integrity sha512-LdiV2mle/2tH8vh6GwGl0ubfUAgvY+9yF9oGI1iiwVyNUVOQamvw5n+OFu6iCNNoyuCY80FFURBn4TZCbTe8LA== dependencies: - "@lukeed/ms" "^2.0.1" + "@lukeed/ms" "^2.0.2" escape-html "~1.0.3" fast-decode-uri-component "^1.0.1" - http-errors "2.0.0" - mime "^3.0.0" - -"@fastify/static@^6.0.0": - version "6.11.2" - resolved "https://registry.yarnpkg.com/@fastify/static/-/static-6.11.2.tgz#1fe40c40daf055a28d29db807b459fcff431d9b6" - integrity sha512-EH7mh7q4MfNdT7N07ZVlwsX/ObngMvQ7KBP0FXAuPov99Fjn80KSJMdxQhhYKAKWW1jXiFdrk8X7d6uGWdZFxg== - dependencies: - "@fastify/accept-negotiator" "^1.0.0" - "@fastify/send" "^2.0.0" - content-disposition "^0.5.3" - fastify-plugin "^4.0.0" - glob "^8.0.1" - p-limit "^3.1.0" + http-errors "^2.0.0" + mime "^3" -"@fastify/swagger-ui@^1.9.3": - version "1.9.3" - resolved "https://registry.yarnpkg.com/@fastify/swagger-ui/-/swagger-ui-1.9.3.tgz#1ec03ea2595cb2e7d6de6ae7c949bebcff8370a5" - integrity sha512-YYqce4CydjDIEry6Zo4JLjVPe5rjS8iGnk3fHiIQnth9sFSLeyG0U1DCH+IyYmLddNDg1uWJOuErlVqnu/jI3w== +"@fastify/static@^8.0.0": + version "8.0.1" + resolved "https://registry.yarnpkg.com/@fastify/static/-/static-8.0.1.tgz#137059a4625c64cce8ee7eb513961c5e23018805" + integrity sha512-7idyhbcgf14v4bjWzUeHEFvnVxvNJ1n5cyGPgFtwTZjnjUQ1wgC7a2FQai7OGKqCKywDEjzbPhAZRW+uEK1LMg== dependencies: - "@fastify/static" "^6.0.0" - fastify-plugin "^4.0.0" - openapi-types "^12.0.2" - rfdc "^1.3.0" - yaml "^2.2.2" + "@fastify/accept-negotiator" "^2.0.0" + "@fastify/send" "^3.1.0" + content-disposition "^0.5.4" + fastify-plugin "^5.0.0" + fastq "^1.17.1" + glob "^11.0.0" -"@fastify/swagger@^8.10.0": - version "8.10.0" - resolved "https://registry.yarnpkg.com/@fastify/swagger/-/swagger-8.10.0.tgz#d978ae9f2d802ab652955d02be7a125f7f6d9f05" - integrity sha512-0o6nd0qWpJbVSv/vbK4bzHSYe7l+PTGPqrQVwWIXVGd7CvXr585SBx+h8EgrMOY80bcOnGreqnjYFOV0osGP5A== +"@fastify/swagger-ui@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@fastify/swagger-ui/-/swagger-ui-5.0.1.tgz#76c348bbaf7e49e3cfb62ebe3cc3fb15ef0eefb3" + integrity sha512-nCDV5l0OTziK8nIeHaLZ30ENFFftZ4Pcs7GHDcqOO6Jp3qSnyOsqBg1/EosM+d1mrCvH4vSlM09xolkjrbuJQQ== dependencies: - fastify-plugin "^4.0.0" + "@fastify/static" "^8.0.0" + fastify-plugin "^5.0.0" + openapi-types "^12.1.3" + rfdc "^1.3.1" + yaml "^2.4.1" + +"@fastify/swagger@^9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@fastify/swagger/-/swagger-9.0.0.tgz#a6013ee3cf4ec0f2562e1455face9eb6ef787d89" + integrity sha512-E7TQbBCbhvS2djGLxJ7t2OFbhc2F+KCsOZCNhh6xQIlJxq9H4ZR5KuLKG+vn6COVqkLxRVUOZ9qtbbzdf5Jfqw== + dependencies: + fastify-plugin "^5.0.0" json-schema-resolver "^2.0.0" - openapi-types "^12.0.0" - rfdc "^1.3.0" - yaml "^2.2.2" + openapi-types "^12.1.3" + rfdc "^1.3.1" + yaml "^2.4.2" "@gar/promisify@^1.0.1", "@gar/promisify@^1.1.3": version "1.1.3" @@ -1857,10 +1853,10 @@ race-signal "^1.0.2" uint8arraylist "^2.4.8" -"@lukeed/ms@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@lukeed/ms/-/ms-2.0.1.tgz#3c2bbc258affd9cc0e0cc7828477383c73afa6ee" - integrity sha512-Xs/4RZltsAL7pkvaNStUQt7netTkyxrS0K+RILcVr3TRMS/ToOg4I6uNfhB9SlGsnWBym4U+EaXq0f0cEMNkHA== +"@lukeed/ms@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@lukeed/ms/-/ms-2.0.2.tgz#07f09e59a74c52f4d88c6db5c1054e819538e2a8" + integrity sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA== "@microsoft/api-extractor-model@7.28.13": version "7.28.13" @@ -3887,10 +3883,10 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" -ajv-formats@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" - integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA== +ajv-formats@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-3.0.1.tgz#3d5dc762bca17679c3c2ea7e90ad6b7532309578" + integrity sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ== dependencies: ajv "^8.0.0" @@ -3904,7 +3900,7 @@ ajv@^6.12.4, ajv@~6.12.6: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.0.0, ajv@^8.10.0, ajv@^8.11.0, ajv@^8.12.0: +ajv@^8.0.0, ajv@^8.12.0: version "8.12.0" resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== @@ -4060,11 +4056,6 @@ archiver@^7.0.0: tar-stream "^3.0.0" zip-stream "^6.0.1" -archy@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz" - integrity sha1-+cjBN1fMHde8N5rHeyxipcKGjEA= - are-we-there-yet@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.0.tgz" @@ -4287,14 +4278,12 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -avvio@^8.3.0: - version "8.3.0" - resolved "https://registry.yarnpkg.com/avvio/-/avvio-8.3.0.tgz#1e019433d935730b814978a583eefac41a65082f" - integrity sha512-VBVH0jubFr9LdFASy/vNtm5giTrnbVquWBhT0fyizuNK2rQ7e7ONU2plZQWUNqtE1EmxFEb+kbSkFRkstiaS9Q== +avvio@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/avvio/-/avvio-9.0.0.tgz#3ae02fb318377006e0e06a3f47842c98d8668607" + integrity sha512-UbYrOXgE/I+knFG+3kJr9AgC7uNo8DG+FGGODpH9Bj1O1kL/QDjBXnTem9leD3VdQKtaHjV3O85DQ7hHh4IIHw== dependencies: - "@fastify/error" "^3.3.0" - archy "^1.0.0" - debug "^4.0.0" + "@fastify/error" "^4.0.0" fastq "^1.17.1" aws-sdk@^2.932.0: @@ -5236,7 +5225,7 @@ constants-browserify@^1.0.0: resolved "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz" integrity sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U= -content-disposition@^0.5.3: +content-disposition@^0.5.4: version "0.5.4" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== @@ -5558,7 +5547,7 @@ de-indent@^1.0.2: resolved "https://registry.yarnpkg.com/de-indent/-/de-indent-1.0.2.tgz#b2038e846dc33baa5796128d0804b455b8c1e21d" integrity sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg== -debug@4, debug@4.3.4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4: +debug@4, debug@4.3.4, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -6568,11 +6557,6 @@ extract-zip@^2.0.1: optionalDependencies: "@types/yauzl" "^2.9.1" -fast-content-type-parse@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/fast-content-type-parse/-/fast-content-type-parse-1.1.0.tgz#4087162bf5af3294d4726ff29b334f72e3a1092c" - integrity sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ== - fast-decode-uri-component@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz" @@ -6614,28 +6598,16 @@ fast-json-stable-stringify@^2.0.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-json-stringify@^5.7.0: - version "5.7.0" - resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-5.7.0.tgz#b0a04c848fdeb6ecd83440c71a4db35067023bed" - integrity sha512-sBVPTgnAZseLu1Qgj6lUbQ0HfjFhZWXAmpZ5AaSGkyLh5gAXBga/uPJjQPHpDFjC9adWIpdOcCLSDTgrZ7snoQ== - dependencies: - "@fastify/deepmerge" "^1.0.0" - ajv "^8.10.0" - ajv-formats "^2.1.1" - fast-deep-equal "^3.1.3" - fast-uri "^2.1.0" - rfdc "^1.2.0" - -fast-json-stringify@^5.8.0: - version "5.13.0" - resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-5.13.0.tgz#3eafc02168713ef934d75000a8cf749492729fe8" - integrity sha512-XjTDWKHP3GoMQUOfnjYUbqeHeEt+PvYgvBdG2fRSmYaORILbSr8xTJvZX+w1YSAP5pw2NwKrGRmQleYueZEoxw== +fast-json-stringify@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-6.0.0.tgz#15c5e85b567ead695773bf55938b56aaaa57d805" + integrity sha512-FGMKZwniMTgZh7zQp9b6XnBVxUmKVahQLQeRQHqwYmPDqDhcEKZ3BaQsxelFFI5PY7nN71OEeiL47/zUWcYe1A== dependencies: - "@fastify/merge-json-schemas" "^0.1.0" - ajv "^8.10.0" - ajv-formats "^2.1.1" + "@fastify/merge-json-schemas" "^0.1.1" + ajv "^8.12.0" + ajv-formats "^3.0.1" fast-deep-equal "^3.1.3" - fast-uri "^2.1.0" + fast-uri "^2.3.0" json-schema-ref-resolver "^1.0.1" rfdc "^1.2.0" @@ -6661,37 +6633,41 @@ fast-safe-stringify@^2.1.1: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== -fast-uri@^2.0.0, fast-uri@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-2.2.0.tgz#519a0f849bef714aad10e9753d69d8f758f7445a" - integrity sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg== +fast-uri@^2.3.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-2.4.0.tgz#67eae6fbbe9f25339d5d3f4c4234787b65d7d55e" + integrity sha512-ypuAmmMKInk5q7XcepxlnUWDLWv4GFtaJqAzWKqn62IpQ3pejtr5dTVbt3vwqVaMKmkNR55sTT+CqUKIaT21BA== -fastify-plugin@^4.0.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-4.5.0.tgz#8b853923a0bba6ab6921bb8f35b81224e6988d91" - integrity sha512-79ak0JxddO0utAXAQ5ccKhvs6vX2MGyHHMMsmZkBANrq3hXc1CHzvNPHOcvTsVMEPl5I+NT+RO4YKMGehOfSIg== +fast-uri@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.0.1.tgz#cddd2eecfc83a71c1be2cc2ef2061331be8a7134" + integrity sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw== -fastify@^4.27.0: - version "4.27.0" - resolved "https://registry.yarnpkg.com/fastify/-/fastify-4.27.0.tgz#e4a9b2a0a7b9efaeaf1140d47fdd4f91b5fcacb1" - integrity sha512-ci9IXzbigB8dyi0mSy3faa3Bsj0xWAPb9JeT4KRzubdSb6pNhcADRUaXCBml6V1Ss/a05kbtQls5LBmhHydoTA== +fastify-plugin@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-5.0.1.tgz#82d44e6fe34d1420bb5a4f7bee434d501e41939f" + integrity sha512-HCxs+YnRaWzCl+cWRYFnHmeRFyR5GVnJTAaCJQiYzQSDwK9MgJdyAsuL3nh0EWRCYMgQ5MeziymvmAhUHYHDUQ== + +fastify@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/fastify/-/fastify-5.0.0.tgz#f8f80bd741bde2de1997c25dbe31e61c91978111" + integrity sha512-Qe4dU+zGOzg7vXjw4EvcuyIbNnMwTmcuOhlOrOJsgwzvjEZmsM/IeHulgJk+r46STjdJS/ZJbxO8N70ODXDMEQ== dependencies: - "@fastify/ajv-compiler" "^3.5.0" - "@fastify/error" "^3.4.0" - "@fastify/fast-json-stringify-compiler" "^4.3.0" + "@fastify/ajv-compiler" "^4.0.0" + "@fastify/error" "^4.0.0" + "@fastify/fast-json-stringify-compiler" "^5.0.0" abstract-logging "^2.0.1" - avvio "^8.3.0" - fast-content-type-parse "^1.1.0" - fast-json-stringify "^5.8.0" - find-my-way "^8.0.0" - light-my-request "^5.11.0" + avvio "^9.0.0" + fast-json-stringify "^6.0.0" + find-my-way "^9.0.0" + light-my-request "^6.0.0" pino "^9.0.0" - process-warning "^3.0.0" + process-warning "^4.0.0" proxy-addr "^2.0.7" - rfdc "^1.3.0" + rfdc "^1.3.1" secure-json-parse "^2.7.0" - semver "^7.5.4" - toad-cache "^3.3.0" + semver "^7.6.0" + toad-cache "^3.7.0" fastq@^1.17.1: version "1.17.1" @@ -6782,14 +6758,14 @@ fill-range@^7.1.1: dependencies: to-regex-range "^5.0.1" -find-my-way@^8.0.0: - version "8.2.2" - resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-8.2.2.tgz#f3e78bc6ead2da4fdaa201335da3228600ed0285" - integrity sha512-Dobi7gcTEq8yszimcfp/R7+owiT4WncAJ7VTTgFH1jYJ5GaG1FbhjwDG820hptN0QDFvzVY3RfCzdInvGPGzjA== +find-my-way@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-9.0.1.tgz#991c3a7af36734480d48cd4ad0889ed168ed6c40" + integrity sha512-/5NN/R0pFWuff16TMajeKt2JyiW+/OE8nOO8vo1DwZTxLaIURb7lcBYPIgRPh61yCNh9l8voeKwcrkUzmB00vw== dependencies: fast-deep-equal "^3.1.3" fast-querystring "^1.0.0" - safe-regex2 "^3.1.0" + safe-regex2 "^4.0.0" find-up@5.0.0, find-up@^5.0.0: version "5.0.0" @@ -7277,6 +7253,18 @@ glob@^10.4.1: package-json-from-dist "^1.0.0" path-scurry "^1.11.1" +glob@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-11.0.0.tgz#6031df0d7b65eaa1ccb9b29b5ced16cea658e77e" + integrity sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g== + dependencies: + foreground-child "^3.1.0" + jackspeak "^4.0.1" + minimatch "^10.0.0" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^2.0.0" + glob@^7.1.3, glob@^7.1.4: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" @@ -7573,7 +7561,7 @@ http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^ resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== -http-errors@2.0.0: +http-errors@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== @@ -8440,6 +8428,15 @@ jackspeak@^3.1.2: optionalDependencies: "@pkgjs/parseargs" "^0.11.0" +jackspeak@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-4.0.1.tgz#9fca4ce961af6083e259c376e9e3541431f5287b" + integrity sha512-cub8rahkh0Q/bw1+GxP7aeSe29hHHn2V4m29nnDlvCdlgU+3UGxkZp7Z53jLUdpX3jdTO0nJZUDl3xvbWc2Xog== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + jake@^10.8.5: version "10.8.5" resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" @@ -8829,14 +8826,14 @@ libp2p@1.4.3: multiformats "^13.1.0" uint8arrays "^5.0.3" -light-my-request@^5.11.0: - version "5.12.0" - resolved "https://registry.yarnpkg.com/light-my-request/-/light-my-request-5.12.0.tgz#e42ed02ddbfa587f82031b21459c6841a6948dfa" - integrity sha512-P526OX6E7aeCIfw/9UyJNsAISfcFETghysaWHQAlQYayynShT08MOj4c6fBCvTWBrHXSvqBAKDp3amUPSCQI4w== +light-my-request@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/light-my-request/-/light-my-request-6.0.0.tgz#97c6d0d5448ea2fc37836f0aefe94298f5a87dde" + integrity sha512-kFkFXrmKCL0EEeOmJybMH5amWFd+AFvlvMlvFTRxCUwbhfapZqDmeLMPoWihntnYY6JpoQDE9k+vOzObF1fDqg== dependencies: cookie "^0.6.0" - process-warning "^3.0.0" - set-cookie-parser "^2.4.1" + process-warning "^4.0.0" + set-cookie-parser "^2.6.0" lines-and-columns@^1.1.6: version "1.2.4" @@ -9066,6 +9063,11 @@ lru-cache@^10.2.0: resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== +lru-cache@^11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.0.1.tgz#3a732fbfedb82c5ba7bca6564ad3f42afcb6e147" + integrity sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ== + lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" @@ -9320,7 +9322,7 @@ mime@2.6.0: resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367" integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg== -mime@^3.0.0: +mime@^3: version "3.0.0" resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7" integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== @@ -9386,6 +9388,13 @@ minimatch@9.0.3, minimatch@^9.0.0, minimatch@^9.0.1: dependencies: brace-expansion "^2.0.1" +minimatch@^10.0.0: + version "10.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.0.1.tgz#ce0521856b453c86e25f2c4c0d03e6ff7ddc440b" + integrity sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ== + dependencies: + brace-expansion "^2.0.1" + minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -9568,10 +9577,10 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -mnemonist@0.39.5: - version "0.39.5" - resolved "https://registry.yarnpkg.com/mnemonist/-/mnemonist-0.39.5.tgz#5850d9b30d1b2bc57cc8787e5caa40f6c3420477" - integrity sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ== +mnemonist@0.39.8: + version "0.39.8" + resolved "https://registry.yarnpkg.com/mnemonist/-/mnemonist-0.39.8.tgz#9078cd8386081afd986cca34b52b5d84ea7a4d38" + integrity sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ== dependencies: obliterator "^2.0.1" @@ -10276,7 +10285,7 @@ open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openapi-types@^12.0.0, openapi-types@^12.0.2: +openapi-types@^12.1.3: version "12.1.3" resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3" integrity sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw== @@ -10372,7 +10381,7 @@ p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.0.2, p-limit@^3.1.0: +p-limit@^3.0.2: version "3.1.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== @@ -10660,6 +10669,14 @@ path-scurry@^1.11.1: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" +path-scurry@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-2.0.0.tgz#9f052289f23ad8bf9397a2a0425e7b8615c58580" + integrity sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg== + dependencies: + lru-cache "^11.0.0" + minipass "^7.1.2" + path-to-regexp@^6.2.0: version "6.3.0" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz#2b6a26a337737a8e1416f9272ed0766b1c0389f4" @@ -10846,6 +10863,11 @@ process-warning@^3.0.0: resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-3.0.0.tgz#96e5b88884187a1dce6f5c3166d611132058710b" integrity sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ== +process-warning@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-4.0.0.tgz#581e3a7a1fb456c5f4fd239f76bce75897682d5a" + integrity sha512-/MyYDxttz7DfGMMHiysAsFE4qF+pQYAA8ziO/3NcRVrQ5fSk+Mns4QZA/oRPFzvcqNoVJXQNWNAsdwBXLUkQKw== + process@^0.11.10: version "0.11.10" resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" @@ -11358,10 +11380,10 @@ restore-cursor@^4.0.0: onetime "^5.1.0" signal-exit "^3.0.2" -ret@~0.4.0: - version "0.4.3" - resolved "https://registry.yarnpkg.com/ret/-/ret-0.4.3.tgz#5243fa30e704a2e78a9b9b1e86079e15891aa85c" - integrity sha512-0f4Memo5QP7WQyUEAYUO3esD/XjOc3Zjjg5CPsAq1p8sIu0XPeMbHJemKA0BO7tV0X7+A0FoEpbmHXWxPyD3wQ== +ret@~0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/ret/-/ret-0.5.0.tgz#30a4d38a7e704bd96dc5ffcbe7ce2a9274c41c95" + integrity sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw== retry@^0.12.0: version "0.12.0" @@ -11387,10 +11409,10 @@ rewiremock@^3.14.5: wipe-node-cache "^2.1.2" wipe-webpack-cache "^2.1.0" -rfdc@^1.1.4, rfdc@^1.2.0, rfdc@^1.3.0: - version "1.3.0" - resolved "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz" - integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== +rfdc@^1.1.4, rfdc@^1.2.0, rfdc@^1.3.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.4.1.tgz#778f76c4fb731d93414e8f925fbecf64cce7f6ca" + integrity sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA== rgb2hex@0.2.5: version "0.2.5" @@ -11550,12 +11572,12 @@ safe-regex-test@^1.0.0: get-intrinsic "^1.1.3" is-regex "^1.1.4" -safe-regex2@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/safe-regex2/-/safe-regex2-3.1.0.tgz#fd7ec23908e2c730e1ce7359a5b72883a87d2763" - integrity sha512-RAAZAGbap2kBfbVhvmnTFv73NWLMvDGOITFYTZBAaY8eR+Ir4ef7Up/e7amo+y1+AH+3PtLkrt9mvcTsG9LXug== +safe-regex2@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/safe-regex2/-/safe-regex2-4.0.0.tgz#5e04d8362cd4884753c8bce9715d4759a5239c0a" + integrity sha512-Hvjfv25jPDVr3U+4LDzBuZPPOymELG3PYcSk5hcevooo1yxxamQL/bHs/GrEPGmMoMEwRrHVGiCA1pXi97B8Ew== dependencies: - ret "~0.4.0" + ret "~0.5.0" safe-stable-stringify@^2.3.1: version "2.4.2" @@ -11654,10 +11676,10 @@ set-blocking@^2.0.0: resolved "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= -set-cookie-parser@^2.4.1: - version "2.4.8" - resolved "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.4.8.tgz" - integrity sha512-edRH8mBKEWNVIVMKejNnuJxleqYE/ZSdcT8/Nem9/mmosx12pctd80s2Oy00KNZzrogMZS5mauK2/ymL1bvlvg== +set-cookie-parser@^2.6.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/set-cookie-parser/-/set-cookie-parser-2.7.0.tgz#ef5552b56dc01baae102acb5fc9fb8cd060c30f9" + integrity sha512-lXLOiqpkUumhRdFF3k1osNXCy9akgx/dyPZ5p8qAg9seJzXr5ZrlqZuWIMuY6ejOsVLE6flJ5/h3lsn57fQ/PQ== set-function-length@^1.1.1: version "1.2.0" @@ -12529,7 +12551,7 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -toad-cache@^3.3.0: +toad-cache@^3.7.0: version "3.7.0" resolved "https://registry.yarnpkg.com/toad-cache/-/toad-cache-3.7.0.tgz#b9b63304ea7c45ec34d91f1d2fa513517025c441" integrity sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw== @@ -13843,10 +13865,10 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== -yaml@^2.2.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.2.tgz#f522db4313c671a0ca963a75670f1c12ea909144" - integrity sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg== +yaml@^2.2.2, yaml@^2.4.1, yaml@^2.4.2: + version "2.5.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.5.1.tgz#c9772aacf62cb7494a95b0c4f1fb065b563db130" + integrity sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q== yargs-parser@20.2.4: version "20.2.4" From ad4ec7713d0ab39c54ea1ffa28d6f6f997769fa0 Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:33:43 -0700 Subject: [PATCH 125/145] feat: move requests from execution payload to beacon block body (#7094) * Move requests from payload to block body * Lint * Add execution requests to engine api * Remove engine_getPayloadBodies*V2 * Update spec test version * Lint * Fix unit test and polish * Remove todo --- .../blocks/verifyBlocksExecutionPayloads.ts | 7 +- .../chain/produceBlock/produceBlockBody.ts | 10 +- .../beacon-node/src/execution/engine/http.ts | 41 ++++-- .../src/execution/engine/interface.ts | 10 +- .../beacon-node/src/execution/engine/mock.ts | 2 - .../src/execution/engine/payloadIdCache.ts | 20 --- .../beacon-node/src/execution/engine/types.ts | 135 ++++++++---------- .../test/sim/electra-interop.test.ts | 8 +- .../test/spec/specTestVersioning.ts | 2 +- .../test/spec/utils/specTestIterator.ts | 2 + .../upgradeLightClientHeader.test.ts | 2 +- .../test/unit/executionEngine/http.test.ts | 12 -- packages/light-client/src/spec/utils.ts | 17 +-- .../src/block/processOperations.ts | 6 +- .../src/slot/upgradeStateToElectra.ts | 9 +- .../state-transition/src/util/execution.ts | 10 +- packages/types/src/electra/sszTypes.ts | 66 +++------ packages/types/src/electra/types.ts | 6 +- packages/types/src/types.ts | 18 ++- packages/types/src/utils/typeguards.ts | 6 +- 20 files changed, 160 insertions(+), 229 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index 284d4c0976ee..e641ff9ae6d9 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -5,7 +5,7 @@ import { isMergeTransitionBlock as isMergeTransitionBlockFn, isExecutionEnabled, } from "@lodestar/state-transition"; -import {bellatrix, Slot, deneb, SignedBeaconBlock} from "@lodestar/types"; +import {bellatrix, Slot, deneb, SignedBeaconBlock, electra} from "@lodestar/types"; import { IForkChoice, assertValidTerminalPowBlock, @@ -302,6 +302,8 @@ export async function verifyBlockExecutionPayload( ? (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.map(kzgCommitmentToVersionedHash) : undefined; const parentBlockRoot = ForkSeq[fork] >= ForkSeq.deneb ? block.message.parentRoot : undefined; + const executionRequests = + ForkSeq[fork] >= ForkSeq.electra ? (block.message.body as electra.BeaconBlockBody).executionRequests : undefined; const logCtx = {slot: block.message.slot, executionBlock: executionPayloadEnabled.blockNumber}; chain.logger.debug("Call engine api newPayload", logCtx); @@ -309,7 +311,8 @@ export async function verifyBlockExecutionPayload( fork, executionPayloadEnabled, versionedHashes, - parentBlockRoot + parentBlockRoot, + executionRequests ); chain.logger.debug("Receive engine api newPayload result", {...logCtx, status: execResult.status}); diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index ba560d5a7ff0..ff8221a326e9 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -17,6 +17,7 @@ import { BlindedBeaconBlockBody, BlindedBeaconBlock, sszTypesFor, + electra, } from "@lodestar/types"; import { CachedBeaconStateAllForks, @@ -258,7 +259,7 @@ export async function produceBlockBody( } const engineRes = await this.executionEngine.getPayload(fork, payloadId); - const {executionPayload, blobsBundle} = engineRes; + const {executionPayload, blobsBundle, executionRequests} = engineRes; shouldOverrideBuilder = engineRes.shouldOverrideBuilder; (blockBody as BeaconBlockBody).executionPayload = executionPayload; @@ -298,6 +299,13 @@ export async function produceBlockBody( } else { blobsResult = {type: BlobsResultType.preDeneb}; } + + if (ForkSeq[fork] >= ForkSeq.electra) { + if (executionRequests === undefined) { + throw Error(`Missing executionRequests response from getPayload at fork=${fork}`); + } + (blockBody as electra.BeaconBlockBody).executionRequests = executionRequests; + } } } catch (e) { this.metrics?.blockPayload.payloadFetchErrors.inc(); diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index a69a5b94bd65..b42424b28998 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -1,4 +1,4 @@ -import {ExecutionPayload, Root, RootHex, Wei} from "@lodestar/types"; +import {ExecutionPayload, ExecutionRequests, Root, RootHex, Wei} from "@lodestar/types"; import {SLOTS_PER_EPOCH, ForkName, ForkSeq} from "@lodestar/params"; import {Logger} from "@lodestar/logger"; import { @@ -37,6 +37,7 @@ import { ExecutionPayloadBody, assertReqSizeLimit, deserializeExecutionPayloadBody, + serializeExecutionRequests, } from "./types.js"; import {getExecutionEngineState} from "./utils.js"; @@ -195,7 +196,8 @@ export class ExecutionEngineHttp implements IExecutionEngine { fork: ForkName, executionPayload: ExecutionPayload, versionedHashes?: VersionedHashes, - parentBlockRoot?: Root + parentBlockRoot?: Root, + executionRequests?: ExecutionRequests ): Promise { const method = ForkSeq[fork] >= ForkSeq.electra @@ -220,12 +222,28 @@ export class ExecutionEngineHttp implements IExecutionEngine { const serializedVersionedHashes = serializeVersionedHashes(versionedHashes); const parentBeaconBlockRoot = serializeBeaconBlockRoot(parentBlockRoot); - const method = ForkSeq[fork] >= ForkSeq.electra ? "engine_newPayloadV4" : "engine_newPayloadV3"; - engineRequest = { - method, - params: [serializedExecutionPayload, serializedVersionedHashes, parentBeaconBlockRoot], - methodOpts: notifyNewPayloadOpts, - }; + if (ForkSeq[fork] >= ForkSeq.electra) { + if (executionRequests === undefined) { + throw Error(`executionRequests required in notifyNewPayload for fork=${fork}`); + } + const serializedExecutionRequests = serializeExecutionRequests(executionRequests); + engineRequest = { + method: "engine_newPayloadV4", + params: [ + serializedExecutionPayload, + serializedVersionedHashes, + parentBeaconBlockRoot, + serializedExecutionRequests, + ], + methodOpts: notifyNewPayloadOpts, + }; + } else { + engineRequest = { + method: "engine_newPayloadV3", + params: [serializedExecutionPayload, serializedVersionedHashes, parentBeaconBlockRoot], + methodOpts: notifyNewPayloadOpts, + }; + } } else { const method = ForkSeq[fork] >= ForkSeq.capella ? "engine_newPayloadV2" : "engine_newPayloadV1"; engineRequest = { @@ -391,6 +409,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { executionPayload: ExecutionPayload; executionPayloadValue: Wei; blobsBundle?: BlobsBundle; + executionRequests?: ExecutionRequests; shouldOverrideBuilder?: boolean; }> { const method = @@ -419,8 +438,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { } async getPayloadBodiesByHash(fork: ForkName, blockHashes: RootHex[]): Promise<(ExecutionPayloadBody | null)[]> { - const method = - ForkSeq[fork] >= ForkSeq.electra ? "engine_getPayloadBodiesByHashV2" : "engine_getPayloadBodiesByHashV1"; + const method = "engine_getPayloadBodiesByHashV1"; assertReqSizeLimit(blockHashes.length, 32); const response = await this.rpc.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], @@ -434,8 +452,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { startBlockNumber: number, blockCount: number ): Promise<(ExecutionPayloadBody | null)[]> { - const method = - ForkSeq[fork] >= ForkSeq.electra ? "engine_getPayloadBodiesByRangeV2" : "engine_getPayloadBodiesByRangeV1"; + const method = "engine_getPayloadBodiesByRangeV1"; assertReqSizeLimit(blockCount, 32); const start = numToQuantity(startBlockNumber); const count = numToQuantity(blockCount); diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index 5226a46ac720..e6f9cfee526b 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -1,12 +1,12 @@ import {ForkName} from "@lodestar/params"; import {KZGCommitment, Blob, KZGProof} from "@lodestar/types/deneb"; -import {Root, RootHex, capella, Wei, ExecutionPayload} from "@lodestar/types"; +import {Root, RootHex, capella, Wei, ExecutionPayload, ExecutionRequests} from "@lodestar/types"; import {DATA} from "../../eth1/provider/utils.js"; -import {PayloadIdCache, PayloadId, WithdrawalV1, DepositRequestV1} from "./payloadIdCache.js"; +import {PayloadIdCache, PayloadId, WithdrawalV1} from "./payloadIdCache.js"; import {ExecutionPayloadBody} from "./types.js"; -export {PayloadIdCache, type PayloadId, type WithdrawalV1, type DepositRequestV1}; +export {PayloadIdCache, type PayloadId, type WithdrawalV1}; export enum ExecutionPayloadStatus { /** given payload is valid */ @@ -134,7 +134,8 @@ export interface IExecutionEngine { fork: ForkName, executionPayload: ExecutionPayload, versionedHashes?: VersionedHashes, - parentBeaconBlockRoot?: Root + parentBeaconBlockRoot?: Root, + executionRequests?: ExecutionRequests ): Promise; /** @@ -171,6 +172,7 @@ export interface IExecutionEngine { executionPayload: ExecutionPayload; executionPayloadValue: Wei; blobsBundle?: BlobsBundle; + executionRequests?: ExecutionRequests; shouldOverrideBuilder?: boolean; }>; diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index 9fed781fa523..4cff2a8d00f4 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -98,10 +98,8 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_getPayloadV3: this.getPayload.bind(this), engine_getPayloadV4: this.getPayload.bind(this), engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this), - engine_getPayloadBodiesByHashV2: this.getPayloadBodiesByHash.bind(this), engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this), engine_getClientVersionV1: this.getClientVersionV1.bind(this), - engine_getPayloadBodiesByRangeV2: this.getPayloadBodiesByRange.bind(this), }; } diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index 005a1ef14322..ea37e0922e9c 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -18,26 +18,6 @@ export type WithdrawalV1 = { amount: QUANTITY; }; -export type DepositRequestV1 = { - pubkey: DATA; - withdrawalCredentials: DATA; - amount: QUANTITY; - signature: DATA; - index: QUANTITY; -}; - -export type WithdrawalRequestV1 = { - sourceAddress: DATA; - validatorPubkey: DATA; - amount: QUANTITY; -}; - -export type ConsolidationRequestV1 = { - sourceAddress: DATA; - sourcePubkey: DATA; - targetPubkey: DATA; -}; - type FcuAttributes = {headBlockHash: DATA; finalizedBlockHash: DATA} & Omit; export class PayloadIdCache { diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 139fc6822780..32fe4cb79d3d 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -1,4 +1,4 @@ -import {capella, deneb, electra, Wei, bellatrix, Root, ExecutionPayload} from "@lodestar/types"; +import {capella, deneb, electra, Wei, bellatrix, Root, ExecutionPayload, ExecutionRequests} from "@lodestar/types"; import { BYTES_PER_LOGS_BLOOM, FIELD_ELEMENTS_PER_BLOB, @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1, DepositRequestV1, WithdrawalRequestV1, ConsolidationRequestV1} from "./payloadIdCache.js"; +import {WithdrawalV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -28,7 +28,7 @@ export type EngineApiRpcParamTypes = { engine_newPayloadV1: [ExecutionPayloadRpc]; engine_newPayloadV2: [ExecutionPayloadRpc]; engine_newPayloadV3: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; - engine_newPayloadV4: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; + engine_newPayloadV4: [ExecutionPayloadRpc, VersionedHashesRpc, DATA, ExecutionRequestsRpc]; /** * 1. Object - Payload validity status with respect to the consensus rules: * - blockHash: DATA, 32 Bytes - block hash value of the payload @@ -58,7 +58,6 @@ export type EngineApiRpcParamTypes = { * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure * */ engine_getPayloadBodiesByHashV1: DATA[][]; - engine_getPayloadBodiesByHashV2: DATA[][]; /** * 1. start: QUANTITY, 64 bits - Starting block number @@ -70,7 +69,6 @@ export type EngineApiRpcParamTypes = { * Object - Instance of ClientVersion */ engine_getClientVersionV1: [ClientVersionRpc]; - engine_getPayloadBodiesByRangeV2: [start: QUANTITY, count: QUANTITY]; }; export type PayloadStatus = { @@ -109,12 +107,10 @@ export type EngineApiRpcReturnTypes = { engine_getPayloadV4: ExecutionPayloadResponse; engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[]; - engine_getPayloadBodiesByHashV2: (ExecutionPayloadBodyRpc | null)[]; engine_getPayloadBodiesByRangeV1: (ExecutionPayloadBodyRpc | null)[]; engine_getClientVersionV1: ClientVersionRpc[]; - engine_getPayloadBodiesByRangeV2: (ExecutionPayloadBodyRpc | null)[]; }; type ExecutionPayloadRpcWithValue = { @@ -122,6 +118,7 @@ type ExecutionPayloadRpcWithValue = { // even though CL tracks this as executionPayloadValue, EL returns this as blockValue blockValue: QUANTITY; blobsBundle?: BlobsBundleRpc; + requests?: ExecutionRequestsRpc; shouldOverrideBuilder?: boolean; }; type ExecutionPayloadResponse = ExecutionPayloadRpc | ExecutionPayloadRpcWithValue; @@ -129,19 +126,11 @@ type ExecutionPayloadResponse = ExecutionPayloadRpc | ExecutionPayloadRpcWithVal export type ExecutionPayloadBodyRpc = { transactions: DATA[]; withdrawals: WithdrawalV1[] | null | undefined; - // currently there is a discepancy between EL and CL field name references for deposit requests - // its likely CL receipt will be renamed to requests - depositRequests: DepositRequestV1[] | null | undefined; - withdrawalRequests: WithdrawalRequestV1[] | null | undefined; - consolidationRequests: ConsolidationRequestV1[] | null | undefined; }; export type ExecutionPayloadBody = { transactions: bellatrix.Transaction[]; withdrawals: capella.Withdrawals | null; - depositRequests: electra.DepositRequests | null; - withdrawalRequests: electra.WithdrawalRequests | null; - consolidationRequests: electra.ConsolidationRequests | null; }; export type ExecutionPayloadRpc = { @@ -163,9 +152,6 @@ export type ExecutionPayloadRpc = { blobGasUsed?: QUANTITY; // DENEB excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB - depositRequests?: DepositRequestRpc[]; // ELECTRA - withdrawalRequests?: WithdrawalRequestRpc[]; // ELECTRA - consolidationRequests?: ConsolidationRequestRpc[]; // ELECTRA }; export type WithdrawalRpc = { @@ -175,9 +161,29 @@ export type WithdrawalRpc = { amount: QUANTITY; }; -export type DepositRequestRpc = DepositRequestV1; -export type WithdrawalRequestRpc = WithdrawalRequestV1; -export type ConsolidationRequestRpc = ConsolidationRequestV1; +export type ExecutionRequestsRpc = { + deposits: DepositRequestRpc[]; + withdrawals: WithdrawalRequestRpc[]; + consolidations: ConsolidationRequestRpc[]; +}; + +export type DepositRequestRpc = { + pubkey: DATA; + withdrawalCredentials: DATA; + amount: QUANTITY; + signature: DATA; + index: QUANTITY; +}; +export type WithdrawalRequestRpc = { + sourceAddress: DATA; + validatorPubkey: DATA; + amount: QUANTITY; +}; +export type ConsolidationRequestRpc = { + sourceAddress: DATA; + sourcePubkey: DATA; + targetPubkey: DATA; +}; export type VersionedHashesRpc = DATA[]; @@ -241,13 +247,7 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload payload.excessBlobGas = numToQuantity(excessBlobGas); } - // ELECTRA adds depositRequests/depositRequests to the ExecutionPayload - if (ForkSeq[fork] >= ForkSeq.electra) { - const {depositRequests, withdrawalRequests, consolidationRequests} = data as electra.ExecutionPayload; - payload.depositRequests = depositRequests.map(serializeDepositRequest); - payload.withdrawalRequests = withdrawalRequests.map(serializeWithdrawalRequest); - payload.consolidationRequests = consolidationRequests.map(serializeConsolidationRequest); - } + // No changes in Electra return payload; } @@ -267,23 +267,27 @@ export function parseExecutionPayload( executionPayload: ExecutionPayload; executionPayloadValue: Wei; blobsBundle?: BlobsBundle; + executionRequests?: ExecutionRequests; shouldOverrideBuilder?: boolean; } { let data: ExecutionPayloadRpc; let executionPayloadValue: Wei; let blobsBundle: BlobsBundle | undefined; + let executionRequests: ExecutionRequests | undefined; let shouldOverrideBuilder: boolean; if (hasPayloadValue(response)) { executionPayloadValue = quantityToBigint(response.blockValue); data = response.executionPayload; blobsBundle = response.blobsBundle ? parseBlobsBundle(response.blobsBundle) : undefined; + executionRequests = response.requests ? deserializeExecutionRequests(response.requests) : undefined; shouldOverrideBuilder = response.shouldOverrideBuilder ?? false; } else { data = response; // Just set it to zero as default executionPayloadValue = BigInt(0); blobsBundle = undefined; + executionRequests = undefined; shouldOverrideBuilder = false; } @@ -334,36 +338,9 @@ export function parseExecutionPayload( (executionPayload as deneb.ExecutionPayload).excessBlobGas = quantityToBigint(excessBlobGas); } - if (ForkSeq[fork] >= ForkSeq.electra) { - // electra adds depositRequests/depositRequests - const {depositRequests, withdrawalRequests, consolidationRequests} = data; - // Geth can also reply with null - if (depositRequests == null) { - throw Error( - `depositRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` - ); - } - (executionPayload as electra.ExecutionPayload).depositRequests = depositRequests.map(deserializeDepositRequest); - - if (withdrawalRequests == null) { - throw Error( - `withdrawalRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` - ); - } - (executionPayload as electra.ExecutionPayload).withdrawalRequests = - withdrawalRequests.map(deserializeWithdrawalRequest); - - if (consolidationRequests == null) { - throw Error( - `consolidationRequests missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` - ); - } - (executionPayload as electra.ExecutionPayload).consolidationRequests = consolidationRequests.map( - deserializeConsolidationRequest - ); - } + // No changes in Electra - return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; + return {executionPayload, executionPayloadValue, blobsBundle, executionRequests, shouldOverrideBuilder}; } export function serializePayloadAttributes(data: PayloadAttributes): PayloadAttributesRpc { @@ -429,7 +406,7 @@ export function deserializeWithdrawal(serialized: WithdrawalRpc): capella.Withdr } as capella.Withdrawal; } -export function serializeDepositRequest(depositRequest: electra.DepositRequest): DepositRequestRpc { +function serializeDepositRequest(depositRequest: electra.DepositRequest): DepositRequestRpc { return { pubkey: bytesToData(depositRequest.pubkey), withdrawalCredentials: bytesToData(depositRequest.withdrawalCredentials), @@ -439,7 +416,7 @@ export function serializeDepositRequest(depositRequest: electra.DepositRequest): }; } -export function deserializeDepositRequest(serialized: DepositRequestRpc): electra.DepositRequest { +function deserializeDepositRequest(serialized: DepositRequestRpc): electra.DepositRequest { return { pubkey: dataToBytes(serialized.pubkey, 48), withdrawalCredentials: dataToBytes(serialized.withdrawalCredentials, 32), @@ -449,7 +426,7 @@ export function deserializeDepositRequest(serialized: DepositRequestRpc): electr } as electra.DepositRequest; } -export function serializeWithdrawalRequest(withdrawalRequest: electra.WithdrawalRequest): WithdrawalRequestRpc { +function serializeWithdrawalRequest(withdrawalRequest: electra.WithdrawalRequest): WithdrawalRequestRpc { return { sourceAddress: bytesToData(withdrawalRequest.sourceAddress), validatorPubkey: bytesToData(withdrawalRequest.validatorPubkey), @@ -457,7 +434,7 @@ export function serializeWithdrawalRequest(withdrawalRequest: electra.Withdrawal }; } -export function deserializeWithdrawalRequest(withdrawalRequest: WithdrawalRequestRpc): electra.WithdrawalRequest { +function deserializeWithdrawalRequest(withdrawalRequest: WithdrawalRequestRpc): electra.WithdrawalRequest { return { sourceAddress: dataToBytes(withdrawalRequest.sourceAddress, 20), validatorPubkey: dataToBytes(withdrawalRequest.validatorPubkey, 48), @@ -465,9 +442,7 @@ export function deserializeWithdrawalRequest(withdrawalRequest: WithdrawalReques }; } -export function serializeConsolidationRequest( - consolidationRequest: electra.ConsolidationRequest -): ConsolidationRequestRpc { +function serializeConsolidationRequest(consolidationRequest: electra.ConsolidationRequest): ConsolidationRequestRpc { return { sourceAddress: bytesToData(consolidationRequest.sourceAddress), sourcePubkey: bytesToData(consolidationRequest.sourcePubkey), @@ -475,9 +450,7 @@ export function serializeConsolidationRequest( }; } -export function deserializeConsolidationRequest( - consolidationRequest: ConsolidationRequestRpc -): electra.ConsolidationRequest { +function deserializeConsolidationRequest(consolidationRequest: ConsolidationRequestRpc): electra.ConsolidationRequest { return { sourceAddress: dataToBytes(consolidationRequest.sourceAddress, 20), sourcePubkey: dataToBytes(consolidationRequest.sourcePubkey, 48), @@ -485,16 +458,29 @@ export function deserializeConsolidationRequest( }; } +export function serializeExecutionRequests(executionRequests: ExecutionRequests): ExecutionRequestsRpc { + const {deposits, withdrawals, consolidations} = executionRequests; + return { + deposits: deposits.map(serializeDepositRequest), + withdrawals: withdrawals.map(serializeWithdrawalRequest), + consolidations: consolidations.map(serializeConsolidationRequest), + }; +} + +export function deserializeExecutionRequests(executionRequests: ExecutionRequestsRpc): ExecutionRequests { + const {deposits, withdrawals, consolidations} = executionRequests; + return { + deposits: deposits.map(deserializeDepositRequest), + withdrawals: withdrawals.map(deserializeWithdrawalRequest), + consolidations: consolidations.map(deserializeConsolidationRequest), + }; +} + export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | null): ExecutionPayloadBody | null { return data ? { transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, - depositRequests: data.depositRequests ? data.depositRequests.map(deserializeDepositRequest) : null, - withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(deserializeWithdrawalRequest) : null, - consolidationRequests: data.consolidationRequests - ? data.consolidationRequests.map(deserializeConsolidationRequest) - : null, } : null; } @@ -504,11 +490,6 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) ? { transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, - depositRequests: data.depositRequests ? data.depositRequests.map(serializeDepositRequest) : null, - withdrawalRequests: data.withdrawalRequests ? data.withdrawalRequests.map(serializeWithdrawalRequest) : null, - consolidationRequests: data.consolidationRequests - ? data.consolidationRequests.map(serializeConsolidationRequest) - : null, } : null; } diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts index d0c00e75fd59..2d08428df558 100644 --- a/packages/beacon-node/test/sim/electra-interop.test.ts +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -171,7 +171,6 @@ describe("executionEngine / ExecutionEngineHttp", function () { blockHash: dataToBytes(newPayloadBlockHash, 32), receiptsRoot: dataToBytes("0x0b67bea29f17eeb290685e01e9a2e4cd77a83471d9985a8ce27997a7ed3ee3f8", 32), blobGasUsed: 0n, - withdrawalRequests: [], }; const parentBeaconBlockRoot = dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32); const payloadResult = await executionEngine.notifyNewPayload( @@ -208,6 +207,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { await sleep(1000); const payloadAndBlockValue = await executionEngine.getPayload(ForkName.electra, payloadId2); const payload = payloadAndBlockValue.executionPayload as electra.ExecutionPayload; + const depositRequests = payloadAndBlockValue.executionRequests?.deposits; if (payload.transactions.length !== 1) { throw Error(`Number of transactions mismatched. Expected: 1, actual: ${payload.transactions.length}`); @@ -219,11 +219,11 @@ describe("executionEngine / ExecutionEngineHttp", function () { } } - if (payload.depositRequests.length !== 1) { - throw Error(`Number of depositRequests mismatched. Expected: 1, actual: ${payload.depositRequests.length}`); + if (depositRequests === undefined || depositRequests.length !== 1) { + throw Error(`Number of depositRequests mismatched. Expected: 1, actual: ${depositRequests?.length}`); } - const actualDepositRequest = payload.depositRequests[0]; + const actualDepositRequest = depositRequests[0]; assert.deepStrictEqual( actualDepositRequest, depositRequestB, diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts index 7229a0236d84..d4bc192f6cad 100644 --- a/packages/beacon-node/test/spec/specTestVersioning.ts +++ b/packages/beacon-node/test/spec/specTestVersioning.ts @@ -15,7 +15,7 @@ import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); export const ethereumConsensusSpecsTests: DownloadTestsOptions = { - specVersion: "v1.5.0-alpha.5", + specVersion: "v1.5.0-alpha.6", // Target directory is the host package root: 'packages/*/spec-tests' outputDir: path.join(__dirname, "../../spec-tests"), specTestsRepoUrl: "https://github.com/ethereum/consensus-spec-tests", diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index d8b4f9c0574c..bd5142627e3f 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -66,6 +66,8 @@ export const defaultSkipOpts: SkipOpts = { /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^electra\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, + // TODO Electra: slashings tests to be enabled in PR#7071 + /^electra\/epoch_processing\/slashings.*/, ], skippedTests: [], skippedRunners: ["merkle_proof", "networking"], diff --git a/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts b/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts index abb520bddafb..6da728be46e9 100644 --- a/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts +++ b/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts @@ -28,7 +28,7 @@ describe("UpgradeLightClientHeader", function () { capella: ssz.capella.LightClientHeader.defaultValue(), bellatrix: ssz.altair.LightClientHeader.defaultValue(), deneb: ssz.deneb.LightClientHeader.defaultValue(), - electra: ssz.electra.LightClientHeader.defaultValue(), + electra: ssz.deneb.LightClientHeader.defaultValue(), }; testSlots = { diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index 5ac4fd4ca670..c9f4ae671e53 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -193,9 +193,6 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], - depositRequests: null, // depositRequests is null pre-electra - withdrawalRequests: null, - consolidationRequests: null, }, null, // null returned for missing blocks { @@ -204,9 +201,6 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella - depositRequests: null, // depositRequests is null pre-electra - withdrawalRequests: null, - consolidationRequests: null, }, ], }; @@ -254,9 +248,6 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], - depositRequests: null, // depositRequests is null pre-electra - withdrawalRequests: null, - consolidationRequests: null, }, null, // null returned for missing blocks { @@ -265,9 +256,6 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella - depositRequests: null, // depositRequests is null pre-electra - withdrawalRequests: null, - consolidationRequests: null, }, ], }; diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 872aa5d9f910..408412464606 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -128,12 +128,7 @@ export function upgradeLightClientHeader( // eslint-disable-next-line no-fallthrough case ForkName.electra: - (upgradedHeader as LightClientHeader).execution.depositRequestsRoot = - ssz.electra.LightClientHeader.fields.execution.fields.depositRequestsRoot.defaultValue(); - (upgradedHeader as LightClientHeader).execution.withdrawalRequestsRoot = - ssz.electra.LightClientHeader.fields.execution.fields.withdrawalRequestsRoot.defaultValue(); - (upgradedHeader as LightClientHeader).execution.consolidationRequestsRoot = - ssz.electra.LightClientHeader.fields.execution.fields.consolidationRequestsRoot.defaultValue(); + // No changes to LightClientHeader in Electra // Break if no further upgrades is required else fall through if (ForkSeq[targetFork] <= ForkSeq.electra) break; @@ -170,16 +165,6 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC } } - if (epoch < config.ELECTRA_FORK_EPOCH) { - if ( - (header as LightClientHeader).execution.depositRequestsRoot !== undefined || - (header as LightClientHeader).execution.withdrawalRequestsRoot !== undefined || - (header as LightClientHeader).execution.consolidationRequestsRoot !== undefined - ) { - return false; - } - } - return isValidMerkleBranch( config .getExecutionForkTypes(header.beacon.slot) diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 6f61e7c242fb..bb52af14ba32 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -67,15 +67,15 @@ export function processOperations( const stateElectra = state as CachedBeaconStateElectra; const bodyElectra = body as electra.BeaconBlockBody; - for (const depositRequest of bodyElectra.executionPayload.depositRequests) { + for (const depositRequest of bodyElectra.executionRequests.deposits) { processDepositRequest(fork, stateElectra, depositRequest); } - for (const elWithdrawalRequest of bodyElectra.executionPayload.withdrawalRequests) { + for (const elWithdrawalRequest of bodyElectra.executionRequests.withdrawals) { processWithdrawalRequest(fork, stateElectra, elWithdrawalRequest); } - for (const elConsolidationRequest of bodyElectra.executionPayload.consolidationRequests) { + for (const elConsolidationRequest of bodyElectra.executionRequests.consolidations) { processConsolidationRequest(stateElectra, elConsolidationRequest); } } diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts index 6600ad98e80a..0bd36a909b46 100644 --- a/packages/state-transition/src/slot/upgradeStateToElectra.ts +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -48,17 +48,11 @@ export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): Cache stateElectraView.inactivityScores = stateElectraCloned.inactivityScores; stateElectraView.currentSyncCommittee = stateElectraCloned.currentSyncCommittee; stateElectraView.nextSyncCommittee = stateElectraCloned.nextSyncCommittee; - stateElectraView.latestExecutionPayloadHeader = ssz.electra.BeaconState.fields.latestExecutionPayloadHeader.toViewDU({ - ...stateElectraCloned.latestExecutionPayloadHeader.toValue(), - depositRequestsRoot: ssz.Root.defaultValue(), - withdrawalRequestsRoot: ssz.Root.defaultValue(), - consolidationRequestsRoot: ssz.Root.defaultValue(), - }); + stateElectraView.latestExecutionPayloadHeader = stateElectraCloned.latestExecutionPayloadHeader; stateElectraView.nextWithdrawalIndex = stateDeneb.nextWithdrawalIndex; stateElectraView.nextWithdrawalValidatorIndex = stateDeneb.nextWithdrawalValidatorIndex; stateElectraView.historicalSummaries = stateElectraCloned.historicalSummaries; - // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default // default value of depositRequestsStartIndex is UNSET_DEPOSIT_REQUESTS_START_INDEX stateElectraView.depositRequestsStartIndex = UNSET_DEPOSIT_REQUESTS_START_INDEX; stateElectraView.depositBalanceToConsume = BigInt(0); @@ -137,7 +131,6 @@ export function upgradeStateToElectraOriginal(stateDeneb: CachedBeaconStateDeneb epoch: stateDeneb.epochCtx.epoch, }); - // latestExecutionPayloadHeader's depositRequestsRoot and withdrawalRequestsRoot set to zeros by default // default value of depositRequestsStartIndex is UNSET_DEPOSIT_REQUESTS_START_INDEX stateElectra.depositRequestsStartIndex = UNSET_DEPOSIT_REQUESTS_START_INDEX; diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 06e654f9f1d2..b9243ebe7874 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -2,7 +2,6 @@ import { bellatrix, capella, deneb, - electra, isBlindedBeaconBlockBody, ssz, BeaconBlock, @@ -171,14 +170,7 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio ).excessBlobGas; } - if (fork >= ForkSeq.electra) { - (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositRequestsRoot = - ssz.electra.DepositRequests.hashTreeRoot((payload as electra.ExecutionPayload).depositRequests); - (bellatrixPayloadFields as electra.ExecutionPayloadHeader).withdrawalRequestsRoot = - ssz.electra.WithdrawalRequests.hashTreeRoot((payload as electra.ExecutionPayload).withdrawalRequests); - (bellatrixPayloadFields as electra.ExecutionPayloadHeader).consolidationRequestsRoot = - ssz.electra.ConsolidationRequests.hashTreeRoot((payload as electra.ExecutionPayload).consolidationRequests); - } + // No change in Electra return bellatrixPayloadFields; } diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 522f7245cc1b..9d995c38efd5 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -8,7 +8,6 @@ import { } from "@chainsafe/ssz"; import { HISTORICAL_ROOTS_LIMIT, - BLOCK_BODY_EXECUTION_PAYLOAD_DEPTH as EXECUTION_PAYLOAD_DEPTH, EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD, @@ -149,25 +148,18 @@ export const ConsolidationRequests = new ListCompositeType( MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD ); -export const ExecutionPayload = new ContainerType( +export const ExecutionRequests = new ContainerType( { - ...denebSsz.ExecutionPayload.fields, - depositRequests: DepositRequests, // New in ELECTRA - withdrawalRequests: WithdrawalRequests, // New in ELECTRA - consolidationRequests: ConsolidationRequests, // New in ELECTRA + deposits: DepositRequests, + withdrawals: WithdrawalRequests, + consolidations: ConsolidationRequests, }, - {typeName: "ExecutionPayload", jsonCase: "eth2"} + {typeName: "ExecutionRequests", jsonCase: "eth2"} ); -export const ExecutionPayloadHeader = new ContainerType( - { - ...denebSsz.ExecutionPayloadHeader.fields, - depositRequestsRoot: Root, // New in ELECTRA - withdrawalRequestsRoot: Root, // New in ELECTRA - consolidationRequestsRoot: Root, // New in ELECTRA - }, - {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} -); +// Explicitly defining electra containers for consistency's sake +export const ExecutionPayloadHeader = denebSsz.ExecutionPayloadHeader; +export const ExecutionPayload = denebSsz.ExecutionPayload; // We have to preserve Fields ordering while changing the type of ExecutionPayload export const BeaconBlockBody = new ContainerType( @@ -181,9 +173,10 @@ export const BeaconBlockBody = new ContainerType( deposits: phase0Ssz.BeaconBlockBody.fields.deposits, voluntaryExits: phase0Ssz.BeaconBlockBody.fields.voluntaryExits, syncAggregate: altairSsz.BeaconBlockBody.fields.syncAggregate, - executionPayload: ExecutionPayload, // Modified in ELECTRA + executionPayload: ExecutionPayload, blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, + executionRequests: ExecutionRequests, // New in ELECTRA:EIP7251 }, {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -215,7 +208,7 @@ export const BlindedBeaconBlockBody = new ContainerType( deposits: phase0Ssz.BeaconBlockBody.fields.deposits, voluntaryExits: phase0Ssz.BeaconBlockBody.fields.voluntaryExits, syncAggregate: altairSsz.SyncAggregate, - executionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + executionPayloadHeader: ExecutionPayloadHeader, blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, }, @@ -256,14 +249,6 @@ export const SignedBuilderBid = new ContainerType( {typeName: "SignedBuilderBid", jsonCase: "eth2"} ); -export const ExecutionPayloadAndBlobsBundle = new ContainerType( - { - executionPayload: ExecutionPayload, // Modified in ELECTRA - blobsBundle: denebSsz.BlobsBundle, - }, - {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} -); - export const PendingBalanceDeposit = new ContainerType( { index: ValidatorIndex, @@ -328,7 +313,7 @@ export const BeaconState = new ContainerType( currentSyncCommittee: altairSsz.SyncCommittee, nextSyncCommittee: altairSsz.SyncCommittee, // Execution - latestExecutionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + latestExecutionPayloadHeader: ExecutionPayloadHeader, // Withdrawals nextWithdrawalIndex: capellaSsz.BeaconState.fields.nextWithdrawalIndex, nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, @@ -347,30 +332,21 @@ export const BeaconState = new ContainerType( {typeName: "BeaconState", jsonCase: "eth2"} ); -export const LightClientHeader = new ContainerType( - { - beacon: phase0Ssz.BeaconBlockHeader, - execution: ExecutionPayloadHeader, // Modified in ELECTRA - executionBranch: new VectorCompositeType(Bytes32, EXECUTION_PAYLOAD_DEPTH), - }, - {typeName: "LightClientHeader", jsonCase: "eth2"} -); - export const LightClientBootstrap = new ContainerType( { - header: LightClientHeader, // Modified in ELECTRA + header: denebSsz.LightClientHeader, currentSyncCommittee: altairSsz.SyncCommittee, - currentSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), // Modified in ELECTRA + currentSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), }, {typeName: "LightClientBootstrap", jsonCase: "eth2"} ); export const LightClientUpdate = new ContainerType( { - attestedHeader: LightClientHeader, // Modified in ELECTRA + attestedHeader: denebSsz.LightClientHeader, nextSyncCommittee: altairSsz.SyncCommittee, nextSyncCommitteeBranch: new VectorCompositeType(Bytes32, NEXT_SYNC_COMMITTEE_DEPTH_ELECTRA), // Modified in ELECTRA - finalizedHeader: LightClientHeader, // Modified in ELECTRA + finalizedHeader: denebSsz.LightClientHeader, finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), // Modified in ELECTRA syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, @@ -380,8 +356,8 @@ export const LightClientUpdate = new ContainerType( export const LightClientFinalityUpdate = new ContainerType( { - attestedHeader: LightClientHeader, - finalizedHeader: LightClientHeader, + attestedHeader: denebSsz.LightClientHeader, + finalizedHeader: denebSsz.LightClientHeader, finalityBranch: new VectorCompositeType(Bytes32, FINALIZED_ROOT_DEPTH_ELECTRA), // Modified in ELECTRA syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, @@ -391,7 +367,7 @@ export const LightClientFinalityUpdate = new ContainerType( export const LightClientOptimisticUpdate = new ContainerType( { - attestedHeader: LightClientHeader, // Modified in ELECTRA + attestedHeader: denebSsz.LightClientHeader, syncAggregate: altairSsz.SyncAggregate, signatureSlot: Slot, }, @@ -400,8 +376,8 @@ export const LightClientOptimisticUpdate = new ContainerType( export const LightClientStore = new ContainerType( { - snapshot: LightClientBootstrap, // Modified in ELECTRA - validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), // Modified in ELECTRA + snapshot: LightClientBootstrap, + validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), }, {typeName: "LightClientStore", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 9a81aec43b53..f7996cf336f9 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -20,8 +20,7 @@ export type ConsolidationRequests = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; - -export type ExecutionPayloadAndBlobsBundle = ValueOf; +export type ExecutionRequests = ValueOf; export type BeaconBlockBody = ValueOf; export type BeaconBlock = ValueOf; @@ -33,13 +32,10 @@ export type BlindedBeaconBlockBody = ValueOf; export type BlindedBeaconBlock = ValueOf; export type SignedBlindedBeaconBlock = ValueOf; -export type FullOrBlindedExecutionPayload = ExecutionPayload | ExecutionPayloadHeader; - export type BuilderBid = ValueOf; export type SignedBuilderBid = ValueOf; export type SSEPayloadAttributes = ValueOf; -export type LightClientHeader = ValueOf; export type LightClientBootstrap = ValueOf; export type LightClientUpdate = ValueOf; export type LightClientFinalityUpdate = ValueOf; diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index 1071eed79a10..08fc06ac6cb9 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -1,4 +1,12 @@ -import {ForkAll, ForkBlobs, ForkExecution, ForkLightClient, ForkName, ForkPreBlobs} from "@lodestar/params"; +import { + ForkAll, + ForkBlobs, + ForkExecution, + ForkLightClient, + ForkName, + ForkPostElectra, + ForkPreBlobs, +} from "@lodestar/params"; import {ts as phase0} from "./phase0/index.js"; import {ts as altair} from "./altair/index.js"; import {ts as bellatrix} from "./bellatrix/index.js"; @@ -172,7 +180,7 @@ type TypesByFork = { BeaconState: electra.BeaconState; SignedBeaconBlock: electra.SignedBeaconBlock; Metadata: altair.Metadata; - LightClientHeader: electra.LightClientHeader; + LightClientHeader: deneb.LightClientHeader; LightClientBootstrap: electra.LightClientBootstrap; LightClientUpdate: electra.LightClientUpdate; LightClientFinalityUpdate: electra.LightClientFinalityUpdate; @@ -181,8 +189,8 @@ type TypesByFork = { BlindedBeaconBlock: electra.BlindedBeaconBlock; BlindedBeaconBlockBody: electra.BlindedBeaconBlockBody; SignedBlindedBeaconBlock: electra.SignedBlindedBeaconBlock; - ExecutionPayload: electra.ExecutionPayload; - ExecutionPayloadHeader: electra.ExecutionPayloadHeader; + ExecutionPayload: deneb.ExecutionPayload; + ExecutionPayloadHeader: deneb.ExecutionPayloadHeader; BuilderBid: electra.BuilderBid; SignedBuilderBid: electra.SignedBuilderBid; SSEPayloadAttributes: electra.SSEPayloadAttributes; @@ -199,6 +207,7 @@ type TypesByFork = { AttesterSlashing: electra.AttesterSlashing; AggregateAndProof: electra.AggregateAndProof; SignedAggregateAndProof: electra.SignedAggregateAndProof; + ExecutionRequests: electra.ExecutionRequests; }; }; @@ -233,6 +242,7 @@ export type SignedBeaconBlockOrContents = TypesByFork[F]["ExecutionPayload"]; export type ExecutionPayloadHeader = TypesByFork[F]["ExecutionPayloadHeader"]; +export type ExecutionRequests = TypesByFork[F]["ExecutionRequests"]; export type BlobsBundle = TypesByFork[F]["BlobsBundle"]; export type Contents = TypesByFork[F]["Contents"]; diff --git a/packages/types/src/utils/typeguards.ts b/packages/types/src/utils/typeguards.ts index a5f5b7808ae5..a892c3a0c9c0 100644 --- a/packages/types/src/utils/typeguards.ts +++ b/packages/types/src/utils/typeguards.ts @@ -1,4 +1,4 @@ -import {ForkBlobs, ForkExecution, ForkPostElectra} from "@lodestar/params"; +import {FINALIZED_ROOT_DEPTH_ELECTRA, ForkBlobs, ForkExecution, ForkPostElectra} from "@lodestar/params"; import { BlockContents, SignedBeaconBlock, @@ -76,7 +76,7 @@ export function isElectraAttestation(attestation: Attestation): attestation is A export function isElectraLightClientUpdate(update: LightClientUpdate): update is LightClientUpdate { const updatePostElectra = update as LightClientUpdate; return ( - updatePostElectra.attestedHeader.execution !== undefined && - updatePostElectra.attestedHeader.execution.depositRequestsRoot !== undefined + updatePostElectra.finalityBranch !== undefined && + updatePostElectra.finalityBranch.length === FINALIZED_ROOT_DEPTH_ELECTRA ); } From f1a77eadb7b00218bd8b4eecbbf6ca638ffe3552 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 25 Sep 2024 13:19:34 +0100 Subject: [PATCH 126/145] feat: set proper user-agent in validator client http requests (#7106) --- packages/cli/src/cmds/validator/handler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index 01c15126b796..53250c2c9a86 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -157,6 +157,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr globalInit: { requestWireFormat: parseWireFormat(args, "http.requestWireFormat"), responseWireFormat: parseWireFormat(args, "http.responseWireFormat"), + headers: {"User-Agent": `Lodestar/${version}`}, }, }, logger, From bf607863e11dd96a255e1ab61792d764a6d0dd2a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 25 Sep 2024 21:39:48 +0100 Subject: [PATCH 127/145] chore: remove signing domain for consolidations (#7092) --- packages/beacon-node/src/api/impl/config/constants.ts | 2 -- packages/params/src/index.ts | 1 - 2 files changed, 3 deletions(-) diff --git a/packages/beacon-node/src/api/impl/config/constants.ts b/packages/beacon-node/src/api/impl/config/constants.ts index 4b239ee4cac6..14ecade15b2e 100644 --- a/packages/beacon-node/src/api/impl/config/constants.ts +++ b/packages/beacon-node/src/api/impl/config/constants.ts @@ -37,7 +37,6 @@ import { BLOB_TX_TYPE, VERSIONED_HASH_VERSION_KZG, COMPOUNDING_WITHDRAWAL_PREFIX, - DOMAIN_CONSOLIDATION, UNSET_DEPOSIT_REQUESTS_START_INDEX, FULL_EXIT_REQUEST_AMOUNT, } from "@lodestar/params"; @@ -71,7 +70,6 @@ export const specConstants = { DOMAIN_SELECTION_PROOF, DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_APPLICATION_BUILDER, - DOMAIN_CONSOLIDATION, // phase0/validator.md TARGET_AGGREGATORS_PER_COMMITTEE, diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index dd5c24ac3a54..aa6e97641526 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -146,7 +146,6 @@ export const DOMAIN_SYNC_COMMITTEE = Uint8Array.from([7, 0, 0, 0]); export const DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF = Uint8Array.from([8, 0, 0, 0]); export const DOMAIN_CONTRIBUTION_AND_PROOF = Uint8Array.from([9, 0, 0, 0]); export const DOMAIN_BLS_TO_EXECUTION_CHANGE = Uint8Array.from([10, 0, 0, 0]); -export const DOMAIN_CONSOLIDATION = Uint8Array.from([11, 0, 0, 0]); // Application specific domains From bb8204606762282036aebb15c216617a4c7df41f Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 26 Sep 2024 21:01:37 +0100 Subject: [PATCH 128/145] feat: add electra support for remote signer (#7100) --- .../validator/src/services/validatorStore.ts | 10 +++--- .../src/util/externalSignerClient.ts | 33 +++++++++++++++--- .../validator/test/e2e/web3signer.test.ts | 34 ++++++++++++------- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 6da4f597d87c..ce507349689f 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -562,13 +562,13 @@ export class ValidatorStore { const signingSlot = aggregate.data.slot; const domain = this.config.getDomain(signingSlot, DOMAIN_AGGREGATE_AND_PROOF); - const signingRoot = - this.config.getForkSeq(duty.slot) >= ForkSeq.electra - ? computeSigningRoot(ssz.electra.AggregateAndProof, aggregateAndProof, domain) - : computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof, domain); + const isPostElectra = this.config.getForkSeq(duty.slot) >= ForkSeq.electra; + const signingRoot = isPostElectra + ? computeSigningRoot(ssz.electra.AggregateAndProof, aggregateAndProof, domain) + : computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof, domain); const signableMessage: SignableMessage = { - type: SignableMessageType.AGGREGATE_AND_PROOF, + type: isPostElectra ? SignableMessageType.AGGREGATE_AND_PROOF_V2 : SignableMessageType.AGGREGATE_AND_PROOF, data: aggregateAndProof, }; diff --git a/packages/validator/src/util/externalSignerClient.ts b/packages/validator/src/util/externalSignerClient.ts index 4fd615ce5280..1d9778375b88 100644 --- a/packages/validator/src/util/externalSignerClient.ts +++ b/packages/validator/src/util/externalSignerClient.ts @@ -1,11 +1,23 @@ import {ContainerType, ValueOf} from "@chainsafe/ssz"; import {fetch} from "@lodestar/api"; -import {phase0, altair, capella, BeaconBlock, BlindedBeaconBlock} from "@lodestar/types"; -import {ForkSeq} from "@lodestar/params"; +import { + phase0, + altair, + capella, + BeaconBlock, + BlindedBeaconBlock, + AggregateAndProof, + sszTypesFor, + ssz, + Slot, + Epoch, + RootHex, + Root, +} from "@lodestar/types"; +import {ForkPreExecution, ForkSeq} from "@lodestar/params"; import {ValidatorRegistrationV1} from "@lodestar/types/bellatrix"; import {BeaconConfig} from "@lodestar/config"; import {computeEpochAtSlot, blindedOrFullBlockToHeader} from "@lodestar/state-transition"; -import {Epoch, Root, RootHex, Slot, ssz} from "@lodestar/types"; import {toHex, toRootHex} from "@lodestar/utils"; import {PubkeyHex} from "../types.js"; @@ -14,6 +26,7 @@ import {PubkeyHex} from "../types.js"; export enum SignableMessageType { AGGREGATION_SLOT = "AGGREGATION_SLOT", AGGREGATE_AND_PROOF = "AGGREGATE_AND_PROOF", + AGGREGATE_AND_PROOF_V2 = "AGGREGATE_AND_PROOF_V2", ATTESTATION = "ATTESTATION", BLOCK_V2 = "BLOCK_V2", DEPOSIT = "DEPOSIT", @@ -63,8 +76,9 @@ const SyncAggregatorSelectionDataType = new ContainerType( export type SignableMessage = | {type: SignableMessageType.AGGREGATION_SLOT; data: {slot: Slot}} | {type: SignableMessageType.AGGREGATE_AND_PROOF; data: phase0.AggregateAndProof} + | {type: SignableMessageType.AGGREGATE_AND_PROOF_V2; data: AggregateAndProof} | {type: SignableMessageType.ATTESTATION; data: phase0.AttestationData} - | {type: SignableMessageType.BLOCK_V2; data: BeaconBlock | BlindedBeaconBlock} + | {type: SignableMessageType.BLOCK_V2; data: BeaconBlock | BlindedBeaconBlock} | {type: SignableMessageType.DEPOSIT; data: ValueOf} | {type: SignableMessageType.RANDAO_REVEAL; data: {epoch: Epoch}} | {type: SignableMessageType.VOLUNTARY_EXIT; data: phase0.VoluntaryExit} @@ -77,6 +91,7 @@ export type SignableMessage = const requiresForkInfo: Record = { [SignableMessageType.AGGREGATION_SLOT]: true, [SignableMessageType.AGGREGATE_AND_PROOF]: true, + [SignableMessageType.AGGREGATE_AND_PROOF_V2]: true, [SignableMessageType.ATTESTATION]: true, [SignableMessageType.BLOCK_V2]: true, [SignableMessageType.DEPOSIT]: false, @@ -203,6 +218,16 @@ function serializerSignableMessagePayload(config: BeaconConfig, payload: Signabl case SignableMessageType.AGGREGATE_AND_PROOF: return {aggregate_and_proof: ssz.phase0.AggregateAndProof.toJson(payload.data)}; + case SignableMessageType.AGGREGATE_AND_PROOF_V2: { + const fork = config.getForkName(payload.data.aggregate.data.slot); + return { + aggregate_and_proof: { + version: fork.toUpperCase(), + data: sszTypesFor(fork).AggregateAndProof.toJson(payload.data), + }, + }; + } + case SignableMessageType.ATTESTATION: return {attestation: ssz.phase0.AttestationData.toJson(payload.data)}; diff --git a/packages/validator/test/e2e/web3signer.test.ts b/packages/validator/test/e2e/web3signer.test.ts index 326dbae8c4f7..ae22bc31446b 100644 --- a/packages/validator/test/e2e/web3signer.test.ts +++ b/packages/validator/test/e2e/web3signer.test.ts @@ -5,7 +5,7 @@ import {computeStartSlotAtEpoch, interopSecretKey, interopSecretKeys} from "@lod import {createBeaconConfig} from "@lodestar/config"; import {genesisData} from "@lodestar/config/networks"; import {getClient, routes} from "@lodestar/api"; -import {ssz} from "@lodestar/types"; +import {ssz, sszTypesFor} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {getKeystoresStr, StartedExternalSigner, startExternalSigner} from "@lodestar/test-utils"; import {Interchange, ISlashingProtection, Signer, SignerType, ValidatorStore} from "../../src/index.js"; @@ -93,17 +93,27 @@ describe("web3signer signature test", function () { await assertSameSignature("signAttestation", duty, attestationData, epoch); }); - it("signAggregateAndProof", async () => { - const aggregateAndProof = ssz.phase0.AggregateAndProof.defaultValue(); - aggregateAndProof.aggregate.data.slot = duty.slot; - aggregateAndProof.aggregate.data.index = duty.committeeIndex; - await assertSameSignature( - "signAggregateAndProof", - duty, - aggregateAndProof.selectionProof, - aggregateAndProof.aggregate - ); - }); + for (const fork of config.forksAscendingEpochOrder) { + it(`signAggregateAndProof ${fork.name}`, async ({skip}) => { + // Only test till the fork the signer version supports + if (ForkSeq[fork.name] > externalSigner.supportedForkSeq) { + skip(); + return; + } + + const aggregateAndProof = sszTypesFor(fork.name).AggregateAndProof.defaultValue(); + const slot = computeStartSlotAtEpoch(fork.epoch); + aggregateAndProof.aggregate.data.slot = slot; + aggregateAndProof.aggregate.data.index = duty.committeeIndex; + + await assertSameSignature( + "signAggregateAndProof", + {...duty, slot}, + aggregateAndProof.selectionProof, + aggregateAndProof.aggregate + ); + }); + } it("signSyncCommitteeSignature", async () => { const beaconBlockRoot = ssz.phase0.BeaconBlockHeader.defaultValue().bodyRoot; From 58dea7543c846e912a0d7422aee0b836f57a3b7a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 26 Sep 2024 21:59:20 +0100 Subject: [PATCH 129/145] feat: use POST variant to get validators from state (#6897) * feat: use POST variant to get validators from state * Fix api handler used during testing * Update more calls to use POST variant --- .../test/e2e/api/impl/lightclient/endpoint.test.ts | 2 +- .../cli/src/cmds/validator/blsToExecutionChange.ts | 2 +- packages/cli/src/cmds/validator/voluntaryExit.ts | 2 +- packages/cli/test/sim/endpoints.test.ts | 10 +++++----- packages/flare/src/cmds/selfSlashAttester.ts | 4 ++-- packages/flare/src/cmds/selfSlashProposer.ts | 4 ++-- packages/validator/src/services/indices.ts | 12 +++++++----- packages/validator/src/validator.ts | 4 +++- .../validator/test/unit/services/attestation.test.ts | 2 +- .../test/unit/services/attestationDuties.test.ts | 2 +- .../test/unit/services/syncCommitteDuties.test.ts | 2 +- .../test/unit/services/syncCommittee.test.ts | 2 +- packages/validator/test/utils/apiStub.ts | 1 + 13 files changed, 27 insertions(+), 22 deletions(-) diff --git a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts index 81154005af68..c746746741cf 100644 --- a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts @@ -122,7 +122,7 @@ describe("lightclient api", function () { const committeeRes = await lightclient.getLightClientCommitteeRoot({startPeriod: 0, count: 1}); committeeRes.assertOk(); const client = getClient({baseUrl: `http://127.0.0.1:${restPort}`}, {config}).beacon; - const validators = (await client.getStateValidators({stateId: "head"})).value(); + const validators = (await client.postStateValidators({stateId: "head"})).value(); const pubkeys = validators.map((v) => v.validator.pubkey); expect(pubkeys.length).toBe(validatorCount); // only 2 validators spreading to 512 committee slots diff --git a/packages/cli/src/cmds/validator/blsToExecutionChange.ts b/packages/cli/src/cmds/validator/blsToExecutionChange.ts index dc55647fda80..7fbdbb029bf5 100644 --- a/packages/cli/src/cmds/validator/blsToExecutionChange.ts +++ b/packages/cli/src/cmds/validator/blsToExecutionChange.ts @@ -60,7 +60,7 @@ like to choose for BLS To Execution Change.", const {genesisValidatorsRoot} = (await client.beacon.getGenesis()).value(); const config = createBeaconConfig(chainForkConfig, genesisValidatorsRoot); - const validators = (await client.beacon.getStateValidators({stateId: "head", validatorIds: [publicKey]})).value(); + const validators = (await client.beacon.postStateValidators({stateId: "head", validatorIds: [publicKey]})).value(); const validator = validators[0]; if (validator === undefined) { throw new Error(`Validator pubkey ${publicKey} not found in state`); diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index a399a763662d..02f1591b59ed 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -207,7 +207,7 @@ function selectSignersToExit(args: VoluntaryExitArgs, signers: Signer[]): Signer async function resolveValidatorIndexes(client: ApiClient, signersToExit: SignerPubkey[]) { const pubkeys = signersToExit.map(({pubkey}) => pubkey); - const validators = (await client.beacon.getStateValidators({stateId: "head", validatorIds: pubkeys})).value(); + const validators = (await client.beacon.postStateValidators({stateId: "head", validatorIds: pubkeys})).value(); const dataByPubkey = new Map(validators.map((item) => [toPubkeyHex(item.validator.pubkey), item])); diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts index 07cd5fec0cc4..a42b9568f9a3 100644 --- a/packages/cli/test/sim/endpoints.test.ts +++ b/packages/cli/test/sim/endpoints.test.ts @@ -40,7 +40,7 @@ await env.start({runTimeoutMs: estimatedTimeoutMs}); const node = env.nodes[0].beacon; await waitForSlot("Wait for 2 slots before checking endpoints", {env, slot: 2}); -const validators = (await node.api.beacon.getStateValidators({stateId: "head"})).value(); +const validators = (await node.api.beacon.postStateValidators({stateId: "head"})).value(); await env.tracker.assert("should have correct validators count called without filters", async () => { assert.equal(validators.length, validatorCount); @@ -55,12 +55,12 @@ await env.tracker.assert("should have correct validator index for second validat }); await env.tracker.assert( - "should return correct number of filtered validators when getStateValidators called with filters", + "should return correct number of filtered validators when postStateValidators called with filters", async () => { const filterPubKey = "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c"; - const res = await node.api.beacon.getStateValidators({stateId: "head", validatorIds: [filterPubKey]}); + const res = await node.api.beacon.postStateValidators({stateId: "head", validatorIds: [filterPubKey]}); assert.equal(res.value().length, 1); @@ -71,12 +71,12 @@ await env.tracker.assert( ); await env.tracker.assert( - "should return correct filtered validators when getStateValidators called with filters", + "should return correct filtered validators when postStateValidators called with filters", async () => { const filterPubKey = "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c"; - const res = await node.api.beacon.getStateValidators({stateId: "head", validatorIds: [filterPubKey]}); + const res = await node.api.beacon.postStateValidators({stateId: "head", validatorIds: [filterPubKey]}); assert.equal(toHexString(res.value()[0].validator.pubkey), filterPubKey); } diff --git a/packages/flare/src/cmds/selfSlashAttester.ts b/packages/flare/src/cmds/selfSlashAttester.ts index a37c6c765bd3..6ccc9d80a98f 100644 --- a/packages/flare/src/cmds/selfSlashAttester.ts +++ b/packages/flare/src/cmds/selfSlashAttester.ts @@ -79,7 +79,7 @@ export async function selfSlashAttesterHandler(args: SelfSlashArgs): Promise sk.toPublicKey().toHex()); - const validators = (await client.beacon.getStateValidators({stateId: "head", validatorIds: pksHex})).value(); + const validators = (await client.beacon.postStateValidators({stateId: "head", validatorIds: pksHex})).value(); // All validators in the batch will be part of the same AttesterSlashing const attestingIndices = validators.map((v) => v.index); @@ -92,7 +92,7 @@ export async function selfSlashAttesterHandler(args: SelfSlashArgs): Promise sk.toPublicKey().toHex()); - const validators = (await client.beacon.getStateValidators({stateId: "head", validatorIds: pksHex})).value(); + const validators = (await client.beacon.postStateValidators({stateId: "head", validatorIds: pksHex})).value(); // Submit all ProposerSlashing for range at once await Promise.all( @@ -88,7 +88,7 @@ export async function selfSlashProposerHandler(args: SelfSlashArgs): Promise { - const validators = (await this.api.beacon.getStateValidators({stateId: "head", validatorIds: pubkeysHex})).value(); + const validators = (await this.api.beacon.postStateValidators({stateId: "head", validatorIds: pubkeysHex})).value(); const newIndices = []; diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 980e64f7eac2..6ffebff2b3bd 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -372,7 +372,9 @@ export class Validator { * Create a signed voluntary exit message for the given validator by its key. */ async signVoluntaryExit(publicKey: string, exitEpoch?: number): Promise { - const validators = (await this.api.beacon.getStateValidators({stateId: "head", validatorIds: [publicKey]})).value(); + const validators = ( + await this.api.beacon.postStateValidators({stateId: "head", validatorIds: [publicKey]}) + ).value(); const validator = validators[0]; if (validator === undefined) { diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index 356503cbdd09..66b722273102 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -105,7 +105,7 @@ describe("AttestationService", function () { ]; // Return empty replies to duties service - api.beacon.getStateValidators.mockResolvedValue( + api.beacon.postStateValidators.mockResolvedValue( mockApiResponse({data: [], meta: {executionOptimistic: false, finalized: false}}) ); api.validator.getAttesterDuties.mockResolvedValue( diff --git a/packages/validator/test/unit/services/attestationDuties.test.ts b/packages/validator/test/unit/services/attestationDuties.test.ts index 751ebcf11205..a9d50eaf42c6 100644 --- a/packages/validator/test/unit/services/attestationDuties.test.ts +++ b/packages/validator/test/unit/services/attestationDuties.test.ts @@ -52,7 +52,7 @@ describe("AttestationDutiesService", function () { index, validator: {...defaultValidator.validator, pubkey: pubkeys[0]}, }; - api.beacon.getStateValidators.mockResolvedValue( + api.beacon.postStateValidators.mockResolvedValue( mockApiResponse({data: [validatorResponse], meta: {executionOptimistic: false, finalized: false}}) ); }); diff --git a/packages/validator/test/unit/services/syncCommitteDuties.test.ts b/packages/validator/test/unit/services/syncCommitteDuties.test.ts index 10bac3ab2fe9..dc43502d5b57 100644 --- a/packages/validator/test/unit/services/syncCommitteDuties.test.ts +++ b/packages/validator/test/unit/services/syncCommitteDuties.test.ts @@ -60,7 +60,7 @@ describe("SyncCommitteeDutiesService", function () { index: indices[i], validator: {...defaultValidator.validator, pubkey: pubkeys[i]}, })); - api.beacon.getStateValidators.mockResolvedValue( + api.beacon.postStateValidators.mockResolvedValue( mockApiResponse({data: validatorResponses, meta: {executionOptimistic: false, finalized: false}}) ); }); diff --git a/packages/validator/test/unit/services/syncCommittee.test.ts b/packages/validator/test/unit/services/syncCommittee.test.ts index 66e63ab72a6c..449f826c3806 100644 --- a/packages/validator/test/unit/services/syncCommittee.test.ts +++ b/packages/validator/test/unit/services/syncCommittee.test.ts @@ -102,7 +102,7 @@ describe("SyncCommitteeService", function () { ]; // Return empty replies to duties service - api.beacon.getStateValidators.mockResolvedValue( + api.beacon.postStateValidators.mockResolvedValue( mockApiResponse({data: [], meta: {executionOptimistic: false, finalized: false}}) ); api.validator.getSyncCommitteeDuties.mockResolvedValue( diff --git a/packages/validator/test/utils/apiStub.ts b/packages/validator/test/utils/apiStub.ts index ef615af03369..4443dab5c4ac 100644 --- a/packages/validator/test/utils/apiStub.ts +++ b/packages/validator/test/utils/apiStub.ts @@ -16,6 +16,7 @@ export function getApiClientStub(): ApiClientStub { return { beacon: { getStateValidators: vi.fn(), + postStateValidators: vi.fn(), publishBlindedBlockV2: vi.fn(), publishBlockV2: vi.fn(), submitPoolSyncCommitteeSignatures: vi.fn(), From 77006ea0ce9ac043b71b7aaa07b9fae6f38f5250 Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 27 Sep 2024 08:15:21 +0700 Subject: [PATCH 130/145] fix: state serialization time (#7109) * fix: release memory to BufferPool * fix: correct the serializeState metric * fix: check types * chore: fix lint --- .../src/chain/archiver/archiveStates.ts | 13 ++++-- .../beacon-node/src/chain/archiver/index.ts | 6 ++- packages/beacon-node/src/chain/chain.ts | 2 +- .../beacon-node/src/chain/serializeState.ts | 11 +++-- .../stateCache/persistentCheckpointsCache.ts | 45 +++++++++++-------- .../src/metrics/metrics/lodestar.ts | 11 ++--- .../stateCache/nHistoricalStates.test.ts | 7 ++- 7 files changed, 55 insertions(+), 40 deletions(-) diff --git a/packages/beacon-node/src/chain/archiver/archiveStates.ts b/packages/beacon-node/src/chain/archiver/archiveStates.ts index 53f2033d80e8..8fd9081ab243 100644 --- a/packages/beacon-node/src/chain/archiver/archiveStates.ts +++ b/packages/beacon-node/src/chain/archiver/archiveStates.ts @@ -8,6 +8,7 @@ import {IStateRegenerator} from "../regen/interface.js"; import {getStateSlotFromBytes} from "../../util/multifork.js"; import {serializeState} from "../serializeState.js"; import {AllocSource, BufferPool} from "../../util/bufferPool.js"; +import {Metrics} from "../../metrics/metrics.js"; /** * Minimum number of epochs between single temp archived states @@ -48,13 +49,13 @@ export class StatesArchiver { * epoch - 1024*2 epoch - 1024 epoch - 32 epoch * ``` */ - async maybeArchiveState(finalized: CheckpointWithHex): Promise { + async maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise { const lastStoredSlot = await this.db.stateArchive.lastKey(); const lastStoredEpoch = computeEpochAtSlot(lastStoredSlot ?? 0); const {archiveStateEpochFrequency} = this.opts; if (finalized.epoch - lastStoredEpoch >= Math.min(PERSIST_TEMP_STATE_EVERY_EPOCHS, archiveStateEpochFrequency)) { - await this.archiveState(finalized); + await this.archiveState(finalized, metrics); // Only check the current and previous intervals const minEpoch = Math.max( @@ -86,7 +87,7 @@ export class StatesArchiver { * Archives finalized states from active bucket to archive bucket. * Only the new finalized state is stored to disk */ - async archiveState(finalized: CheckpointWithHex): Promise { + async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise { // starting from Mar 2024, the finalized state could be from disk or in memory const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized); const {rootHex} = finalized; @@ -99,10 +100,14 @@ export class StatesArchiver { this.logger.verbose("Archived finalized state bytes", {epoch: finalized.epoch, slot, root: rootHex}); } else { // serialize state using BufferPool if provided + const timer = metrics?.stateSerializeDuration.startTimer({source: AllocSource.ARCHIVE_STATE}); await serializeState( finalizedStateOrBytes, AllocSource.ARCHIVE_STATE, - (stateBytes) => this.db.stateArchive.putBinary(finalizedStateOrBytes.slot, stateBytes), + (stateBytes) => { + timer?.(); + return this.db.stateArchive.putBinary(finalizedStateOrBytes.slot, stateBytes); + }, this.bufferPool ); // don't delete states before the finalized state, auto-prune will take care of it diff --git a/packages/beacon-node/src/chain/archiver/index.ts b/packages/beacon-node/src/chain/archiver/index.ts index 294c2281e19b..45169b2fa802 100644 --- a/packages/beacon-node/src/chain/archiver/index.ts +++ b/packages/beacon-node/src/chain/archiver/index.ts @@ -4,6 +4,7 @@ import {IBeaconDb} from "../../db/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; import {IBeaconChain} from "../interface.js"; import {ChainEvent} from "../emitter.js"; +import {Metrics} from "../../metrics/metrics.js"; import {StatesArchiver, StatesArchiverOpts} from "./archiveStates.js"; import {archiveBlocks} from "./archiveBlocks.js"; @@ -45,7 +46,8 @@ export class Archiver { private readonly chain: IBeaconChain, private readonly logger: Logger, signal: AbortSignal, - opts: ArchiverOpts + opts: ArchiverOpts, + private readonly metrics?: Metrics | null ) { this.archiveBlobEpochs = opts.archiveBlobEpochs; this.statesArchiver = new StatesArchiver(chain.regen, db, logger, opts, chain.bufferPool); @@ -105,7 +107,7 @@ export class Archiver { this.prevFinalized = finalized; // should be after ArchiveBlocksTask to handle restart cleanly - await this.statesArchiver.maybeArchiveState(finalized); + await this.statesArchiver.maybeArchiveState(finalized, this.metrics); this.chain.regen.pruneOnFinalized(finalizedEpoch); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 0b0ac30edf40..696f4bd51dc3 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -345,7 +345,7 @@ export class BeaconChain implements IBeaconChain { this.bls = bls; this.emitter = emitter; - this.archiver = new Archiver(db, this, logger, signal, opts); + this.archiver = new Archiver(db, this, logger, signal, opts, metrics); // always run PrepareNextSlotScheduler except for fork_choice spec tests if (!opts?.disablePrepareNextSlot) { new PrepareNextSlotScheduler(this, this.config, metrics, this.logger, signal); diff --git a/packages/beacon-node/src/chain/serializeState.ts b/packages/beacon-node/src/chain/serializeState.ts index cbb2ecd18cff..c6e796cd614c 100644 --- a/packages/beacon-node/src/chain/serializeState.ts +++ b/packages/beacon-node/src/chain/serializeState.ts @@ -15,19 +15,18 @@ export async function serializeState( const size = state.type.tree_serializedSize(state.node); let stateBytes: Uint8Array | null = null; if (bufferPool) { - const bufferWithKey = bufferPool.alloc(size, source); + using bufferWithKey = bufferPool.alloc(size, source); if (bufferWithKey) { stateBytes = bufferWithKey.buffer; const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength); state.serializeToBytes({uint8Array: stateBytes, dataView}, 0); + return processFn(stateBytes); } + // release the buffer back to the pool automatically } - if (!stateBytes) { - // we already have metrics in BufferPool so no need to do it here - stateBytes = state.serialize(); - } + // we already have metrics in BufferPool so no need to do it here + stateBytes = state.serialize(); return processFn(stateBytes); - // release the buffer back to the pool automatically } diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 74e85e6fa574..07c96f224bee 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -94,7 +94,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { private readonly cache: MapTracker; /** Epoch -> Set */ private readonly epochIndex = new MapDef>(() => new Set()); - private readonly metrics: Metrics["cpStateCache"] | null | undefined; + private readonly metrics: Metrics | null | undefined; private readonly logger: Logger; private readonly clock: IClock | null | undefined; private readonly signal: AbortSignal | undefined; @@ -123,7 +123,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { ) { this.cache = new MapTracker(metrics?.cpStateCache); if (metrics) { - this.metrics = metrics.cpStateCache; + this.metrics = metrics; metrics.cpStateCache.size.addCollect(() => { let persistCount = 0; let inMemoryCount = 0; @@ -194,24 +194,26 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const {persistedKey, stateBytes} = stateOrStateBytesData; const logMeta = {persistedKey: toHex(persistedKey)}; this.logger.debug("Reload: read state successful", logMeta); - this.metrics?.stateReloadSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0); + this.metrics?.cpStateCache.stateReloadSecFromSlot.observe( + this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 + ); const seedState = this.findSeedStateToReload(cp); - this.metrics?.stateReloadEpochDiff.observe(Math.abs(seedState.epochCtx.epoch - cp.epoch)); + this.metrics?.cpStateCache.stateReloadEpochDiff.observe(Math.abs(seedState.epochCtx.epoch - cp.epoch)); this.logger.debug("Reload: found seed state", {...logMeta, seedSlot: seedState.slot}); try { // 80% of validators serialization time comes from memory allocation, this is to avoid it - const sszTimer = this.metrics?.stateReloadValidatorsSerializeDuration.startTimer(); + const sszTimer = this.metrics?.cpStateCache.stateReloadValidatorsSerializeDuration.startTimer(); // automatically free the buffer pool after this scope using validatorsBytesWithKey = this.serializeStateValidators(seedState); let validatorsBytes = validatorsBytesWithKey?.buffer; if (validatorsBytes == null) { // fallback logic in case we can't use the buffer pool - this.metrics?.stateReloadValidatorsSerializeAllocCount.inc(); + this.metrics?.cpStateCache.stateReloadValidatorsSerializeAllocCount.inc(); validatorsBytes = seedState.validators.serialize(); } sszTimer?.(); - const timer = this.metrics?.stateReloadDuration.startTimer(); + const timer = this.metrics?.cpStateCache.stateReloadDuration.startTimer(); const newCachedState = loadCachedBeaconState(seedState, stateBytes, {}, validatorsBytes); newCachedState.commit(); const stateRoot = toRootHex(newCachedState.hashTreeRoot()); @@ -275,7 +277,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } const persistedKey = cacheItem.value; - const dbReadTimer = this.metrics?.stateReloadDbReadTime.startTimer(); + const dbReadTimer = this.metrics?.cpStateCache.stateReloadDbReadTime.startTimer(); const stateBytes = await this.datastore.read(persistedKey); dbReadTimer?.(); @@ -289,7 +291,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * Similar to get() api without reloading from disk */ get(cpOrKey: CheckpointHex | string, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { - this.metrics?.lookups.inc(); + this.metrics?.cpStateCache.lookups.inc(); const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey); const cacheItem = this.cache.get(cpKey); @@ -297,7 +299,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { return null; } - this.metrics?.hits.inc(); + this.metrics?.cpStateCache.hits.inc(); if (cpKey === this.preComputedCheckpoint) { this.preComputedCheckpointHits = (this.preComputedCheckpointHits ?? 0) + 1; @@ -305,7 +307,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { if (isInMemoryCacheItem(cacheItem)) { const {state} = cacheItem; - this.metrics?.stateClonedCount.observe(state.clonedCount); + this.metrics?.cpStateCache.stateClonedCount.observe(state.clonedCount); return state.clone(opts?.dontTransferCache); } @@ -319,7 +321,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const cpHex = toCheckpointHex(cp); const key = toCacheKey(cpHex); const cacheItem = this.cache.get(key); - this.metrics?.adds.inc(); + this.metrics?.cpStateCache.adds.inc(); if (cacheItem !== undefined && isPersistedCacheItem(cacheItem)) { const persistedKey = cacheItem.value; // was persisted to disk, set back to memory @@ -683,7 +685,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta); } else { // persist and do not update epochIndex - this.metrics?.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0); + this.metrics?.cpStateCache.statePersistSecFromSlot.observe( + this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0 + ); const cpPersist = {epoch: epoch, root: fromHex(rootHex)}; // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory. // As monitored on holesky as of Jan 2024: @@ -691,14 +695,19 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s) // - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s) // - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization - const timer = this.metrics?.stateSerializeDuration.startTimer(); + const timer = this.metrics?.stateSerializeDuration.startTimer({ + source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, + }); persistedKey = await serializeState( state, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, - (stateBytes) => this.datastore.write(cpPersist, stateBytes), + (stateBytes) => { + timer?.(); + return this.datastore.write(cpPersist, stateBytes); + }, this.bufferPool ); - timer?.(); + persistCount++; this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", { ...logMeta, @@ -718,7 +727,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.cache.delete(cpKey); this.epochIndex.get(epoch)?.delete(rootHex); } - this.metrics?.statePruneFromMemoryCount.inc(); + this.metrics?.cpStateCache.statePruneFromMemoryCount.inc(); this.logger.verbose("Pruned checkpoint state from memory", logMeta); } } @@ -742,7 +751,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { if (persistedKey) { await this.datastore.remove(persistedKey); persistCount++; - this.metrics?.persistedStateRemoveCount.inc(); + this.metrics?.cpStateCache.persistedStateRemoveCount.inc(); } } this.cache.delete(key); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index fb4ea61b6523..bac740b7ee04 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1218,11 +1218,6 @@ export function createLodestarMetrics( help: "Histogram of cloned count per state every time state.clone() is called", buckets: [1, 2, 5, 10, 50, 250], }), - stateSerializeDuration: register.histogram({ - name: "lodestar_cp_state_cache_state_serialize_seconds", - help: "Histogram of time to serialize state to db", - buckets: [0.1, 0.5, 1, 2, 3, 4], - }), numStatesUpdated: register.histogram({ name: "lodestar_cp_state_cache_state_updated_count", help: "Histogram of number of state cache items updated every time removing and adding pubkeys to pubkey cache", @@ -1434,6 +1429,12 @@ export function createLodestarMetrics( name: "lodestar_unhandled_promise_rejections_total", help: "UnhandledPromiseRejection total count", }), + stateSerializeDuration: register.histogram<{source: AllocSource}>({ + name: "lodestar_state_serialize_seconds", + help: "Histogram of time to serialize state", + labelNames: ["source"], + buckets: [0.1, 0.5, 1, 2, 3, 4], + }), // regen.getState metrics regenGetState: { diff --git a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts index 005b28baeefc..fd14b7b8b8c0 100644 --- a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts +++ b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts @@ -412,11 +412,10 @@ describe("regen/reload states with n-historical states configuration", function )?.value ).toEqual(reloadCount); - const stateSszMetricValues = await (followupBn.metrics?.cpStateCache.stateSerializeDuration as Histogram).get(); + const stateSszMetricValues = await (followupBn.metrics?.stateSerializeDuration as Histogram).get(); expect( - stateSszMetricValues?.values.find( - (value) => value.metricName === "lodestar_cp_state_cache_state_serialize_seconds_count" - )?.value + stateSszMetricValues?.values.find((value) => value.metricName === "lodestar_state_serialize_seconds_count") + ?.value ).toEqual(persistCount); // assert number of persisted/in-memory states From 94b6c5b0db4e60c7406309367f5b2d7253e8d3c5 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 27 Sep 2024 19:39:18 +0100 Subject: [PATCH 131/145] chore: use signing slot to determine fork (#7112) --- packages/validator/src/services/validatorStore.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index ce507349689f..c2b18bfd7e09 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -531,7 +531,7 @@ export class ValidatorStore { data: attestationData, }; - if (this.config.getForkSeq(duty.slot) >= ForkSeq.electra) { + if (this.config.getForkSeq(signingSlot) >= ForkSeq.electra) { return { aggregationBits: BitArray.fromSingleBit(duty.committeeLength, duty.validatorCommitteeIndex), data: attestationData, @@ -562,7 +562,7 @@ export class ValidatorStore { const signingSlot = aggregate.data.slot; const domain = this.config.getDomain(signingSlot, DOMAIN_AGGREGATE_AND_PROOF); - const isPostElectra = this.config.getForkSeq(duty.slot) >= ForkSeq.electra; + const isPostElectra = this.config.getForkSeq(signingSlot) >= ForkSeq.electra; const signingRoot = isPostElectra ? computeSigningRoot(ssz.electra.AggregateAndProof, aggregateAndProof, domain) : computeSigningRoot(ssz.phase0.AggregateAndProof, aggregateAndProof, domain); @@ -802,7 +802,7 @@ export class ValidatorStore { throw Error(`Inconsistent duties during signing: duty.slot ${duty.slot} != att.slot ${data.slot}`); } - const isPostElectra = this.config.getForkSeq(duty.slot) >= ForkSeq.electra; + const isPostElectra = this.config.getForkSeq(data.slot) >= ForkSeq.electra; if (!isPostElectra && duty.committeeIndex != data.index) { throw Error( `Inconsistent duties during signing: duty.committeeIndex ${duty.committeeIndex} != att.committeeIndex ${data.index}` From f87ee1ab5613409748bff29c11249eae28736c4a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 27 Sep 2024 20:05:09 +0100 Subject: [PATCH 132/145] feat: add validator identities endpoint (#7107) * feat: add validator identities endpoint * Test against official spec release from beacon-apis repo * Prepopulate validatorIdentities in else clause * Log warning if validator index can't be resolved --- .../api/src/beacon/routes/beacon/index.ts | 1 + .../api/src/beacon/routes/beacon/state.ts | 52 +++++++++++++++++++ .../api/test/unit/beacon/oapiSpec.test.ts | 4 +- .../api/test/unit/beacon/testData/beacon.ts | 7 +++ .../src/api/impl/beacon/state/index.ts | 38 +++++++++++++- 5 files changed, 99 insertions(+), 3 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/index.ts b/packages/api/src/beacon/routes/beacon/index.ts index f70792f9d76f..39d7d995dfb1 100644 --- a/packages/api/src/beacon/routes/beacon/index.ts +++ b/packages/api/src/beacon/routes/beacon/index.ts @@ -25,6 +25,7 @@ export type { export type { StateId, ValidatorId, + ValidatorIdentities, ValidatorStatus, FinalityCheckpoints, ValidatorResponse, diff --git a/packages/api/src/beacon/routes/beacon/state.ts b/packages/api/src/beacon/routes/beacon/state.ts index 4f8d414f5b6c..e3cfe2e16edf 100644 --- a/packages/api/src/beacon/routes/beacon/state.ts +++ b/packages/api/src/beacon/routes/beacon/state.ts @@ -53,6 +53,14 @@ export const ValidatorResponseType = new ContainerType({ status: new StringType(), validator: ssz.phase0.Validator, }); +export const ValidatorIdentityType = new ContainerType( + { + index: ssz.ValidatorIndex, + pubkey: ssz.BLSPubkey, + activationEpoch: ssz.UintNum64, + }, + {jsonCase: "eth2"} +); export const EpochCommitteeResponseType = new ContainerType({ index: ssz.CommitteeIndex, slot: ssz.Slot, @@ -73,6 +81,7 @@ export const EpochSyncCommitteeResponseType = new ContainerType( {jsonCase: "eth2"} ); export const ValidatorResponseListType = ArrayOf(ValidatorResponseType); +export const ValidatorIdentitiesType = ArrayOf(ValidatorIdentityType); export const EpochCommitteeResponseListType = ArrayOf(EpochCommitteeResponseType); export const ValidatorBalanceListType = ArrayOf(ValidatorBalanceType); @@ -84,6 +93,7 @@ export type ValidatorBalance = ValueOf; export type EpochSyncCommitteeResponse = ValueOf; export type ValidatorResponseList = ValueOf; +export type ValidatorIdentities = ValueOf; export type EpochCommitteeResponseList = ValueOf; export type ValidatorBalanceList = ValueOf; @@ -191,6 +201,26 @@ export type Endpoints = { ExecutionOptimisticAndFinalizedMeta >; + /** + * Get validator identities from state + * + * Returns filterable list of validators identities. + * + * Identities will be returned for all indices or public keys that match known validators. If an index or public key does not + * match any known validator, no identity will be returned but this will not cause an error. There are no guarantees for the + * returned data in terms of ordering. + */ + postStateValidatorIdentities: Endpoint< + "POST", + StateArgs & { + /** An array of values, with each value either a hex encoded public key (any bytes48 with 0x prefix) or a validator index */ + validatorIds?: ValidatorId[]; + }, + {params: {state_id: string}; body: string[]}, + ValidatorIdentities, + ExecutionOptimisticAndFinalizedMeta + >; + /** * Get validator balances from state * Returns filterable list of validator balances. @@ -404,6 +434,28 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({ + params: {state_id: stateId.toString()}, + body: toValidatorIdsStr(validatorIds) || [], + }), + parseReqJson: ({params, body = []}) => ({ + stateId: params.state_id, + validatorIds: fromValidatorIdsStr(body), + }), + schema: { + params: {state_id: Schema.StringRequired}, + body: Schema.UintOrStringArray, + }, + }), + resp: { + data: ValidatorIdentitiesType, + meta: ExecutionOptimisticAndFinalizedCodec, + }, + }, getStateValidatorBalances: { url: "/eth/v1/beacon/states/{state_id}/validator_balances", method: "GET", diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index a84b2cc36767..89079cd0768c 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -21,9 +21,9 @@ import {testData as validatorTestData} from "./testData/validator.js"; // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const version = "v2.6.0-alpha.1"; +const version = "v3.0.0-alpha.6"; const openApiFile: OpenApiFile = { - url: `https://raw.githubusercontent.com/nflaig/beacon-api-spec/main/${version}/beacon-node-oapi.json`, + url: `https://github.com/ethereum/beacon-APIs/releases/download/${version}/beacon-node-oapi.json`, filepath: path.join(__dirname, "../../../oapi-schemas/beacon-node-oapi.json"), version: RegExp(version), }; diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index b0e958f4bc58..23ab13147454 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -191,6 +191,13 @@ export const testData: GenericServerTestCases = { args: {stateId: "head", validatorIds: [pubkeyHex, 1300], statuses: ["active_ongoing"]}, res: {data: [validatorResponse], meta: {executionOptimistic: true, finalized: false}}, }, + postStateValidatorIdentities: { + args: {stateId: "head", validatorIds: [1300]}, + res: { + data: [{index: 1300, pubkey: ssz.BLSPubkey.defaultValue(), activationEpoch: 1}], + meta: {executionOptimistic: true, finalized: false}, + }, + }, getStateValidator: { args: {stateId: "head", validatorId: pubkeyHex}, res: {data: validatorResponse, meta: {executionOptimistic: true, finalized: false}}, diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index 77e2fd5aab46..a5e46a2da4c0 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -22,7 +22,8 @@ import { export function getBeaconStateApi({ chain, config, -}: Pick): ApplicationMethods { + logger, +}: Pick): ApplicationMethods { async function getState( stateId: routes.beacon.StateId ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean}> { @@ -98,6 +99,8 @@ export function getBeaconStateApi({ currentEpoch ); validatorResponses.push(validatorResponse); + } else { + logger.warn(resp.reason, {id}); } } return { @@ -130,6 +133,39 @@ export function getBeaconStateApi({ return this.getStateValidators(args, context); }, + async postStateValidatorIdentities({stateId, validatorIds = []}) { + const {state, executionOptimistic, finalized} = await getStateResponse(chain, stateId); + const {pubkey2index} = chain.getHeadState().epochCtx; + + let validatorIdentities: routes.beacon.ValidatorIdentities; + + if (validatorIds.length) { + validatorIdentities = []; + for (const id of validatorIds) { + const resp = getStateValidatorIndex(id, state, pubkey2index); + if (resp.valid) { + const index = resp.validatorIndex; + const {pubkey, activationEpoch} = state.validators.getReadonly(index); + validatorIdentities.push({index, pubkey, activationEpoch}); + } else { + logger.warn(resp.reason, {id}); + } + } + } else { + const validatorsArr = state.validators.getAllReadonlyValues(); + validatorIdentities = new Array(validatorsArr.length) as routes.beacon.ValidatorIdentities; + for (let i = 0; i < validatorsArr.length; i++) { + const {pubkey, activationEpoch} = validatorsArr[i]; + validatorIdentities[i] = {index: i, pubkey, activationEpoch}; + } + } + + return { + data: validatorIdentities, + meta: {executionOptimistic, finalized}, + }; + }, + async getStateValidator({stateId, validatorId}) { const {state, executionOptimistic, finalized} = await getStateResponse(chain, stateId); const {pubkey2index} = chain.getHeadState().epochCtx; From 13d1a378b831b9d0a3f3ced89e317aa45147023c Mon Sep 17 00:00:00 2001 From: twoeths Date: Mon, 30 Sep 2024 08:59:55 +0700 Subject: [PATCH 133/145] fix: n-historical-states state serialization panel (#7111) --- dashboards/lodestar_state_cache_regen.json | 79 ++++++++++++++-------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/dashboards/lodestar_state_cache_regen.json b/dashboards/lodestar_state_cache_regen.json index 426cde226022..1cca1d26c561 100644 --- a/dashboards/lodestar_state_cache_regen.json +++ b/dashboards/lodestar_state_cache_regen.json @@ -425,7 +425,32 @@ }, "mappings": [] }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "epochs_in-memory" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 8, @@ -1325,30 +1350,6 @@ "value": "none" } ] - }, - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "sec_from_slot" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] } ] }, @@ -1378,7 +1379,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_cp_state_cache_state_serialize_seconds_sum[$rate_interval])\n/\nrate(lodestar_cp_state_cache_state_serialize_seconds_count[$rate_interval])", + "expr": "rate(lodestar_state_serialize_seconds_sum{source=\"persistent_checkpoints_cache_state\"}[$rate_interval])\n/\nrate(lodestar_state_serialize_seconds_count{source=\"persistent_checkpoints_cache_state\"}[$rate_interval])", "hide": false, "instant": false, "legendFormat": "serialize_duration", @@ -1475,6 +1476,30 @@ "id": "unit" } ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "reload_duration" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] } ] }, @@ -1618,7 +1643,7 @@ "options": { "mode": "exclude", "names": [ - "from_persistent" + "from_memory" ], "prefix": "All except:", "readOnly": true From d69d809e7ab5d5a2cf5030429e83b6af1d5af31a Mon Sep 17 00:00:00 2001 From: NC <17676176+ensi321@users.noreply.github.com> Date: Sun, 29 Sep 2024 19:58:19 -0700 Subject: [PATCH 134/145] feat: update correlation penalty computation (#7071) * Update slashing calculation to be more aligned with spec style * Update calculation * Lint * Fix rounding issue * Update calculation * Enable slashing spec test * lint --- .../beacon-node/test/spec/utils/specTestIterator.ts | 2 -- .../state-transition/src/epoch/processSlashings.ts | 12 ++++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index bd5142627e3f..d8b4f9c0574c 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -66,8 +66,6 @@ export const defaultSkipOpts: SkipOpts = { /^capella\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^deneb\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, /^electra\/light_client\/single_merkle_proof\/BeaconBlockBody.*/, - // TODO Electra: slashings tests to be enabled in PR#7071 - /^electra\/epoch_processing\/slashings.*/, ], skippedTests: [], skippedRunners: ["merkle_proof", "networking"], diff --git a/packages/state-transition/src/epoch/processSlashings.ts b/packages/state-transition/src/epoch/processSlashings.ts index ba4b483dffc2..23ee815cabb3 100644 --- a/packages/state-transition/src/epoch/processSlashings.ts +++ b/packages/state-transition/src/epoch/processSlashings.ts @@ -50,6 +50,10 @@ export function processSlashings( totalBalanceByIncrement ); const increment = EFFECTIVE_BALANCE_INCREMENT; + + const penaltyPerEffectiveBalanceIncrement = Math.floor( + (adjustedTotalSlashingBalanceByIncrement * increment) / totalBalanceByIncrement + ); const penalties: number[] = []; const penaltiesByEffectiveBalanceIncrement = new Map(); @@ -57,8 +61,12 @@ export function processSlashings( const effectiveBalanceIncrement = effectiveBalanceIncrements[index]; let penalty = penaltiesByEffectiveBalanceIncrement.get(effectiveBalanceIncrement); if (penalty === undefined) { - const penaltyNumeratorByIncrement = effectiveBalanceIncrement * adjustedTotalSlashingBalanceByIncrement; - penalty = Math.floor(penaltyNumeratorByIncrement / totalBalanceByIncrement) * increment; + if (fork < ForkSeq.electra) { + const penaltyNumeratorByIncrement = effectiveBalanceIncrement * adjustedTotalSlashingBalanceByIncrement; + penalty = Math.floor(penaltyNumeratorByIncrement / totalBalanceByIncrement) * increment; + } else { + penalty = penaltyPerEffectiveBalanceIncrement * effectiveBalanceIncrement; + } penaltiesByEffectiveBalanceIncrement.set(effectiveBalanceIncrement, penalty); } From fe7e21be0459dfba49ebe04e5a3efc1b1ee836d4 Mon Sep 17 00:00:00 2001 From: twoeths Date: Mon, 30 Sep 2024 23:09:52 +0700 Subject: [PATCH 135/145] feat: use napi-rs pubkey-index-map (#7091) * feat: use napi-rs pubkey-index-map * fix: export toMemoryEfficientHexStr() * fix: state-transition CachedBeaconState unit test * chore: remove commented code --- packages/beacon-node/package.json | 1 + .../src/api/impl/beacon/state/index.ts | 3 +- .../src/api/impl/beacon/state/utils.ts | 5 +-- .../src/api/impl/validator/index.ts | 2 +- packages/beacon-node/src/chain/chain.ts | 2 +- .../historicalState/getHistoricalState.ts | 2 +- .../src/chain/historicalState/worker.ts | 8 ++--- packages/beacon-node/src/chain/interface.ts | 2 +- .../src/chain/rewards/attestationsRewards.ts | 3 +- .../updateUnfinalizedPubkeys.test.ts | 17 +++++---- .../unit/api/impl/beacon/state/utils.test.ts | 8 ++++- packages/beacon-node/test/utils/state.ts | 2 +- packages/state-transition/package.json | 1 + .../src/block/processConsolidationRequest.ts | 2 +- .../src/block/processDeposit.ts | 2 +- .../src/block/processWithdrawalRequest.ts | 2 +- .../state-transition/src/cache/epochCache.ts | 15 ++++---- .../state-transition/src/cache/pubkeyCache.ts | 21 +---------- .../src/cache/syncCommitteeCache.ts | 4 +-- packages/state-transition/src/index.ts | 2 +- packages/state-transition/test/perf/util.ts | 2 +- .../perf/util/loadState/loadState.test.ts | 3 +- .../test/unit/cachedBeaconState.test.ts | 4 +-- .../test/unit/upgradeState.test.ts | 2 +- .../test/unit/util/cachedBeaconState.test.ts | 3 +- packages/state-transition/test/utils/state.ts | 2 +- yarn.lock | 36 +++++++++++++++++++ 27 files changed, 94 insertions(+), 62 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index f101206bfd9e..4deaf7c6ac16 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -105,6 +105,7 @@ "@chainsafe/prometheus-gc-stats": "^1.0.0", "@chainsafe/ssz": "^0.17.1", "@chainsafe/threads": "^1.11.1", + "@chainsafe/pubkey-index-map": "2.0.0", "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^10.0.1", "@fastify/cors": "^10.0.1", diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index a5e46a2da4c0..669aaa9d1fb3 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -9,6 +9,7 @@ import { getRandaoMix, } from "@lodestar/state-transition"; import {EPOCHS_PER_HISTORICAL_VECTOR} from "@lodestar/params"; +import {fromHex} from "@lodestar/utils"; import {ApiError} from "../../errors.js"; import {ApiModules} from "../../types.js"; import { @@ -200,7 +201,7 @@ export function getBeaconStateApi({ } balances.push({index: id, balance: state.balances.get(id)}); } else { - const index = headState.epochCtx.pubkey2index.get(id); + const index = headState.epochCtx.pubkey2index.get(fromHex(id)); if (index != null && index <= state.validators.length) { balances.push({index, balance: state.balances.get(index)}); } diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index 40b1e2815263..de21c6a244ed 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -1,6 +1,7 @@ +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {routes} from "@lodestar/api"; import {FAR_FUTURE_EPOCH, GENESIS_SLOT} from "@lodestar/params"; -import {BeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; +import {BeaconStateAllForks} from "@lodestar/state-transition"; import {BLSPubkey, Epoch, phase0, RootHex, Slot, ValidatorIndex} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice"; @@ -187,7 +188,7 @@ export function getStateValidatorIndex( // typeof id === Uint8Array const validatorIndex = pubkey2index.get(id); - if (validatorIndex === undefined) { + if (validatorIndex === null) { return {valid: false, code: 404, reason: "Validator pubkey not found in state"}; } if (validatorIndex >= state.validators.length) { diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 4876975c1bea..2347d7086d46 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -1359,7 +1359,7 @@ export function getValidatorApi( const filteredRegistrations = registrations.filter((registration) => { const {pubkey} = registration.message; const validatorIndex = headState.epochCtx.pubkey2index.get(pubkey); - if (validatorIndex === undefined) return false; + if (validatorIndex === null) return false; const validator = headState.validators.getReadonly(validatorIndex); const status = getValidatorStatus(validator, currentEpoch); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 696f4bd51dc3..371f660abe2e 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1,5 +1,6 @@ import path from "node:path"; import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import { BeaconStateAllForks, CachedBeaconStateAllForks, @@ -10,7 +11,6 @@ import { getEffectiveBalanceIncrementsZeroInactive, isCachedBeaconState, Index2PubkeyCache, - PubkeyIndexMap, EpochShuffling, computeEndSlotAtEpoch, computeAnchorCheckpoint, diff --git a/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts b/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts index ada4f3c284d7..1f352b3d683a 100644 --- a/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts +++ b/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts @@ -1,9 +1,9 @@ +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import { BeaconStateAllForks, CachedBeaconStateAllForks, DataAvailableStatus, ExecutionPayloadStatus, - PubkeyIndexMap, createCachedBeaconState, stateTransition, } from "@lodestar/state-transition"; diff --git a/packages/beacon-node/src/chain/historicalState/worker.ts b/packages/beacon-node/src/chain/historicalState/worker.ts index a07207cac5f5..2ea673d0faff 100644 --- a/packages/beacon-node/src/chain/historicalState/worker.ts +++ b/packages/beacon-node/src/chain/historicalState/worker.ts @@ -1,13 +1,9 @@ import worker from "node:worker_threads"; import {Transfer, expose} from "@chainsafe/threads/worker"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {createBeaconConfig, chainConfigFromJson} from "@lodestar/config"; import {getNodeLogger} from "@lodestar/logger/node"; -import { - EpochTransitionStep, - PubkeyIndexMap, - StateCloneSource, - StateHashTreeRootSource, -} from "@lodestar/state-transition"; +import {EpochTransitionStep, StateCloneSource, StateHashTreeRootSource} from "@lodestar/state-transition"; import {LevelDbController} from "@lodestar/db"; import {RegistryMetricCreator, collectNodeJSMetrics} from "../../metrics/index.js"; import {JobFnQueue} from "../../util/queue/fnQueue.js"; diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 5185662eaa4f..531f60dc0e63 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -1,4 +1,5 @@ import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import { UintNum64, Root, @@ -21,7 +22,6 @@ import { CachedBeaconStateAllForks, EpochShuffling, Index2PubkeyCache, - PubkeyIndexMap, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import {Logger} from "@lodestar/utils"; diff --git a/packages/beacon-node/src/chain/rewards/attestationsRewards.ts b/packages/beacon-node/src/chain/rewards/attestationsRewards.ts index e909e4b1b57e..70fb27de239a 100644 --- a/packages/beacon-node/src/chain/rewards/attestationsRewards.ts +++ b/packages/beacon-node/src/chain/rewards/attestationsRewards.ts @@ -24,6 +24,7 @@ import { isInInactivityLeak, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; +import {fromHex} from "@lodestar/utils"; export type AttestationsRewards = routes.beacon.AttestationsRewards; type IdealAttestationsReward = routes.beacon.IdealAttestationsReward; @@ -143,7 +144,7 @@ function computeTotalAttestationsRewardsAltair( const {flags} = transitionCache; const {epochCtx, config} = state; const validatorIndices = validatorIds - .map((id) => (typeof id === "number" ? id : epochCtx.pubkey2index.get(id))) + .map((id) => (typeof id === "number" ? id : epochCtx.pubkey2index.get(fromHex(id)))) .filter((index) => index !== undefined); // Validator indices to include in the result const inactivityPenaltyDenominator = config.INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_ALTAIR; diff --git a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts index 46659ab3d287..eab66d2bee53 100644 --- a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts +++ b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts @@ -1,10 +1,11 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; -import {Map} from "immutable"; +import {Map as ImmutableMap} from "immutable"; import {toBufferBE} from "bigint-buffer"; import {digest} from "@chainsafe/as-sha256"; import {SecretKey} from "@chainsafe/blst"; -import {ssz} from "@lodestar/types"; -import {type CachedBeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; +import {ValidatorIndex, ssz} from "@lodestar/types"; +import {type CachedBeaconStateAllForks, toMemoryEfficientHexStr} from "@lodestar/state-transition"; import {bytesToBigInt, intToBytes} from "@lodestar/utils"; import {InMemoryCheckpointStateCache, BlockStateCacheImpl} from "../../../../src/chain/stateCache/index.js"; import {BlockStateCache} from "../../../../src/chain/stateCache/types.js"; @@ -31,7 +32,7 @@ describe("updateUnfinalizedPubkeys perf tests", function () { itBench({ id: `updateUnfinalizedPubkeys - updating ${numPubkeysToBeFinalized} pubkeys`, beforeEach: async () => { - baseState.epochCtx.unfinalizedPubkey2index = Map(unfinalizedPubkey2Index.map); + baseState.epochCtx.unfinalizedPubkey2index = ImmutableMap(unfinalizedPubkey2Index); baseState.epochCtx.pubkey2index = new PubkeyIndexMap(); baseState.epochCtx.index2pubkey = []; @@ -80,12 +81,14 @@ describe("updateUnfinalizedPubkeys perf tests", function () { }); } - function generatePubkey2Index(startIndex: number, endIndex: number): PubkeyIndexMap { - const pubkey2Index = new PubkeyIndexMap(); + type PubkeyHex = string; + + function generatePubkey2Index(startIndex: number, endIndex: number): Map { + const pubkey2Index = new Map(); const pubkeys = generatePubkeys(endIndex - startIndex); for (let i = startIndex; i < endIndex; i++) { - pubkey2Index.set(pubkeys[i], i); + pubkey2Index.set(toMemoryEfficientHexStr(pubkeys[i]), i); } return pubkey2Index; diff --git a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts index a6020c0a3c13..79a8f3383e5d 100644 --- a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts +++ b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts @@ -117,7 +117,13 @@ describe("beacon state api utils", function () { // "validator id not in state" expect(getStateValidatorIndex(String(state.validators.length), state, pubkey2index).valid).toBe(false); // "validator pubkey not in state" - expect(getStateValidatorIndex("0xabcd", state, pubkey2index).valid).toBe(false); + expect( + getStateValidatorIndex( + "0xa99af0913a2834ef4959637e8d7c4e17f0b63adc587d36ab43510452db3102d0771a4554ea4118a33913827d5ee80b76", + state, + pubkey2index + ).valid + ).toBe(false); }); it("should return valid: true on validator indices / pubkeys in the state", () => { diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index 49a27435bdd7..6ad85f3422f7 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -1,10 +1,10 @@ import {SecretKey} from "@chainsafe/blst"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {config as minimalConfig} from "@lodestar/config/default"; import { BeaconStateAllForks, CachedBeaconStateAllForks, createCachedBeaconState, - PubkeyIndexMap, CachedBeaconStateBellatrix, BeaconStateBellatrix, CachedBeaconStateElectra, diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 2df87a522a63..6816e5679cf5 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -66,6 +66,7 @@ "@lodestar/config": "^1.22.0", "@lodestar/params": "^1.22.0", "@lodestar/types": "^1.22.0", + "@chainsafe/pubkey-index-map": "2.0.0", "@lodestar/utils": "^1.22.0", "bigint-buffer": "^1.1.5", "immutable": "^4.3.2" diff --git a/packages/state-transition/src/block/processConsolidationRequest.ts b/packages/state-transition/src/block/processConsolidationRequest.ts index 71b85e927336..691ecd5eca0b 100644 --- a/packages/state-transition/src/block/processConsolidationRequest.ts +++ b/packages/state-transition/src/block/processConsolidationRequest.ts @@ -24,7 +24,7 @@ export function processConsolidationRequest( const sourceIndex = state.epochCtx.getValidatorIndex(sourcePubkey); const targetIndex = state.epochCtx.getValidatorIndex(targetPubkey); - if (sourceIndex === undefined || targetIndex === undefined) { + if (sourceIndex === null || targetIndex === null) { return; } diff --git a/packages/state-transition/src/block/processDeposit.ts b/packages/state-transition/src/block/processDeposit.ts index 7e343d7fc33d..ee75dff0dfd1 100644 --- a/packages/state-transition/src/block/processDeposit.ts +++ b/packages/state-transition/src/block/processDeposit.ts @@ -65,7 +65,7 @@ export function applyDeposit( const {pubkey, withdrawalCredentials, amount} = deposit; const cachedIndex = epochCtx.getValidatorIndex(pubkey); - if (cachedIndex === undefined || !Number.isSafeInteger(cachedIndex) || cachedIndex >= validators.length) { + if (cachedIndex === null || !Number.isSafeInteger(cachedIndex) || cachedIndex >= validators.length) { if (isValidDepositSignature(config, pubkey, withdrawalCredentials, amount, deposit.signature)) { addValidatorToRegistry(fork, state, pubkey, withdrawalCredentials, amount); } diff --git a/packages/state-transition/src/block/processWithdrawalRequest.ts b/packages/state-transition/src/block/processWithdrawalRequest.ts index 0587a06ee179..e8a64ec63e41 100644 --- a/packages/state-transition/src/block/processWithdrawalRequest.ts +++ b/packages/state-transition/src/block/processWithdrawalRequest.ts @@ -33,7 +33,7 @@ export function processWithdrawalRequest( // bail out if validator is not in beacon state // note that we don't need to check for 6110 unfinalized vals as they won't be eligible for withdraw/exit anyway const validatorIndex = pubkey2index.get(withdrawalRequest.validatorPubkey); - if (validatorIndex === undefined) { + if (validatorIndex === null) { return; } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 3783c7f91f0d..5e901e33d992 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,5 +1,6 @@ import {PublicKey} from "@chainsafe/blst"; import * as immutable from "immutable"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import { BLSSignature, CommitteeIndex, @@ -53,7 +54,6 @@ import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from " import {BeaconStateAllForks, BeaconStateAltair} from "./types.js"; import { Index2PubkeyCache, - PubkeyIndexMap, UnfinalizedPubkeyIndexMap, syncPubkeys, toMemoryEfficientHexStr, @@ -1023,9 +1023,9 @@ export class EpochCache { return this.index2pubkey[index]; } - getValidatorIndex(pubkey: Uint8Array | PubkeyHex): ValidatorIndex | undefined { + getValidatorIndex(pubkey: Uint8Array): ValidatorIndex | null { if (this.isPostElectra()) { - return this.pubkey2index.get(pubkey) ?? this.unfinalizedPubkey2index.get(toMemoryEfficientHexStr(pubkey)); + return this.pubkey2index.get(pubkey) ?? this.unfinalizedPubkey2index.get(toMemoryEfficientHexStr(pubkey)) ?? null; } else { return this.pubkey2index.get(pubkey); } @@ -1059,17 +1059,20 @@ export class EpochCache { * Add finalized validator index and pubkey into finalized cache. * Since addFinalizedPubkey() primarily takes pubkeys from unfinalized cache, it can take pubkey hex string directly */ - addFinalizedPubkey(index: ValidatorIndex, pubkey: PubkeyHex | Uint8Array, metrics?: EpochCacheMetrics): void { + addFinalizedPubkey(index: ValidatorIndex, pubkeyOrHex: PubkeyHex | Uint8Array, metrics?: EpochCacheMetrics): void { + const pubkey = typeof pubkeyOrHex === "string" ? fromHex(pubkeyOrHex) : pubkeyOrHex; const existingIndex = this.pubkey2index.get(pubkey); - if (existingIndex !== undefined) { + if (existingIndex !== null) { if (existingIndex === index) { // Repeated insert. metrics?.finalizedPubkeyDuplicateInsert.inc(); return; } else { // attempt to insert the same pubkey with different index, should never happen. - throw Error("inserted existing pubkey into finalizedPubkey2index cache with a different index"); + throw Error( + `inserted existing pubkey into finalizedPubkey2index cache with a different index, index=${index} priorIndex=${existingIndex}` + ); } } diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index 44fe6df45592..f96436ec14f4 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,4 +1,5 @@ import {PublicKey} from "@chainsafe/blst"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import * as immutable from "immutable"; import {ValidatorIndex, phase0} from "@lodestar/types"; @@ -39,26 +40,6 @@ export function newUnfinalizedPubkeyIndexMap(): UnfinalizedPubkeyIndexMap { return immutable.Map(); } -export class PubkeyIndexMap { - // We don't really need the full pubkey. We could just use the first 20 bytes like an Ethereum address - readonly map = new Map(); - - get size(): number { - return this.map.size; - } - - /** - * Must support reading with string for API support where pubkeys are already strings - */ - get(key: Uint8Array | PubkeyHex): ValidatorIndex | undefined { - return this.map.get(toMemoryEfficientHexStr(key)); - } - - set(key: Uint8Array | PubkeyHex, value: ValidatorIndex): void { - this.map.set(toMemoryEfficientHexStr(key), value); - } -} - /** * Checks the pubkey indices against a state and adds missing pubkeys * diff --git a/packages/state-transition/src/cache/syncCommitteeCache.ts b/packages/state-transition/src/cache/syncCommitteeCache.ts index b0a93f121369..b9a65302c3e2 100644 --- a/packages/state-transition/src/cache/syncCommitteeCache.ts +++ b/packages/state-transition/src/cache/syncCommitteeCache.ts @@ -1,7 +1,7 @@ import {CompositeViewDU} from "@chainsafe/ssz"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ssz, ValidatorIndex} from "@lodestar/types"; import {toPubkeyHex} from "@lodestar/utils"; -import {PubkeyIndexMap} from "./pubkeyCache.js"; type SyncComitteeValidatorIndexMap = Map; @@ -82,7 +82,7 @@ function computeSyncCommitteeIndices( const pubkeys = syncCommittee.pubkeys.getAllReadonly(); for (const pubkey of pubkeys) { const validatorIndex = pubkey2index.get(pubkey); - if (validatorIndex === undefined) { + if (validatorIndex === null) { throw Error(`SyncCommittee pubkey is unknown ${toPubkeyHex(pubkey)}`); } diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 4ed801e3c490..600bbf173462 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -41,11 +41,11 @@ export { EpochCacheError, EpochCacheErrorCode, } from "./cache/epochCache.js"; +export {toMemoryEfficientHexStr} from "./cache/pubkeyCache.js"; export {type EpochTransitionCache, beforeProcessEpoch} from "./cache/epochTransitionCache.js"; // Aux data-structures export { - PubkeyIndexMap, type Index2PubkeyCache, type UnfinalizedPubkeyIndexMap, newUnfinalizedPubkeyIndexMap, diff --git a/packages/state-transition/test/perf/util.ts b/packages/state-transition/test/perf/util.ts index f3c2eaef91e5..c764e2d039f9 100644 --- a/packages/state-transition/test/perf/util.ts +++ b/packages/state-transition/test/perf/util.ts @@ -1,5 +1,6 @@ import {BitArray, fromHexString} from "@chainsafe/ssz"; import {PublicKey, SecretKey} from "@chainsafe/blst"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {phase0, ssz, Slot, BeaconState} from "@lodestar/types"; import {config} from "@lodestar/config/default"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; @@ -17,7 +18,6 @@ import { interopSecretKey, computeEpochAtSlot, getActiveValidatorIndices, - PubkeyIndexMap, newFilledArray, createCachedBeaconState, computeCommitteeCount, diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index 25b43e242d02..9f6175e95684 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -1,8 +1,9 @@ import {itBench, setBenchOpts} from "@dapplion/benchmark"; import {PublicKey} from "@chainsafe/blst"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {loadState} from "../../../../src/util/loadState/loadState.js"; import {createCachedBeaconState} from "../../../../src/cache/stateCache.js"; -import {Index2PubkeyCache, PubkeyIndexMap} from "../../../../src/cache/pubkeyCache.js"; +import {Index2PubkeyCache} from "../../../../src/cache/pubkeyCache.js"; import {generatePerfTestCachedStateAltair} from "../../util.js"; /** diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 092dda321610..96c026340143 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -1,11 +1,11 @@ import {fromHexString} from "@chainsafe/ssz"; import {describe, it, expect} from "vitest"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ssz} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {config as defaultConfig} from "@lodestar/config/default"; import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {createCachedBeaconStateTest} from "../utils/state.js"; -import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; import {interopPubkeysCached} from "../utils/interop.js"; import {modifyStateSameValidator, newStateWithValidators} from "../utils/capella.js"; @@ -83,7 +83,7 @@ describe("CachedBeaconState", () => { expect(state1.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); expect(state2.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); - expect(state1.epochCtx.getValidatorIndex(pubkey2)).toBe(undefined); + expect(state1.epochCtx.getValidatorIndex(pubkey2)).toBe(null); expect(state2.epochCtx.getValidatorIndex(pubkey2)).toBe(index2); }); diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index df9b052542f9..301cb105dc98 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -1,4 +1,5 @@ import {expect, describe, it} from "vitest"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {ssz} from "@lodestar/types"; import {ForkName} from "@lodestar/params"; import {createBeaconConfig, ChainForkConfig, createChainForkConfig} from "@lodestar/config"; @@ -7,7 +8,6 @@ import {config as chainConfig} from "@lodestar/config/default"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; import {upgradeStateToElectra} from "../../src/slot/upgradeStateToElectra.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; -import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; describe("upgradeState", () => { it("upgradeStateToDeneb", () => { diff --git a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts index 654e0752adb8..c85a8c7a2ffd 100644 --- a/packages/state-transition/test/unit/util/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/util/cachedBeaconState.test.ts @@ -1,8 +1,9 @@ import {describe, it} from "vitest"; +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {createBeaconConfig} from "@lodestar/config"; import {config} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; -import {createCachedBeaconState, PubkeyIndexMap} from "../../../src/index.js"; +import {createCachedBeaconState} from "../../../src/index.js"; describe("CachedBeaconState", () => { it("Create empty CachedBeaconState", () => { diff --git a/packages/state-transition/test/utils/state.ts b/packages/state-transition/test/utils/state.ts index 29a1f98b5562..9a79faf74480 100644 --- a/packages/state-transition/test/utils/state.ts +++ b/packages/state-transition/test/utils/state.ts @@ -1,3 +1,4 @@ +import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {config as minimalConfig} from "@lodestar/config/default"; import { EPOCHS_PER_HISTORICAL_VECTOR, @@ -18,7 +19,6 @@ import { CachedBeaconStateAllForks, BeaconStateAllForks, createCachedBeaconState, - PubkeyIndexMap, } from "../../src/index.js"; import {BeaconStateCache} from "../../src/cache/stateCache.js"; import {EpochCacheOpts} from "../../src/cache/epochCache.js"; diff --git a/yarn.lock b/yarn.lock index f546d770acd8..a96092ec2fff 100644 --- a/yarn.lock +++ b/yarn.lock @@ -586,6 +586,42 @@ resolved "https://registry.yarnpkg.com/@chainsafe/prometheus-gc-stats/-/prometheus-gc-stats-1.0.2.tgz#585f8f1555251db156d7e50ef8c86dd4f3e78f70" integrity sha512-h3mFKduSX85XMVbOdWOYvx9jNq99jGcRVNyW5goGOqju1CsI+ZJLhu5z4zBb/G+ksL0R4uLVulu/mIMe7Y0rNg== +"@chainsafe/pubkey-index-map-darwin-arm64@2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/pubkey-index-map-darwin-arm64/-/pubkey-index-map-darwin-arm64-2.0.0.tgz#e468e772787f2411ecab8e8316da6c801356b72d" + integrity sha512-7eROFdQvwN1b0zJy0YJd1wBSv8j+Sp8tc3HsyaLQvjX7w93LcPPe+2Y5QpMkECBFzD2BcvKFpYxIvkDzV2v8rw== + +"@chainsafe/pubkey-index-map-darwin-x64@2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/pubkey-index-map-darwin-x64/-/pubkey-index-map-darwin-x64-2.0.0.tgz#995755f71bcb49e5393a6af122c11a850aef4ce4" + integrity sha512-HfKIV83Y+AOugw0jaeUIHqe4Ikfwo47baFg97fpdcpUwPfWnw4Blej5C1zAyEX2IuUo2S1D450neTBSUgSdNCA== + +"@chainsafe/pubkey-index-map-linux-arm64-gnu@2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/pubkey-index-map-linux-arm64-gnu/-/pubkey-index-map-linux-arm64-gnu-2.0.0.tgz#0c25ffb451d9861515e26e68006aa08e18ebc42d" + integrity sha512-t7Tdy+m9lZF2gqs0LmxFTAztNe6tDuSxje0xS8LTYanBSWQ6ADbWjTxcp/63yBbIYGzncigePZG2iis9nxB95Q== + +"@chainsafe/pubkey-index-map-linux-x64-gnu@2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/pubkey-index-map-linux-x64-gnu/-/pubkey-index-map-linux-x64-gnu-2.0.0.tgz#26c3628faaeb1ef9b47952cc03ae209d7e9656d8" + integrity sha512-1DKoITe7ZwjClhCBpIZq7SOIOJbUNLxgsFuV7e0ZcBq+tz5UqhKB8SSRzNn7THoo+XRg1mJiDyFPzDKGxHxRkg== + +"@chainsafe/pubkey-index-map-win32-x64-msvc@2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/pubkey-index-map-win32-x64-msvc/-/pubkey-index-map-win32-x64-msvc-2.0.0.tgz#0e35c67ed9dcaaee6ff9e582ed6a733d852473e9" + integrity sha512-hnEZBtTFxTl52lytogexOtzqPQyUKKB28mLbLTZnl2OicsEfNcczJpgF6o1uQ0O0zktAn/m1Tc6/iHmQg2VuhQ== + +"@chainsafe/pubkey-index-map@2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@chainsafe/pubkey-index-map/-/pubkey-index-map-2.0.0.tgz#a8262b353e8335e9acf5e750353a53c55e5cf9be" + integrity sha512-2mVvWrHGApF3mPS7ecp8k3dI/C3QF5824bpQNSRWDsmZEU9H3HzITIj256v14QiB+22MIitpWkBc6hl2bjhJ+Q== + optionalDependencies: + "@chainsafe/pubkey-index-map-darwin-arm64" "2.0.0" + "@chainsafe/pubkey-index-map-darwin-x64" "2.0.0" + "@chainsafe/pubkey-index-map-linux-arm64-gnu" "2.0.0" + "@chainsafe/pubkey-index-map-linux-x64-gnu" "2.0.0" + "@chainsafe/pubkey-index-map-win32-x64-msvc" "2.0.0" + "@chainsafe/ssz@^0.11.1": version "0.11.1" resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.11.1.tgz#d4aec883af2ec5196ae67b96242c467da20b2476" From c4952eef7ee2e362e08a313c128e22e394bc44e0 Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 1 Oct 2024 21:29:22 +0700 Subject: [PATCH 136/145] fix: adjust n-historical-states param (#7104) * fix: default params of n-historical-states config * chore: improve log * feat: enable nHistoricalStates by default * chore: fix typo in log --- packages/beacon-node/src/chain/options.ts | 2 +- packages/beacon-node/src/chain/regen/regen.ts | 2 +- .../chain/stateCache/fifoBlockStateCache.ts | 9 +++++-- .../stateCache/persistentCheckpointsCache.ts | 7 +++--- .../src/network/processor/gossipHandlers.ts | 24 ++++++++++--------- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 7c7cfcdde75b..bc2b73256272 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -111,7 +111,7 @@ export const defaultChainOptions: IChainOptions = { // batching too much may block the I/O thread so if useWorker=false, suggest this value to be 32 // since this batch attestation work is designed to work with useWorker=true, make this the lowest value minSameMessageSignatureSetsToBatch: 2, - nHistoricalStates: false, + nHistoricalStates: true, nHistoricalStatesFileDataStore: false, maxBlockStates: DEFAULT_MAX_BLOCK_STATES, maxCPStateEpochsInMemory: DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY, diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 08fda70d9184..7c663c6e0d3d 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -218,7 +218,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { blockPromises[i] = this.modules.db.block.get(fromHex(protoBlock.blockRoot)); } - const logCtx = {stateRoot, replaySlots: replaySlots.join(",")}; + const logCtx = {stateRoot, caller, replaySlots: replaySlots.join(",")}; this.modules.logger.debug("Replaying blocks to get state", logCtx); const loadBlocksTimer = this.modules.metrics?.regenGetState.loadBlocks.startTimer({caller}); diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index d38fc323174c..7766daf3c5b3 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -13,9 +13,14 @@ export type FIFOBlockStateCacheOpts = { }; /** - * Regen state if there's a reorg distance > 32 slots. + * Given `maxSkipSlots` = 32 and `DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE` = 32, lodestar doesn't need to + * reload states in order to process a gossip block. + * + * |-----------------------------------------------|-----------------------------------------------| + * maxSkipSlots DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE ^ + * clock slot */ -export const DEFAULT_MAX_BLOCK_STATES = 32; +export const DEFAULT_MAX_BLOCK_STATES = 64; /** * New implementation of BlockStateCache that keeps the most recent n states consistently diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 07c96f224bee..0719efcfd309 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -53,10 +53,11 @@ type CacheItem = InMemoryCacheItem | PersistedCacheItem; type LoadedStateBytesData = {persistedKey: DatastoreKey; stateBytes: Uint8Array}; /** - * Before n-historical states, lodestar keeps mostly 3 states in memory with 1 finalized state - * Since Jan 2024, lodestar stores the finalized state in disk and keeps up to 2 epochs in memory + * Before n-historical states, lodestar keeps all checkpoint states since finalized + * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state + * may not be available in memory, and stay on disk instead. */ -export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 2; +export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 3; /** * An implementation of CheckpointStateCache that keep up to n epoch checkpoint states in memory and persist the rest to disk diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 9e6f08a803c1..8df9f7b78126 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -144,6 +144,18 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler const blockInputMeta = config.getForkSeq(signedBlock.message.slot) >= ForkSeq.deneb ? blockInputRes.blockInputMeta : {}; + const logCtx = { + slot: slot, + root: blockHex, + currentSlot: chain.clock.currentSlot, + peerId: peerIdStr, + delaySec, + ...blockInputMeta, + recvToValLatency, + }; + + logger.debug("Received gossip block", {...logCtx}); + try { await validateGossipBlock(config, chain, signedBlock, fork); @@ -153,17 +165,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler metrics?.gossipBlock.gossipValidation.recvToValidation.observe(recvToValidation); metrics?.gossipBlock.gossipValidation.validationTime.observe(validationTime); - logger.debug("Received gossip block", { - slot: slot, - root: blockHex, - curentSlot: chain.clock.currentSlot, - peerId: peerIdStr, - delaySec, - ...blockInputMeta, - recvToValLatency, - recvToValidation, - validationTime, - }); + logger.debug("Validated gossip block", {...logCtx, recvToValidation, validationTime}); return blockInput; } catch (e) { From b457778eefb31ed4286a9da9597d51a0818b0f79 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 2 Oct 2024 14:05:16 +0100 Subject: [PATCH 137/145] feat: allow to configure api client when initializing light client (#7121) --- packages/light-client/README.md | 6 +++--- packages/light-client/src/utils/api.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/light-client/README.md b/packages/light-client/README.md index cd3a0265008a..0323e8fc4326 100644 --- a/packages/light-client/README.md +++ b/packages/light-client/README.md @@ -62,7 +62,7 @@ import { const config = getChainForkConfigFromNetwork("sepolia"); const logger = getConsoleLogger({logDebug: Boolean(process.env.DEBUG)}); -const api = getApiFromUrl({urls: ["https://lodestar-sepolia.chainsafe.io"]}, {config}); +const api = getApiFromUrl("https://lodestar-sepolia.chainsafe.io", "sepolia"); const lightclient = await Lightclient.initializeFromCheckpointRoot({ config, @@ -82,11 +82,11 @@ await lightclient.start(); logger.info("Lightclient synced"); lightclient.emitter.on(LightclientEvent.lightClientFinalityHeader, async (finalityUpdate) => { - logger.info(finalityUpdate); + logger.info("Received finality update", {slot: finalityUpdate.beacon.slot}); }); lightclient.emitter.on(LightclientEvent.lightClientOptimisticHeader, async (optimisticUpdate) => { - logger.info(optimisticUpdate); + logger.info("Received optimistic update", {slot: optimisticUpdate.beacon.slot}); }); ``` diff --git a/packages/light-client/src/utils/api.ts b/packages/light-client/src/utils/api.ts index 7947aa96dd3e..6ccb187052e1 100644 --- a/packages/light-client/src/utils/api.ts +++ b/packages/light-client/src/utils/api.ts @@ -1,13 +1,13 @@ -import {getClient, ApiClient} from "@lodestar/api"; +import {getClient, ApiClient, ApiRequestInit} from "@lodestar/api"; import {ChainForkConfig, createChainForkConfig} from "@lodestar/config"; import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; -export function getApiFromUrl(url: string, network: NetworkName): ApiClient { +export function getApiFromUrl(url: string, network: NetworkName, init?: ApiRequestInit): ApiClient { if (!(network in networksChainConfig)) { throw Error(`Invalid network name "${network}". Valid options are: ${Object.keys(networksChainConfig).join()}`); } - return getClient({urls: [url]}, {config: createChainForkConfig(networksChainConfig[network])}); + return getClient({urls: [url], globalInit: init}, {config: createChainForkConfig(networksChainConfig[network])}); } export function getChainForkConfigFromNetwork(network: NetworkName): ChainForkConfig { From bf4a25f0219481d176996b81206b0ca66f971aba Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 2 Oct 2024 16:24:08 +0100 Subject: [PATCH 138/145] fix: update HTTP error response format to be spec compliant (#7110) * fix: update HTTP error response format to be spec compliant * Fix error assertions * Remove group tag from keymanager option * Fix body has already been read * Assert deepEqual --- packages/beacon-node/src/api/rest/base.ts | 38 +++++++++++++++++-- packages/beacon-node/src/api/rest/index.ts | 1 + packages/cli/src/cmds/dev/options.ts | 4 ++ packages/cli/src/cmds/validator/handler.ts | 1 + .../src/cmds/validator/keymanager/server.ts | 1 + packages/cli/src/cmds/validator/options.ts | 6 +++ .../cli/src/options/beaconNodeOptions/api.ts | 9 +++++ packages/cli/test/sim/endpoints.test.ts | 16 +++++++- .../unit/options/beaconNodeOptions.test.ts | 2 + 9 files changed, 74 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index 5f191bf76beb..e8ea85f67af5 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -15,6 +15,7 @@ export type RestApiServerOpts = { bearerToken?: string; headerLimit?: number; bodyLimit?: number; + stacktraces?: boolean; swaggerUI?: boolean; }; @@ -29,11 +30,27 @@ export type RestApiServerMetrics = SocketMetrics & { errors: Gauge<{operationId: string}>; }; +/** + * Error response body format as defined in beacon-api spec + * + * See https://github.com/ethereum/beacon-APIs/blob/v2.5.0/types/http.yaml + */ +type ErrorResponse = { + code: number; + message: string; + stacktraces?: string[]; +}; + /** * Error code used by Fastify if media type is not supported (415) */ const INVALID_MEDIA_TYPE_CODE = errorCodes.FST_ERR_CTP_INVALID_MEDIA_TYPE().code; +/** + * Error code used by Fastify if JSON schema validation failed + */ +const SCHEMA_VALIDATION_ERROR_CODE = errorCodes.FST_ERR_VALIDATION().code; + /** * REST API powered by `fastify` server. */ @@ -71,15 +88,30 @@ export class RestApiServer { // To parse our ApiError -> statusCode server.setErrorHandler((err, _req, res) => { + const stacktraces = opts.stacktraces ? err.stack?.split("\n") : undefined; if (err.validation) { - void res.status(400).send(err.validation); + const {instancePath, message} = err.validation[0]; + const payload: ErrorResponse = { + code: err.statusCode ?? 400, + message: `${instancePath.substring(instancePath.lastIndexOf("/") + 1)} ${message}`, + stacktraces, + }; + void res.status(400).send(payload); } else { // Convert our custom ApiError into status code const statusCode = err instanceof ApiError ? err.statusCode : 500; - void res.status(statusCode).send(err); + const payload: ErrorResponse = {code: statusCode, message: err.message, stacktraces}; + void res.status(statusCode).send(payload); } }); + server.setNotFoundHandler((req, res) => { + const message = `Route ${req.raw.method}:${req.raw.url} not found`; + this.logger.warn(message); + const payload: ErrorResponse = {code: 404, message}; + void res.code(404).send(payload); + }); + if (opts.cors) { void server.register(fastifyCors, {origin: opts.cors}); } @@ -127,7 +159,7 @@ export class RestApiServer { const operationId = getOperationId(req); - if (err instanceof ApiError || err.code === INVALID_MEDIA_TYPE_CODE) { + if (err instanceof ApiError || [INVALID_MEDIA_TYPE_CODE, SCHEMA_VALIDATION_ERROR_CODE].includes(err.code)) { this.logger.warn(`Req ${req.id} ${operationId} failed`, {reason: err.message}); } else { this.logger.error(`Req ${req.id} ${operationId} error`, {}, err); diff --git a/packages/beacon-node/src/api/rest/index.ts b/packages/beacon-node/src/api/rest/index.ts index e27ed6bd9139..6beaf061588c 100644 --- a/packages/beacon-node/src/api/rest/index.ts +++ b/packages/beacon-node/src/api/rest/index.ts @@ -22,6 +22,7 @@ export const beaconRestApiServerOpts: BeaconRestApiServerOpts = { cors: "*", // beacon -> validator API is trusted, and for large amounts of keys the payload is multi-MB bodyLimit: 20 * 1024 * 1024, // 20MB for big block + blobs + stacktraces: false, }; export type BeaconRestApiServerModules = RestApiServerModules & { diff --git a/packages/cli/src/cmds/dev/options.ts b/packages/cli/src/cmds/dev/options.ts index c484150e58d7..5286b81729c6 100644 --- a/packages/cli/src/cmds/dev/options.ts +++ b/packages/cli/src/cmds/dev/options.ts @@ -90,6 +90,10 @@ const externalOptionsOverrides: Partial = { @@ -141,6 +142,11 @@ export const keymanagerOptions: CliCommandOptions = { type: "number", description: "Defines the maximum payload, in bytes, the server is allowed to accept", }, + "keymanager.stacktraces": { + hidden: true, + type: "boolean", + description: "Return stacktraces in HTTP error responses", + }, }; export const validatorOptions: CliCommandOptions = { diff --git a/packages/cli/src/options/beaconNodeOptions/api.ts b/packages/cli/src/options/beaconNodeOptions/api.ts index 996136f262ec..bed0105fd944 100644 --- a/packages/cli/src/options/beaconNodeOptions/api.ts +++ b/packages/cli/src/options/beaconNodeOptions/api.ts @@ -12,6 +12,7 @@ export type ApiArgs = { "rest.port": number; "rest.headerLimit"?: number; "rest.bodyLimit"?: number; + "rest.stacktraces"?: boolean; "rest.swaggerUI"?: boolean; }; @@ -26,6 +27,7 @@ export function parseArgs(args: ApiArgs): IBeaconNodeOptions["api"] { port: args["rest.port"], headerLimit: args["rest.headerLimit"], bodyLimit: args["rest.bodyLimit"], + stacktraces: args["rest.stacktraces"], swaggerUI: args["rest.swaggerUI"], }, }; @@ -92,6 +94,13 @@ export const options: CliCommandOptions = { description: "Defines the maximum payload, in bytes, the server is allowed to accept", }, + "rest.stacktraces": { + hidden: true, + type: "boolean", + description: "Return stacktraces in HTTP error responses", + group: "api", + }, + "rest.swaggerUI": { type: "boolean", description: "Enable Swagger UI for API exploration at http://{address}:{port}/documentation", diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts index a42b9568f9a3..b276ad4787c6 100644 --- a/packages/cli/test/sim/endpoints.test.ts +++ b/packages/cli/test/sim/endpoints.test.ts @@ -2,7 +2,7 @@ import path from "node:path"; import assert from "node:assert"; import {toHexString} from "@chainsafe/ssz"; -import {routes} from "@lodestar/api"; +import {routes, fetch} from "@lodestar/api"; import {Simulation} from "../utils/crucible/simulation.js"; import {BeaconClient, ExecutionClient} from "../utils/crucible/interfaces.js"; import {defineSimTestConfig, logFilesDir} from "../utils/crucible/utils/index.js"; @@ -105,6 +105,20 @@ await env.tracker.assert( } ); +await env.tracker.assert("should return HTTP error responses in a spec compliant format", async () => { + // ApiError with status 400 is thrown by handler + const res1 = await node.api.beacon.getStateValidator({stateId: "current", validatorId: 1}); + assert.deepEqual(JSON.parse(await res1.errorBody()), {code: 400, message: "Invalid block id 'current'"}); + + // JSON schema validation failed + const res2 = await node.api.beacon.getPoolAttestationsV2({slot: "current" as unknown as number, committeeIndex: 123}); + assert.deepEqual(JSON.parse(await res2.errorBody()), {code: 400, message: "slot must be integer"}); + + // Route does not exist + const res3 = await fetch(`${node.restPublicUrl}/not/implemented/route`); + assert.deepEqual(JSON.parse(await res3.text()), {code: 404, message: "Route GET:/not/implemented/route not found"}); +}); + await env.tracker.assert("BN Not Synced", async () => { const expectedSyncStatus: routes.node.SyncingStatus = { headSlot: 2, diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index d74ae73b966f..f873102edf74 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -17,6 +17,7 @@ describe("options / beaconNodeOptions", () => { "rest.port": 7654, "rest.headerLimit": 16384, "rest.bodyLimit": 30e6, + "rest.stacktraces": true, "chain.blsVerifyAllMultiThread": true, "chain.blsVerifyAllMainThread": true, @@ -122,6 +123,7 @@ describe("options / beaconNodeOptions", () => { port: 7654, headerLimit: 16384, bodyLimit: 30e6, + stacktraces: true, }, }, chain: { From a19655d95b02861840430a676cebaa9e79ec023c Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 2 Oct 2024 17:11:06 +0100 Subject: [PATCH 139/145] fix: update multiple api errors to be spec compliant (#7113) * fix: update HTTP error response format to be spec compliant * Fix error assertions * Remove group tag from keymanager option * Fix body has already been read * Assert deepEqual * fix: update multiple api errors to be spec compliant * Reorder test cases * Update res numbering * Use strict deep equal --- packages/api/src/utils/client/response.ts | 5 ++- .../src/api/impl/beacon/pool/index.ts | 32 ++++++++----------- packages/beacon-node/src/api/impl/errors.ts | 13 ++++++++ packages/beacon-node/src/api/rest/base.ts | 14 +++++++- packages/cli/test/sim/endpoints.test.ts | 24 +++++++++++--- 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/packages/api/src/utils/client/response.ts b/packages/api/src/utils/client/response.ts index ed008273588a..fdcb2afda943 100644 --- a/packages/api/src/utils/client/response.ts +++ b/packages/api/src/utils/client/response.ts @@ -189,8 +189,11 @@ export class ApiResponse extends Response { private getErrorMessage(): string { const errBody = this.resolvedErrorBody(); try { - const errJson = JSON.parse(errBody) as {message?: string}; + const errJson = JSON.parse(errBody) as {message?: string; failures?: {message: string}[]}; if (errJson.message) { + if (errJson.failures) { + return `${errJson.message}\n` + errJson.failures.map((e) => e.message).join("\n"); + } return errJson.message; } else { return errBody; diff --git a/packages/beacon-node/src/api/impl/beacon/pool/index.ts b/packages/beacon-node/src/api/impl/beacon/pool/index.ts index 398238aa2508..e01b172f1e72 100644 --- a/packages/beacon-node/src/api/impl/beacon/pool/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/pool/index.ts @@ -16,7 +16,7 @@ import { SyncCommitteeError, } from "../../../../chain/errors/index.js"; import {validateGossipFnRetryUnknownRoot} from "../../../../network/processor/gossipHandlers.js"; -import {ApiError} from "../../errors.js"; +import {ApiError, FailureList, IndexedError} from "../../errors.js"; export function getBeaconPoolApi({ chain, @@ -88,7 +88,7 @@ export function getBeaconPoolApi({ async submitPoolAttestationsV2({signedAttestations}) { const seenTimestampSec = Date.now() / 1000; - const errors: Error[] = []; + const failures: FailureList = []; await Promise.all( signedAttestations.map(async (attestation, i) => { @@ -127,7 +127,7 @@ export function getBeaconPoolApi({ return; } - errors.push(e as Error); + failures.push({index: i, message: (e as Error).message}); logger.error(`Error on submitPoolAttestations [${i}]`, logCtx, e as Error); if (e instanceof AttestationError && e.action === GossipAction.REJECT) { chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "api_reject"); @@ -136,10 +136,8 @@ export function getBeaconPoolApi({ }) ); - if (errors.length > 1) { - throw Error("Multiple errors on submitPoolAttestations\n" + errors.map((e) => e.message).join("\n")); - } else if (errors.length === 1) { - throw errors[0]; + if (failures.length > 0) { + throw new IndexedError("Error processing attestations", failures); } }, @@ -168,7 +166,7 @@ export function getBeaconPoolApi({ }, async submitPoolBLSToExecutionChange({blsToExecutionChanges}) { - const errors: Error[] = []; + const failures: FailureList = []; await Promise.all( blsToExecutionChanges.map(async (blsToExecutionChange, i) => { @@ -184,7 +182,7 @@ export function getBeaconPoolApi({ await network.publishBlsToExecutionChange(blsToExecutionChange); } } catch (e) { - errors.push(e as Error); + failures.push({index: i, message: (e as Error).message}); logger.error( `Error on submitPoolBLSToExecutionChange [${i}]`, {validatorIndex: blsToExecutionChange.message.validatorIndex}, @@ -194,10 +192,8 @@ export function getBeaconPoolApi({ }) ); - if (errors.length > 1) { - throw Error("Multiple errors on submitPoolBLSToExecutionChange\n" + errors.map((e) => e.message).join("\n")); - } else if (errors.length === 1) { - throw errors[0]; + if (failures.length > 0) { + throw new IndexedError("Error processing BLS to execution changes", failures); } }, @@ -221,7 +217,7 @@ export function getBeaconPoolApi({ // TODO: Fetch states at signature slots const state = chain.getHeadState(); - const errors: Error[] = []; + const failures: FailureList = []; await Promise.all( signatures.map(async (signature, i) => { @@ -261,7 +257,7 @@ export function getBeaconPoolApi({ return; } - errors.push(e as Error); + failures.push({index: i, message: (e as Error).message}); logger.debug( `Error on submitPoolSyncCommitteeSignatures [${i}]`, {slot: signature.slot, validatorIndex: signature.validatorIndex}, @@ -274,10 +270,8 @@ export function getBeaconPoolApi({ }) ); - if (errors.length > 1) { - throw Error("Multiple errors on submitPoolSyncCommitteeSignatures\n" + errors.map((e) => e.message).join("\n")); - } else if (errors.length === 1) { - throw errors[0]; + if (failures.length > 0) { + throw new IndexedError("Error processing sync committee signatures", failures); } }, }; diff --git a/packages/beacon-node/src/api/impl/errors.ts b/packages/beacon-node/src/api/impl/errors.ts index 848691f7cf6d..609f40f83a12 100644 --- a/packages/beacon-node/src/api/impl/errors.ts +++ b/packages/beacon-node/src/api/impl/errors.ts @@ -35,3 +35,16 @@ export class OnlySupportedByDVT extends ApiError { super(501, "Only supported by distributed validator middleware clients"); } } + +// Error thrown when processing multiple items failed - https://github.com/ethereum/beacon-APIs/blob/e7f7d70423b0abfe9d9f33b701be2ec03e44eb02/types/http.yaml#L175 +export class IndexedError extends ApiError { + failures: FailureList; + + constructor(message: string, failures: FailureList) { + super(400, message); + + this.failures = failures.sort((a, b) => a.index - b.index); + } +} + +export type FailureList = {index: number; message: string}[]; diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index e8ea85f67af5..10318b1b3ba7 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -5,7 +5,7 @@ import bearerAuthPlugin from "@fastify/bearer-auth"; import {addSszContentTypeParser} from "@lodestar/api/server"; import {ErrorAborted, Gauge, Histogram, Logger} from "@lodestar/utils"; import {isLocalhostIP} from "../../util/ip.js"; -import {ApiError, NodeIsSyncing} from "../impl/errors.js"; +import {ApiError, FailureList, IndexedError, NodeIsSyncing} from "../impl/errors.js"; import {HttpActiveSocketsTracker, SocketMetrics} from "./activeSockets.js"; export type RestApiServerOpts = { @@ -41,6 +41,10 @@ type ErrorResponse = { stacktraces?: string[]; }; +type IndexedErrorResponse = ErrorResponse & { + failures?: FailureList; +}; + /** * Error code used by Fastify if media type is not supported (415) */ @@ -97,6 +101,14 @@ export class RestApiServer { stacktraces, }; void res.status(400).send(payload); + } else if (err instanceof IndexedError) { + const payload: IndexedErrorResponse = { + code: err.statusCode, + message: err.message, + failures: err.failures, + stacktraces, + }; + void res.status(err.statusCode).send(payload); } else { // Convert our custom ApiError into status code const statusCode = err instanceof ApiError ? err.statusCode : 500; diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts index b276ad4787c6..6a119fc219d7 100644 --- a/packages/cli/test/sim/endpoints.test.ts +++ b/packages/cli/test/sim/endpoints.test.ts @@ -3,6 +3,7 @@ import path from "node:path"; import assert from "node:assert"; import {toHexString} from "@chainsafe/ssz"; import {routes, fetch} from "@lodestar/api"; +import {ssz} from "@lodestar/types"; import {Simulation} from "../utils/crucible/simulation.js"; import {BeaconClient, ExecutionClient} from "../utils/crucible/interfaces.js"; import {defineSimTestConfig, logFilesDir} from "../utils/crucible/utils/index.js"; @@ -108,15 +109,30 @@ await env.tracker.assert( await env.tracker.assert("should return HTTP error responses in a spec compliant format", async () => { // ApiError with status 400 is thrown by handler const res1 = await node.api.beacon.getStateValidator({stateId: "current", validatorId: 1}); - assert.deepEqual(JSON.parse(await res1.errorBody()), {code: 400, message: "Invalid block id 'current'"}); + assert.deepStrictEqual(JSON.parse(await res1.errorBody()), {code: 400, message: "Invalid block id 'current'"}); // JSON schema validation failed const res2 = await node.api.beacon.getPoolAttestationsV2({slot: "current" as unknown as number, committeeIndex: 123}); - assert.deepEqual(JSON.parse(await res2.errorBody()), {code: 400, message: "slot must be integer"}); + assert.deepStrictEqual(JSON.parse(await res2.errorBody()), {code: 400, message: "slot must be integer"}); + + // Error processing multiple items + const signedAttestations = Array.from({length: 3}, () => ssz.phase0.Attestation.defaultValue()); + const res3 = await node.api.beacon.submitPoolAttestationsV2({signedAttestations}); + const errBody = JSON.parse(await res3.errorBody()) as {code: number; message: string; failures: unknown[]}; + assert.equal(errBody.code, 400); + assert.equal(errBody.message, "Error processing attestations"); + assert.equal(errBody.failures.length, signedAttestations.length); + assert.deepStrictEqual(errBody.failures[0], { + index: 0, + message: "ATTESTATION_ERROR_NOT_EXACTLY_ONE_AGGREGATION_BIT_SET", + }); // Route does not exist - const res3 = await fetch(`${node.restPublicUrl}/not/implemented/route`); - assert.deepEqual(JSON.parse(await res3.text()), {code: 404, message: "Route GET:/not/implemented/route not found"}); + const res4 = await fetch(`${node.restPublicUrl}/not/implemented/route`); + assert.deepStrictEqual(JSON.parse(await res4.text()), { + code: 404, + message: "Route GET:/not/implemented/route not found", + }); }); await env.tracker.assert("BN Not Synced", async () => { From 0d1fd9c839a84cfe91e0b9e6af95c27261bb5ef0 Mon Sep 17 00:00:00 2001 From: twoeths Date: Fri, 4 Oct 2024 19:29:20 +0700 Subject: [PATCH 140/145] chore: remove IndexedGossipQueueAvgTime (#7125) --- .../network/processor/gossipQueues/index.ts | 12 +- .../processor/gossipQueues/indexedAvgTime.ts | 129 ------------------ .../network/processor/gossipQueues/types.ts | 11 +- 3 files changed, 2 insertions(+), 150 deletions(-) delete mode 100644 packages/beacon-node/src/network/processor/gossipQueues/indexedAvgTime.ts diff --git a/packages/beacon-node/src/network/processor/gossipQueues/index.ts b/packages/beacon-node/src/network/processor/gossipQueues/index.ts index 347458c91445..968c11501426 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues/index.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues/index.ts @@ -3,16 +3,8 @@ import {GossipType} from "../../gossip/interface.js"; import {PendingGossipsubMessage} from "../types.js"; import {getGossipAttestationIndex} from "../../../util/sszBytes.js"; import {LinearGossipQueue} from "./linear.js"; -import { - DropType, - GossipQueue, - GossipQueueOpts, - QueueType, - isIndexedGossipQueueAvgTimeOpts, - isIndexedGossipQueueMinSizeOpts, -} from "./types.js"; +import {DropType, GossipQueue, GossipQueueOpts, QueueType, isIndexedGossipQueueMinSizeOpts} from "./types.js"; import {IndexedGossipQueueMinSize} from "./indexed.js"; -import {IndexedGossipQueueAvgTime} from "./indexedAvgTime.js"; /** * In normal condition, the higher this value the more efficient the signature verification. @@ -120,8 +112,6 @@ export function createGossipQueues(beaconAttestationBatchValidation = false): { return mapValues(gossipQueueOpts, (opts) => { if (isIndexedGossipQueueMinSizeOpts(opts)) { return new IndexedGossipQueueMinSize(opts); - } else if (isIndexedGossipQueueAvgTimeOpts(opts)) { - return new IndexedGossipQueueAvgTime(opts); } else { return new LinearGossipQueue(opts); } diff --git a/packages/beacon-node/src/network/processor/gossipQueues/indexedAvgTime.ts b/packages/beacon-node/src/network/processor/gossipQueues/indexedAvgTime.ts deleted file mode 100644 index b7bba4c132bb..000000000000 --- a/packages/beacon-node/src/network/processor/gossipQueues/indexedAvgTime.ts +++ /dev/null @@ -1,129 +0,0 @@ -import {GossipQueue, IndexedGossipQueueOpts} from "./types.js"; - -type ItemList = { - items: T[]; - avgRecvTimestampMs: number; -}; - -function listScore(list: ItemList): number { - return list.items.length / Math.max(1000, Date.now() - list.avgRecvTimestampMs); -} - -/** - * An implementation of GossipQueue that tries to run the batch with highest score first. - * TODO: add unit tests - * - index items by indexFn using a map - * - compute avgRecvTimestampMs for each key every time we add new item - * - on next, pick the key with the highest score (check the score function above) - */ -export class IndexedGossipQueueAvgTime implements GossipQueue { - private _length = 0; - private indexedItems: Map> = new Map(); - - constructor(private readonly opts: IndexedGossipQueueOpts) {} - - get length(): number { - return this._length; - } - - get keySize(): number { - return this.indexedItems.size; - } - - clear(): void { - this.indexedItems = new Map(); - this._length = 0; - } - - // not implemented for this gossip queue - getDataAgeMs(): number[] { - return []; - } - - /** - * Add item to gossip queue. If queue is full, drop first item of first key. - * Return number of items dropped - */ - add(item: T): number { - const key = this.opts.indexFn(item); - if (key == null) { - // this comes from getAttDataBase64FromAttestationSerialized() return type - // should not happen - return 0; - } - item.indexed = key; - let list = this.indexedItems.get(key); - if (list == null) { - list = { - items: [], - avgRecvTimestampMs: Date.now(), - }; - this.indexedItems.set(key, list); - } else { - list.avgRecvTimestampMs = (list.avgRecvTimestampMs * list.items.length + Date.now()) / (list.items.length + 1); - list.items.push(item); - } - this._length++; - if (this._length <= this.opts.maxLength) { - return 0; - } - - // overload, need to drop more items - const firstKey = this.indexedItems.keys().next().value as string; - // there should be at least 1 key - if (firstKey == null) { - return 0; - } - const firstList = this.indexedItems.get(firstKey); - // should not happen - if (firstList == null) { - return 0; - } - - const deletedItem = firstList.items.shift(); - if (deletedItem != null) { - this._length--; - if (firstList.items.length === 0) { - this.indexedItems.delete(firstKey); - } - return 1; - } else { - return 0; - } - } - - /** - * Try to get list of items of the same key with highest score - */ - next(): T[] | null { - let maxScore = 0; - let maxScoreKey: string | undefined; - for (const [key, list] of this.indexedItems) { - const score = listScore(list); - if (score > maxScore) { - maxScore = score; - maxScoreKey = key; - } - } - - if (maxScoreKey == null) { - return null; - } - const items = this.indexedItems.get(maxScoreKey)?.items; - if (items == null) { - // should not happen - return null; - } - this.indexedItems.delete(maxScoreKey); - this._length = Math.max(0, this._length - items.length); - return items; - } - - getAll(): T[] { - const items: T[] = []; - for (const list of this.indexedItems.values()) { - items.push(...list.items); - } - return items; - } -} diff --git a/packages/beacon-node/src/network/processor/gossipQueues/types.ts b/packages/beacon-node/src/network/processor/gossipQueues/types.ts index 448f44c3f5d7..58034e82ed06 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues/types.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues/types.ts @@ -1,4 +1,4 @@ -export type GossipQueueOpts = LinearGossipQueueOpts | IndexedGossipQueueOpts | IndexedGossipQueueMinSizeOpts; +export type GossipQueueOpts = LinearGossipQueueOpts | IndexedGossipQueueMinSizeOpts; export type LinearGossipQueueOpts = { type: QueueType; @@ -25,15 +25,6 @@ export function isIndexedGossipQueueMinSizeOpts(opts: GossipQueueOpts): op ); } -export function isIndexedGossipQueueAvgTimeOpts(opts: GossipQueueOpts): opts is IndexedGossipQueueOpts { - const avgTimeOpts = opts as IndexedGossipQueueMinSizeOpts; - return ( - avgTimeOpts.indexFn !== undefined && - avgTimeOpts.minChunkSize === undefined && - avgTimeOpts.maxChunkSize === undefined - ); -} - export interface GossipQueue { length: number; keySize: number; From 5f7b611eeb4adab186919b042bc6422a9ba5aa75 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 7 Oct 2024 20:26:37 +0100 Subject: [PATCH 141/145] feat: add standard endpoint to retrieve fork choice context (#7127) --- packages/api/src/beacon/routes/debug.ts | 53 +++++++++++++++++++ .../api/test/unit/beacon/oapiSpec.test.ts | 1 - .../api/test/unit/beacon/testData/debug.ts | 30 ++++++++++- .../beacon-node/src/api/impl/debug/index.ts | 31 +++++++++++ packages/beacon-node/src/api/rest/base.ts | 2 +- 5 files changed, 114 insertions(+), 3 deletions(-) diff --git a/packages/api/src/beacon/routes/debug.ts b/packages/api/src/beacon/routes/debug.ts index 8099fcac020e..590ecf71dd9c 100644 --- a/packages/api/src/beacon/routes/debug.ts +++ b/packages/api/src/beacon/routes/debug.ts @@ -58,11 +58,34 @@ const DebugChainHeadType = new ContainerType( {jsonCase: "eth2"} ); +const ForkChoiceNodeType = new ContainerType( + { + slot: ssz.Slot, + blockRoot: stringType, + parentRoot: stringType, + justifiedEpoch: ssz.Epoch, + finalizedEpoch: ssz.Epoch, + weight: ssz.UintNum64, + validity: new StringType<"valid" | "invalid" | "optimistic">(), + executionBlockHash: stringType, + }, + {jsonCase: "eth2"} +); +const ForkChoiceResponseType = new ContainerType( + { + justifiedCheckpoint: ssz.phase0.Checkpoint, + finalizedCheckpoint: ssz.phase0.Checkpoint, + forkChoiceNodes: ArrayOf(ForkChoiceNodeType), + }, + {jsonCase: "eth2"} +); + const ProtoNodeListType = ArrayOf(ProtoNodeType); const DebugChainHeadListType = ArrayOf(DebugChainHeadType); type ProtoNodeList = ValueOf; type DebugChainHeadList = ValueOf; +type ForkChoiceResponse = ValueOf; export type Endpoints = { /** @@ -77,6 +100,18 @@ export type Endpoints = { EmptyMeta >; + /** + * Retrieves all current fork choice context + */ + getDebugForkChoice: Endpoint< + // ⏎ + "GET", + EmptyArgs, + EmptyRequest, + ForkChoiceResponse, + EmptyMeta + >; + /** * Dump all ProtoArray's nodes to debug */ @@ -115,6 +150,24 @@ export function getDefinitions(_config: ChainForkConfig): RouteDefinitions ({ + ...(data as ForkChoiceResponse), + }), + fromResponse: (resp) => ({ + data: resp as ForkChoiceResponse, + }), + }, + }, + }, getProtoArrayNodes: { url: "/eth/v0/debug/forkchoice", method: "GET", diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index 89079cd0768c..2b8a8254dd6b 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -57,7 +57,6 @@ const ignoredOperations = [ /* missing route */ "getDepositSnapshot", // Won't fix for now, see https://github.com/ChainSafe/lodestar/issues/5697 "getNextWithdrawals", // https://github.com/ChainSafe/lodestar/issues/5696 - "getDebugForkChoice", // https://github.com/ChainSafe/lodestar/issues/5700 /* Must support ssz response body */ "getLightClientUpdatesByRange", // https://github.com/ChainSafe/lodestar/issues/6841 ]; diff --git a/packages/api/test/unit/beacon/testData/debug.ts b/packages/api/test/unit/beacon/testData/debug.ts index aac3b379ff4d..cb2799939ae3 100644 --- a/packages/api/test/unit/beacon/testData/debug.ts +++ b/packages/api/test/unit/beacon/testData/debug.ts @@ -4,13 +4,41 @@ import {ssz} from "@lodestar/types"; import {Endpoints} from "../../../../src/beacon/routes/debug.js"; import {GenericServerTestCases} from "../../../utils/genericServerTest.js"; -const rootHex = toHexString(Buffer.alloc(32, 1)); +const root = new Uint8Array(32).fill(1); +const rootHex = toHexString(root); export const testData: GenericServerTestCases = { getDebugChainHeadsV2: { args: undefined, res: {data: [{slot: 1, root: rootHex, executionOptimistic: true}]}, }, + getDebugForkChoice: { + args: undefined, + res: { + data: { + justifiedCheckpoint: { + epoch: 2, + root, + }, + finalizedCheckpoint: { + epoch: 1, + root, + }, + forkChoiceNodes: [ + { + slot: 1, + blockRoot: rootHex, + parentRoot: rootHex, + justifiedEpoch: 1, + finalizedEpoch: 1, + weight: 1, + validity: "valid", + executionBlockHash: rootHex, + }, + ], + }, + }, + }, getProtoArrayNodes: { args: undefined, res: { diff --git a/packages/beacon-node/src/api/impl/debug/index.ts b/packages/beacon-node/src/api/impl/debug/index.ts index 4edb8ba9b2dd..e5b6450b206f 100644 --- a/packages/beacon-node/src/api/impl/debug/index.ts +++ b/packages/beacon-node/src/api/impl/debug/index.ts @@ -1,6 +1,8 @@ import {routes} from "@lodestar/api"; import {ApplicationMethods} from "@lodestar/api/server"; +import {ExecutionStatus} from "@lodestar/fork-choice"; import {BeaconState} from "@lodestar/types"; +import {ZERO_HASH_HEX} from "@lodestar/params"; import {getStateResponseWithRegen} from "../beacon/state/utils.js"; import {ApiModules} from "../types.js"; import {isOptimisticBlock} from "../../../util/forkChoice.js"; @@ -22,6 +24,35 @@ export function getDebugApi({ }; }, + async getDebugForkChoice() { + return { + data: { + justifiedCheckpoint: chain.forkChoice.getJustifiedCheckpoint(), + finalizedCheckpoint: chain.forkChoice.getFinalizedCheckpoint(), + forkChoiceNodes: chain.forkChoice.getAllNodes().map((node) => ({ + slot: node.slot, + blockRoot: node.blockRoot, + parentRoot: node.parentRoot, + justifiedEpoch: node.justifiedEpoch, + finalizedEpoch: node.finalizedEpoch, + weight: node.weight, + validity: (() => { + switch (node.executionStatus) { + case ExecutionStatus.Valid: + return "valid"; + case ExecutionStatus.Invalid: + return "invalid"; + case ExecutionStatus.Syncing: + case ExecutionStatus.PreMerge: + return "optimistic"; + } + })(), + executionBlockHash: node.executionPayloadBlockHash ?? ZERO_HASH_HEX, + })), + }, + }; + }, + async getProtoArrayNodes() { const nodes = chain.forkChoice.getAllNodes().map((node) => ({ // if node has executionPayloadNumber, it will overwrite the below default diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index 10318b1b3ba7..276583dc7281 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -96,7 +96,7 @@ export class RestApiServer { if (err.validation) { const {instancePath, message} = err.validation[0]; const payload: ErrorResponse = { - code: err.statusCode ?? 400, + code: 400, message: `${instancePath.substring(instancePath.lastIndexOf("/") + 1)} ${message}`, stacktraces, }; From 1fa3f37ab7a6ea53888650cb23da43456886c935 Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 8 Oct 2024 02:31:39 +0700 Subject: [PATCH 142/145] refactor: remove beaconAttestationBatchValidation flag (#7129) * chore: remove beaconAttestationBatchValidation flag * chore: refactor SequentialGossipHandlers vs BatchGossipHandlers * chore: remove unused validateGossipAttestation() function * chore: lint * chore: fix lint * chore: SequentialGossipType vs BatchGossipType * chore: simplify getGossipHandlers() --- .../src/chain/validation/attestation.ts | 46 ++---------- .../src/network/gossip/interface.ts | 19 ++--- packages/beacon-node/src/network/options.ts | 2 - .../src/network/processor/gossipHandlers.ts | 71 ++----------------- .../network/processor/gossipQueues/index.ts | 26 +++---- .../src/network/processor/index.ts | 2 +- .../perf/chain/validation/attestation.test.ts | 22 +----- .../attestation/validateAttestation.test.ts | 29 ++++---- .../src/options/beaconNodeOptions/network.ts | 9 --- .../unit/options/beaconNodeOptions.test.ts | 2 - 10 files changed, 51 insertions(+), 177 deletions(-) diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index 4ceaf64d658d..119a8ee1a899 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -38,7 +38,6 @@ import {MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC} from "../../constants/index.js"; import {RegenCaller} from "../regen/index.js"; import { getAggregationBitsFromAttestationSerialized, - getAttDataFromAttestationSerialized, getAttDataFromSignedAggregateAndProofElectra, getCommitteeBitsFromAttestationSerialized, getCommitteeBitsFromSignedAggregateAndProofElectra, @@ -75,9 +74,8 @@ export type GossipAttestation = { serializedData: Uint8Array; // available in NetworkProcessor since we check for unknown block root attestations attSlot: Slot; - // for old LIFO linear gossip queue we don't have attDataBase64 // for indexed gossip queue we have attDataBase64 - attDataBase64?: SeenAttDataKey | null; + attDataBase64: SeenAttDataKey; }; export type Step0Result = AttestationValidationResult & { @@ -85,20 +83,6 @@ export type Step0Result = AttestationValidationResult & { validatorIndex: number; }; -/** - * Validate a single gossip attestation, do not prioritize bls signature set - */ -export async function validateGossipAttestation( - fork: ForkName, - chain: IBeaconChain, - attestationOrBytes: GossipAttestation, - /** Optional, to allow verifying attestations through API with unknown subnet */ - subnet: number -): Promise { - const prioritizeBls = false; - return validateAttestation(fork, chain, attestationOrBytes, subnet, prioritizeBls); -} - /** * Verify gossip attestations of the same attestation data. The main advantage is we can batch verify bls signatures * through verifySignatureSetsSameMessage bls api to improve performance. @@ -111,7 +95,7 @@ export async function validateGossipAttestationsSameAttData( attestationOrBytesArr: GossipAttestation[], subnet: number, // for unit test, consumers do not need to pass this - step0ValidationFn = validateGossipAttestationNoSignatureCheck + step0ValidationFn = validateAttestationNoSignatureCheck ): Promise { if (attestationOrBytesArr.length === 0) { return {results: [], batchableBls: false}; @@ -213,22 +197,10 @@ export async function validateApiAttestation( attestationOrBytes: ApiAttestation ): Promise { const prioritizeBls = true; - return validateAttestation(fork, chain, attestationOrBytes, null, prioritizeBls); -} + const subnet = null; -/** - * Validate a single unaggregated attestation - * subnet is null for api attestations - */ -export async function validateAttestation( - fork: ForkName, - chain: IBeaconChain, - attestationOrBytes: AttestationOrBytes, - subnet: number | null, - prioritizeBls = false -): Promise { try { - const step0Result = await validateGossipAttestationNoSignatureCheck(fork, chain, attestationOrBytes, subnet); + const step0Result = await validateAttestationNoSignatureCheck(fork, chain, attestationOrBytes, subnet); const {attestation, signatureSet, validatorIndex} = step0Result; const isValid = await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}); @@ -256,7 +228,7 @@ export async function validateAttestation( * Only deserialize the attestation if needed, use the cached AttestationData instead * This is to avoid deserializing similar attestation multiple times which could help the gc */ -async function validateGossipAttestationNoSignatureCheck( +async function validateAttestationNoSignatureCheck( fork: ForkName, chain: IBeaconChain, attestationOrBytes: AttestationOrBytes, @@ -801,9 +773,6 @@ export function computeSubnetForSlot(shuffling: EpochShuffling, slot: number, co * Return fork-dependent seen attestation key * - for pre-electra, it's the AttestationData base64 * - for electra and later, it's the AttestationData base64 + committeeBits base64 - * - * we always have attDataBase64 from the IndexedGossipQueue, getAttDataFromAttestationSerialized() just for backward compatible when beaconAttestationBatchValidation is false - * TODO: remove beaconAttestationBatchValidation flag since the batch attestation is stable */ export function getSeenAttDataKeyFromGossipAttestation( fork: ForkName, @@ -811,13 +780,12 @@ export function getSeenAttDataKeyFromGossipAttestation( ): SeenAttDataKey | null { const {attDataBase64, serializedData} = attestation; if (isForkPostElectra(fork)) { - const attData = attDataBase64 ?? getAttDataFromAttestationSerialized(serializedData); const committeeBits = getCommitteeBitsFromAttestationSerialized(serializedData); - return attData && committeeBits ? attDataBase64 + committeeBits : null; + return attDataBase64 && committeeBits ? attDataBase64 + committeeBits : null; } // pre-electra - return attDataBase64 ?? getAttDataFromAttestationSerialized(serializedData); + return attDataBase64; } /** diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index ab9b8a65978d..af5e3888d04f 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -36,6 +36,9 @@ export enum GossipType { bls_to_execution_change = "bls_to_execution_change", } +export type SequentialGossipType = Exclude; +export type BatchGossipType = GossipType.beacon_attestation; + export enum GossipEncoding { ssz_snappy = "ssz_snappy", } @@ -181,25 +184,25 @@ export type GossipHandlerParamGeneric = { }; export type GossipHandlers = { - [K in GossipType]: DefaultGossipHandler | BatchGossipHandler; + [K in GossipType]: SequentialGossipHandler | BatchGossipHandler; }; -export type DefaultGossipHandler = ( +export type SequentialGossipHandler = ( gossipHandlerParam: GossipHandlerParamGeneric ) => Promise; -export type DefaultGossipHandlers = { - [K in GossipType]: DefaultGossipHandler; +export type SequentialGossipHandlers = { + [K in SequentialGossipType]: SequentialGossipHandler; +}; + +export type BatchGossipHandlers = { + [K in BatchGossipType]: BatchGossipHandler; }; export type BatchGossipHandler = ( gossipHandlerParams: GossipHandlerParamGeneric[] ) => Promise<(null | GossipActionError)[]>; -export type BatchGossipHandlers = { - [K in GossipType]?: BatchGossipHandler; -}; - // eslint-disable-next-line @typescript-eslint/no-explicit-any export type ResolvedType Promise> = F extends (...args: any) => Promise ? T diff --git a/packages/beacon-node/src/network/options.ts b/packages/beacon-node/src/network/options.ts index d2070873261b..ebb321584d12 100644 --- a/packages/beacon-node/src/network/options.ts +++ b/packages/beacon-node/src/network/options.ts @@ -40,8 +40,6 @@ export const defaultNetworkOptions: NetworkOptions = { maxYoungGenerationSizeMb: 152, // subscribe 2 slots before aggregator dutied slot to get stable mesh peers as monitored on goerli slotsToSubscribeBeforeAggregatorDuty: 2, - // this should only be set to true if useWorker is true - beaconAttestationBatchValidation: true, // This will enable the light client server by default disableLightClientServer: false, }; diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 8df9f7b78126..90c131a943ed 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -20,7 +20,7 @@ import { } from "../../chain/errors/index.js"; import { BatchGossipHandlers, - DefaultGossipHandlers, + SequentialGossipHandlers, GossipHandlerParamGeneric, GossipHandlers, GossipType, @@ -36,8 +36,6 @@ import { validateGossipBlsToExecutionChange, AggregateAndProofValidationResult, validateGossipAttestationsSameAttData, - validateGossipAttestation, - AttestationValidationResult, GossipAttestation, } from "../../chain/validation/index.js"; import {NetworkEvent, NetworkEventBus} from "../events.js"; @@ -64,8 +62,6 @@ import {AggregatorTracker} from "./aggregatorTracker.js"; export type GossipHandlerOpts = { /** By default pass gossip attestations to forkchoice */ dontSendGossipAttestationsToForkchoice?: boolean; - /** By default don't validate gossip attestations in batch */ - beaconAttestationBatchValidation?: boolean; }; export type ValidatorFnsModules = { @@ -96,20 +92,15 @@ const BLOCK_AVAILABILITY_CUTOFF_MS = 3_000; * - Ethereum Consensus gossipsub protocol strictly defined a single topic for message */ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): GossipHandlers { - const defaultHandlers = getDefaultHandlers(modules, options); - if (options.beaconAttestationBatchValidation) { - const batchHandlers = getBatchHandlers(modules, options); - return {...defaultHandlers, ...batchHandlers}; - } - return defaultHandlers; + return {...getSequentialHandlers(modules, options), ...getBatchHandlers(modules, options)}; } /** * Default handlers validate gossip messages one by one. * We only have a choice to do batch validation for beacon_attestation topic. */ -function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): DefaultGossipHandlers { - const {chain, config, metrics, events, logger, core, aggregatorTracker} = modules; +function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): SequentialGossipHandlers { + const {chain, config, metrics, events, logger, core} = modules; async function validateBeaconBlock( signedBlock: SignedBeaconBlock, @@ -458,58 +449,6 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler chain.emitter.emit(routes.events.EventType.attestation, signedAggregateAndProof.message.aggregate); }, - [GossipType.beacon_attestation]: async ({ - gossipData, - topic, - seenTimestampSec, - }: GossipHandlerParamGeneric): Promise => { - const {serializedData, msgSlot} = gossipData; - if (msgSlot == undefined) { - throw Error("msgSlot is undefined for beacon_attestation topic"); - } - const {subnet, fork} = topic; - - // do not deserialize gossipSerializedData here, it's done in validateGossipAttestation only if needed - let validationResult: AttestationValidationResult; - try { - validationResult = await validateGossipAttestation( - fork, - chain, - {attestation: null, serializedData, attSlot: msgSlot}, - subnet - ); - } catch (e) { - if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - chain.persistInvalidSszBytes(ssz.phase0.Attestation.typeName, serializedData, "gossip_reject"); - } - throw e; - } - - // Handler - const {indexedAttestation, attDataRootHex, attestation, committeeIndex} = validationResult; - metrics?.registerGossipUnaggregatedAttestation(seenTimestampSec, indexedAttestation); - - try { - // Node may be subscribe to extra subnets (long-lived random subnets). For those, validate the messages - // but don't add to attestation pool, to save CPU and RAM - if (aggregatorTracker.shouldAggregate(subnet, indexedAttestation.data.slot)) { - const insertOutcome = chain.attestationPool.add(committeeIndex, attestation, attDataRootHex); - metrics?.opPool.attestationPoolInsertOutcome.inc({insertOutcome}); - } - } catch (e) { - logger.error("Error adding unaggregated attestation to pool", {subnet}, e as Error); - } - - if (!options.dontSendGossipAttestationsToForkchoice) { - try { - chain.forkChoice.onAttestation(indexedAttestation, attDataRootHex); - } catch (e) { - logger.debug("Error adding gossip unaggregated attestation to forkchoice", {subnet}, e as Error); - } - } - - chain.emitter.emit(routes.events.EventType.attestation, attestation); - }, [GossipType.attester_slashing]: async ({ gossipData, @@ -660,7 +599,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler /** * For now, only beacon_attestation topic is batched. */ -function getBatchHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): Partial { +function getBatchHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): BatchGossipHandlers { const {chain, metrics, logger, aggregatorTracker} = modules; return { [GossipType.beacon_attestation]: async ( diff --git a/packages/beacon-node/src/network/processor/gossipQueues/index.ts b/packages/beacon-node/src/network/processor/gossipQueues/index.ts index 968c11501426..12596a42b7a1 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues/index.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues/index.ts @@ -1,5 +1,5 @@ import {mapValues} from "@lodestar/utils"; -import {GossipType} from "../../gossip/interface.js"; +import {BatchGossipType, GossipType, SequentialGossipType} from "../../gossip/interface.js"; import {PendingGossipsubMessage} from "../types.js"; import {getGossipAttestationIndex} from "../../../util/sszBytes.js"; import {LinearGossipQueue} from "./linear.js"; @@ -20,8 +20,8 @@ export const MIN_SIGNATURE_SETS_TO_BATCH_VERIFY = 32; /** * Numbers from https://github.com/sigp/lighthouse/blob/b34a79dc0b02e04441ba01fd0f304d1e203d877d/beacon_node/network/src/beacon_processor/mod.rs#L69 */ -const defaultGossipQueueOpts: { - [K in GossipType]: GossipQueueOpts; +const linearGossipQueueOpts: { + [K in SequentialGossipType]: GossipQueueOpts; } = { // validation gossip block asap [GossipType.beacon_block]: {maxLength: 1024, type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}}, @@ -37,15 +37,6 @@ const defaultGossipQueueOpts: { type: QueueType.LIFO, dropOpts: {type: DropType.count, count: 1}, }, - // lighthouse has attestation_queue 16384 and unknown_block_attestation_queue 8192, we use single queue - // this topic may cause node to be overload and drop 100% of lower priority queues - // so we want to drop it by ratio until node is stable enough (queue is empty) - // start with dropping 1% of the queue, then increase 1% more each time. Reset when queue is empty - [GossipType.beacon_attestation]: { - maxLength: 24576, - type: QueueType.LIFO, - dropOpts: {type: DropType.ratio, start: 0.01, step: 0.01}, - }, [GossipType.voluntary_exit]: {maxLength: 4096, type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}}, [GossipType.proposer_slashing]: {maxLength: 4096, type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}}, [GossipType.attester_slashing]: {maxLength: 4096, type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}}, @@ -74,9 +65,11 @@ const defaultGossipQueueOpts: { }; const indexedGossipQueueOpts: { - [K in GossipType]?: GossipQueueOpts; + [K in BatchGossipType]: GossipQueueOpts; } = { [GossipType.beacon_attestation]: { + // lighthouse has attestation_queue 16384 and unknown_block_attestation_queue 8192, we use single queue + // this topic may cause node to be overload and drop 100% of lower priority queues maxLength: 24576, indexFn: (item: PendingGossipsubMessage) => { // Note indexFn is fork agnostic despite changes introduced in Electra @@ -103,12 +96,11 @@ const indexedGossipQueueOpts: { * By topic is too specific, so by type groups all similar objects in the same queue. All in the same won't allow * to customize different queue behaviours per object type (see `gossipQueueOpts`). */ -export function createGossipQueues(beaconAttestationBatchValidation = false): { +export function createGossipQueues(): { [K in GossipType]: GossipQueue; } { - const gossipQueueOpts = beaconAttestationBatchValidation - ? {...defaultGossipQueueOpts, ...indexedGossipQueueOpts} - : defaultGossipQueueOpts; + const gossipQueueOpts = {...linearGossipQueueOpts, ...indexedGossipQueueOpts}; + return mapValues(gossipQueueOpts, (opts) => { if (isIndexedGossipQueueMinSizeOpts(opts)) { return new IndexedGossipQueueMinSize(opts); diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index 9a1dcfb32fa0..5cfed6c20346 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -172,7 +172,7 @@ export class NetworkProcessor { this.metrics = metrics; this.logger = logger; this.events = events; - this.gossipQueues = createGossipQueues(this.opts.beaconAttestationBatchValidation); + this.gossipQueues = createGossipQueues(); this.gossipTopicConcurrency = mapValues(this.gossipQueues, () => 0); this.gossipValidatorFn = getGossipValidatorFn(modules.gossipHandlers ?? getGossipHandlers(modules, opts), modules); this.gossipValidatorBatchFn = getGossipValidatorBatchFn( diff --git a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts index f285317474d6..e942e1ea17d0 100644 --- a/packages/beacon-node/test/perf/chain/validation/attestation.test.ts +++ b/packages/beacon-node/test/perf/chain/validation/attestation.test.ts @@ -3,7 +3,7 @@ import {expect} from "chai"; import {ssz} from "@lodestar/types"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../state-transition/test/perf/util.js"; -import {validateAttestation, validateGossipAttestationsSameAttData} from "../../../../src/chain/validation/index.js"; +import {validateGossipAttestationsSameAttData} from "../../../../src/chain/validation/index.js"; import {getAttestationValidData} from "../../../utils/validationData/attestation.js"; import {getAttDataFromAttestationSerialized} from "../../../../src/util/sszBytes.js"; @@ -29,25 +29,7 @@ describe("validate gossip attestation", () => { }); const attSlot = attestation0.data.slot; - const serializedData = ssz.phase0.Attestation.serialize(attestation0); const fork = chain.config.getForkName(stateSlot); - itBench({ - id: `validate gossip attestation - vc ${vc}`, - beforeEach: () => chain.seenAttesters["validatorIndexesByEpoch"].clear(), - fn: async () => { - await validateAttestation( - fork, - chain, - { - attestation: null, - serializedData, - attSlot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), - }, - subnet0 - ); - }, - }); for (const chunkSize of [32, 64, 128, 256]) { const attestations = [attestation0]; @@ -67,7 +49,7 @@ describe("validate gossip attestation", () => { attestation: null, serializedData, attSlot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }; }); diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts index 45d293ffbb33..90d37a74289d 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateAttestation.test.ts @@ -2,6 +2,7 @@ import {BitArray} from "@chainsafe/ssz"; import {describe, expect, it} from "vitest"; import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; +import {LodestarError} from "@lodestar/utils"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../../../state-transition/test/perf/util.js"; import {AttestationErrorCode, GossipErrorCode} from "../../../../../src/chain/errors/index.js"; @@ -12,7 +13,7 @@ import { getSeenAttDataKeyFromGossipAttestation, getSeenAttDataKeyFromSignedAggregateAndProof, validateApiAttestation, - validateAttestation, + validateGossipAttestationsSameAttData, } from "../../../../../src/chain/validation/index.js"; import {getAttDataFromAttestationSerialized} from "../../../../../src/util/sszBytes.js"; import {memoOnce} from "../../../../utils/cache.js"; @@ -75,7 +76,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.BAD_TARGET_EPOCH @@ -94,7 +95,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.PAST_SLOT @@ -113,7 +114,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.FUTURE_SLOT @@ -138,7 +139,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -158,7 +159,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.NOT_EXACTLY_ONE_AGGREGATION_BIT_SET @@ -182,7 +183,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT @@ -202,7 +203,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.INVALID_TARGET_ROOT @@ -229,7 +230,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.WRONG_NUMBER_OF_AGGREGATION_BITS @@ -248,7 +249,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, invalidSubnet, AttestationErrorCode.INVALID_SUBNET_ID @@ -268,7 +269,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.ATTESTATION_ALREADY_KNOWN @@ -290,7 +291,7 @@ describe("validateAttestation", () => { attestation: null, serializedData, attSlot: attestation.data.slot, - attDataBase64: getAttDataFromAttestationSerialized(serializedData), + attDataBase64: getAttDataFromAttestationSerialized(serializedData) as string, }, subnet, AttestationErrorCode.INVALID_SIGNATURE @@ -314,7 +315,9 @@ describe("validateAttestation", () => { errorCode: string ): Promise { const fork = chain.config.getForkName(stateSlot); - await expectRejectedWithLodestarError(validateAttestation(fork, chain, attestationOrBytes, subnet), errorCode); + const {results} = await validateGossipAttestationsSameAttData(fork, chain, [attestationOrBytes], subnet); + expect(results.length).toEqual(1); + expect((results[0].err as LodestarError<{code: string}>).type.code).toEqual(errorCode); } }); diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 25ba036a5dbf..bfe9c7710e86 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -26,7 +26,6 @@ export type NetworkArgs = { "network.connectToDiscv5Bootnodes"?: boolean; "network.discv5FirstQueryDelayMs"?: number; "network.dontSendGossipAttestationsToForkchoice"?: boolean; - "network.beaconAttestationBatchValidation"?: boolean; "network.allowPublishToZeroPeers"?: boolean; "network.gossipsubD"?: number; "network.gossipsubDLow"?: number; @@ -144,7 +143,6 @@ export function parseArgs(args: NetworkArgs): IBeaconNodeOptions["network"] { connectToDiscv5Bootnodes: args["network.connectToDiscv5Bootnodes"], discv5FirstQueryDelayMs: args["network.discv5FirstQueryDelayMs"], dontSendGossipAttestationsToForkchoice: args["network.dontSendGossipAttestationsToForkchoice"], - beaconAttestationBatchValidation: args["network.beaconAttestationBatchValidation"], allowPublishToZeroPeers: args["network.allowPublishToZeroPeers"], gossipsubD: args["network.gossipsubD"], gossipsubDLow: args["network.gossipsubDLow"], @@ -321,13 +319,6 @@ export const options: CliCommandOptions = { group: "network", }, - "network.beaconAttestationBatchValidation": { - hidden: true, - type: "boolean", - description: "Validate gossip attestations in batches", - group: "network", - }, - "network.allowPublishToZeroPeers": { hidden: true, type: "boolean", diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index f873102edf74..879b5bfa2fc9 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -95,7 +95,6 @@ describe("options / beaconNodeOptions", () => { "network.blockCountPeerLimit": 500, "network.rateTrackerTimeoutMs": 60000, "network.dontSendGossipAttestationsToForkchoice": true, - "network.beaconAttestationBatchValidation": true, "network.allowPublishToZeroPeers": true, "network.gossipsubD": 4, "network.gossipsubDLow": 2, @@ -206,7 +205,6 @@ describe("options / beaconNodeOptions", () => { connectToDiscv5Bootnodes: true, discv5FirstQueryDelayMs: 1000, dontSendGossipAttestationsToForkchoice: true, - beaconAttestationBatchValidation: true, allowPublishToZeroPeers: true, gossipsubD: 4, gossipsubDLow: 2, From c04157c9696ce7b9b12518f61ec638b872927c4b Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Tue, 8 Oct 2024 16:01:47 +0100 Subject: [PATCH 143/145] fix: ensure generated historical state slot matches requested slot (#7135) --- .../src/chain/historicalState/getHistoricalState.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts b/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts index 1f352b3d683a..a9f254dea3f2 100644 --- a/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts +++ b/packages/beacon-node/src/chain/historicalState/getHistoricalState.ts @@ -102,6 +102,10 @@ export async function getHistoricalState( metrics?.stateTransitionBlocks.observe(blockCount); transitionTimer?.(); + if (state.slot !== slot) { + throw Error(`Failed to generate historical state for slot ${slot}`); + } + const serializeTimer = metrics?.stateSerializationTime.startTimer(); const stateBytes = state.serialize(); serializeTimer?.(); From 955f4569289d66e0a2ee171dfa7bd9984ce57ba7 Mon Sep 17 00:00:00 2001 From: Phil Ngo <58080811+philknows@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:04:02 -0400 Subject: [PATCH 144/145] chore: add drips network ownership address (#7128) add drips network ownership address --- funding.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/funding.json b/funding.json index 872321cdca5b..3ff79f54b693 100644 --- a/funding.json +++ b/funding.json @@ -1,5 +1,10 @@ { "opRetro": { "projectId": "0x8ec88058175ef4c1c9b1f26910c4d4f2cfa733d6fcd1dbd9385476a313d9e12d" + }, + "drips": { + "ethereum": { + "ownedBy": "0x94107e24Ba695aeb884fe9e896BA0Bbc14D3B509" + } } } From 068fbae928989295706443887c645d476bb95695 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Wed, 9 Oct 2024 09:43:09 +0100 Subject: [PATCH 145/145] refactor: move validator status type and util to @lodestar/types (#7140) --- .../api/src/beacon/routes/beacon/state.ts | 14 +-- .../src/api/impl/beacon/state/index.ts | 9 +- .../src/api/impl/beacon/state/utils.ts | 36 +------ .../src/api/impl/validator/index.ts | 2 +- .../unit/api/impl/beacon/state/utils.test.ts | 100 +----------------- packages/types/src/index.ts | 1 + packages/types/src/utils/validatorStatus.ts | 46 ++++++++ .../types/test/unit/validatorStatus.test.ts | 100 ++++++++++++++++++ 8 files changed, 155 insertions(+), 153 deletions(-) create mode 100644 packages/types/src/utils/validatorStatus.ts create mode 100644 packages/types/test/unit/validatorStatus.test.ts diff --git a/packages/api/src/beacon/routes/beacon/state.ts b/packages/api/src/beacon/routes/beacon/state.ts index e3cfe2e16edf..1489f48c297e 100644 --- a/packages/api/src/beacon/routes/beacon/state.ts +++ b/packages/api/src/beacon/routes/beacon/state.ts @@ -2,7 +2,7 @@ import {ContainerType, ValueOf} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; -import {phase0, CommitteeIndex, Slot, Epoch, ssz, RootHex, StringType} from "@lodestar/types"; +import {phase0, CommitteeIndex, Slot, Epoch, ssz, RootHex, StringType, ValidatorStatus} from "@lodestar/types"; import {Endpoint, RequestCodec, RouteDefinitions, Schema} from "../../../utils/index.js"; import {ArrayOf, JsonOnlyReq} from "../../../utils/codecs.js"; import {ExecutionOptimisticAndFinalizedCodec, ExecutionOptimisticAndFinalizedMeta} from "../../../utils/metadata.js"; @@ -24,17 +24,7 @@ export type StateArgs = { export type ValidatorId = string | number; -export type ValidatorStatus = - | "active" - | "pending_initialized" - | "pending_queued" - | "active_ongoing" - | "active_exiting" - | "active_slashed" - | "exited_unslashed" - | "exited_slashed" - | "withdrawal_possible" - | "withdrawal_done"; +export type {ValidatorStatus}; export const RandaoResponseType = new ContainerType({ randao: ssz.Root, diff --git a/packages/beacon-node/src/api/impl/beacon/state/index.ts b/packages/beacon-node/src/api/impl/beacon/state/index.ts index 669aaa9d1fb3..2bf758a8e286 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/index.ts @@ -9,16 +9,11 @@ import { getRandaoMix, } from "@lodestar/state-transition"; import {EPOCHS_PER_HISTORICAL_VECTOR} from "@lodestar/params"; +import {getValidatorStatus} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {ApiError} from "../../errors.js"; import {ApiModules} from "../../types.js"; -import { - filterStateValidatorsByStatus, - getStateValidatorIndex, - getValidatorStatus, - getStateResponse, - toValidatorResponse, -} from "./utils.js"; +import {filterStateValidatorsByStatus, getStateValidatorIndex, getStateResponse, toValidatorResponse} from "./utils.js"; export function getBeaconStateApi({ chain, diff --git a/packages/beacon-node/src/api/impl/beacon/state/utils.ts b/packages/beacon-node/src/api/impl/beacon/state/utils.ts index de21c6a244ed..5e9d3be01221 100644 --- a/packages/beacon-node/src/api/impl/beacon/state/utils.ts +++ b/packages/beacon-node/src/api/impl/beacon/state/utils.ts @@ -1,8 +1,8 @@ import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map"; import {routes} from "@lodestar/api"; -import {FAR_FUTURE_EPOCH, GENESIS_SLOT} from "@lodestar/params"; +import {GENESIS_SLOT} from "@lodestar/params"; import {BeaconStateAllForks} from "@lodestar/state-transition"; -import {BLSPubkey, Epoch, phase0, RootHex, Slot, ValidatorIndex} from "@lodestar/types"; +import {BLSPubkey, Epoch, getValidatorStatus, phase0, RootHex, Slot, ValidatorIndex} from "@lodestar/types"; import {fromHex} from "@lodestar/utils"; import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice"; import {IBeaconChain} from "../../../../chain/index.js"; @@ -83,38 +83,6 @@ export async function getStateResponseWithRegen( return res; } -/** - * Get the status of the validator - * based on conditions outlined in https://hackmd.io/ofFJ5gOmQpu1jjHilHbdQQ - */ -export function getValidatorStatus(validator: phase0.Validator, currentEpoch: Epoch): routes.beacon.ValidatorStatus { - // pending - if (validator.activationEpoch > currentEpoch) { - if (validator.activationEligibilityEpoch === FAR_FUTURE_EPOCH) { - return "pending_initialized"; - } else if (validator.activationEligibilityEpoch < FAR_FUTURE_EPOCH) { - return "pending_queued"; - } - } - // active - if (validator.activationEpoch <= currentEpoch && currentEpoch < validator.exitEpoch) { - if (validator.exitEpoch === FAR_FUTURE_EPOCH) { - return "active_ongoing"; - } else if (validator.exitEpoch < FAR_FUTURE_EPOCH) { - return validator.slashed ? "active_slashed" : "active_exiting"; - } - } - // exited - if (validator.exitEpoch <= currentEpoch && currentEpoch < validator.withdrawableEpoch) { - return validator.slashed ? "exited_slashed" : "exited_unslashed"; - } - // withdrawal - if (validator.withdrawableEpoch <= currentEpoch) { - return validator.effectiveBalance !== 0 ? "withdrawal_possible" : "withdrawal_done"; - } - throw new Error("ValidatorStatus unknown"); -} - export function toValidatorResponse( index: ValidatorIndex, validator: phase0.Validator, diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 2347d7086d46..8a500125ad06 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -41,6 +41,7 @@ import { BeaconBlock, BlockContents, BlindedBeaconBlock, + getValidatorStatus, } from "@lodestar/types"; import {ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {fromHex, toHex, resolveOrRacePromises, prettyWeiToEth, toRootHex} from "@lodestar/utils"; @@ -61,7 +62,6 @@ import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/va import {CommitteeSubscription} from "../../../network/subnets/index.js"; import {ApiModules} from "../types.js"; import {RegenCaller} from "../../../chain/regen/index.js"; -import {getValidatorStatus} from "../beacon/state/utils.js"; import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js"; import {SCHEDULER_LOOKAHEAD_FACTOR} from "../../../chain/prepareNextSlot.js"; import {ChainEvent, CheckpointHex, CommonBlockBody} from "../../../chain/index.js"; diff --git a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts index 79a8f3383e5d..39c936c0d025 100644 --- a/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts +++ b/packages/beacon-node/test/unit/api/impl/beacon/state/utils.test.ts @@ -1,107 +1,9 @@ import {describe, it, expect} from "vitest"; import {toHexString} from "@chainsafe/ssz"; -import {phase0} from "@lodestar/types"; -import {getValidatorStatus, getStateValidatorIndex} from "../../../../../../src/api/impl/beacon/state/utils.js"; +import {getStateValidatorIndex} from "../../../../../../src/api/impl/beacon/state/utils.js"; import {generateCachedAltairState} from "../../../../../utils/state.js"; describe("beacon state api utils", function () { - describe("getValidatorStatus", function () { - it("should return PENDING_INITIALIZED", function () { - const validator = { - activationEpoch: 1, - activationEligibilityEpoch: Infinity, - } as phase0.Validator; - const currentEpoch = 0; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("pending_initialized"); - }); - it("should return PENDING_QUEUED", function () { - const validator = { - activationEpoch: 1, - activationEligibilityEpoch: 101010101101010, - } as phase0.Validator; - const currentEpoch = 0; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("pending_queued"); - }); - it("should return ACTIVE_ONGOING", function () { - const validator = { - activationEpoch: 1, - exitEpoch: Infinity, - } as phase0.Validator; - const currentEpoch = 1; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("active_ongoing"); - }); - it("should return ACTIVE_SLASHED", function () { - const validator = { - activationEpoch: 1, - exitEpoch: 101010101101010, - slashed: true, - } as phase0.Validator; - const currentEpoch = 1; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("active_slashed"); - }); - it("should return ACTIVE_EXITING", function () { - const validator = { - activationEpoch: 1, - exitEpoch: 101010101101010, - slashed: false, - } as phase0.Validator; - const currentEpoch = 1; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("active_exiting"); - }); - it("should return EXITED_SLASHED", function () { - const validator = { - exitEpoch: 1, - withdrawableEpoch: 3, - slashed: true, - } as phase0.Validator; - const currentEpoch = 2; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("exited_slashed"); - }); - it("should return EXITED_UNSLASHED", function () { - const validator = { - exitEpoch: 1, - withdrawableEpoch: 3, - slashed: false, - } as phase0.Validator; - const currentEpoch = 2; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("exited_unslashed"); - }); - it("should return WITHDRAWAL_POSSIBLE", function () { - const validator = { - withdrawableEpoch: 1, - effectiveBalance: 32, - } as phase0.Validator; - const currentEpoch = 1; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("withdrawal_possible"); - }); - it("should return WITHDRAWAL_DONE", function () { - const validator = { - withdrawableEpoch: 1, - effectiveBalance: 0, - } as phase0.Validator; - const currentEpoch = 1; - const status = getValidatorStatus(validator, currentEpoch); - expect(status).toBe("withdrawal_done"); - }); - it("should error", function () { - const validator = {} as phase0.Validator; - const currentEpoch = 0; - try { - getValidatorStatus(validator, currentEpoch); - } catch (error) { - expect(error).toHaveProperty("message", "ValidatorStatus unknown"); - } - }); - }); - describe("getStateValidatorIndex", () => { const state = generateCachedAltairState(); const pubkey2index = state.epochCtx.pubkey2index; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index e0745834c7d1..dc9139dc967e 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -9,3 +9,4 @@ export * from "./utils/typeguards.js"; export {StringType, stringType} from "./utils/stringType.js"; // Container utils export * from "./utils/container.js"; +export * from "./utils/validatorStatus.js"; diff --git a/packages/types/src/utils/validatorStatus.ts b/packages/types/src/utils/validatorStatus.ts new file mode 100644 index 000000000000..e14a4b14c412 --- /dev/null +++ b/packages/types/src/utils/validatorStatus.ts @@ -0,0 +1,46 @@ +import {FAR_FUTURE_EPOCH} from "@lodestar/params"; +import {Epoch, phase0} from "../types.js"; + +export type ValidatorStatus = + | "active" + | "pending_initialized" + | "pending_queued" + | "active_ongoing" + | "active_exiting" + | "active_slashed" + | "exited_unslashed" + | "exited_slashed" + | "withdrawal_possible" + | "withdrawal_done"; + +/** + * Get the status of the validator + * based on conditions outlined in https://hackmd.io/ofFJ5gOmQpu1jjHilHbdQQ + */ +export function getValidatorStatus(validator: phase0.Validator, currentEpoch: Epoch): ValidatorStatus { + // pending + if (validator.activationEpoch > currentEpoch) { + if (validator.activationEligibilityEpoch === FAR_FUTURE_EPOCH) { + return "pending_initialized"; + } else if (validator.activationEligibilityEpoch < FAR_FUTURE_EPOCH) { + return "pending_queued"; + } + } + // active + if (validator.activationEpoch <= currentEpoch && currentEpoch < validator.exitEpoch) { + if (validator.exitEpoch === FAR_FUTURE_EPOCH) { + return "active_ongoing"; + } else if (validator.exitEpoch < FAR_FUTURE_EPOCH) { + return validator.slashed ? "active_slashed" : "active_exiting"; + } + } + // exited + if (validator.exitEpoch <= currentEpoch && currentEpoch < validator.withdrawableEpoch) { + return validator.slashed ? "exited_slashed" : "exited_unslashed"; + } + // withdrawal + if (validator.withdrawableEpoch <= currentEpoch) { + return validator.effectiveBalance !== 0 ? "withdrawal_possible" : "withdrawal_done"; + } + throw new Error("ValidatorStatus unknown"); +} diff --git a/packages/types/test/unit/validatorStatus.test.ts b/packages/types/test/unit/validatorStatus.test.ts new file mode 100644 index 000000000000..8d04c0f98e3d --- /dev/null +++ b/packages/types/test/unit/validatorStatus.test.ts @@ -0,0 +1,100 @@ +import {describe, it, expect} from "vitest"; +import {getValidatorStatus} from "../../src/utils/validatorStatus.js"; +import {phase0} from "../../src/types.js"; + +describe("getValidatorStatus", function () { + it("should return PENDING_INITIALIZED", function () { + const validator = { + activationEpoch: 1, + activationEligibilityEpoch: Infinity, + } as phase0.Validator; + const currentEpoch = 0; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("pending_initialized"); + }); + it("should return PENDING_QUEUED", function () { + const validator = { + activationEpoch: 1, + activationEligibilityEpoch: 101010101101010, + } as phase0.Validator; + const currentEpoch = 0; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("pending_queued"); + }); + it("should return ACTIVE_ONGOING", function () { + const validator = { + activationEpoch: 1, + exitEpoch: Infinity, + } as phase0.Validator; + const currentEpoch = 1; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("active_ongoing"); + }); + it("should return ACTIVE_SLASHED", function () { + const validator = { + activationEpoch: 1, + exitEpoch: 101010101101010, + slashed: true, + } as phase0.Validator; + const currentEpoch = 1; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("active_slashed"); + }); + it("should return ACTIVE_EXITING", function () { + const validator = { + activationEpoch: 1, + exitEpoch: 101010101101010, + slashed: false, + } as phase0.Validator; + const currentEpoch = 1; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("active_exiting"); + }); + it("should return EXITED_SLASHED", function () { + const validator = { + exitEpoch: 1, + withdrawableEpoch: 3, + slashed: true, + } as phase0.Validator; + const currentEpoch = 2; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("exited_slashed"); + }); + it("should return EXITED_UNSLASHED", function () { + const validator = { + exitEpoch: 1, + withdrawableEpoch: 3, + slashed: false, + } as phase0.Validator; + const currentEpoch = 2; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("exited_unslashed"); + }); + it("should return WITHDRAWAL_POSSIBLE", function () { + const validator = { + withdrawableEpoch: 1, + effectiveBalance: 32, + } as phase0.Validator; + const currentEpoch = 1; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("withdrawal_possible"); + }); + it("should return WITHDRAWAL_DONE", function () { + const validator = { + withdrawableEpoch: 1, + effectiveBalance: 0, + } as phase0.Validator; + const currentEpoch = 1; + const status = getValidatorStatus(validator, currentEpoch); + expect(status).toBe("withdrawal_done"); + }); + it("should error", function () { + const validator = {} as phase0.Validator; + const currentEpoch = 0; + try { + getValidatorStatus(validator, currentEpoch); + } catch (error) { + expect(error).toHaveProperty("message", "ValidatorStatus unknown"); + } + }); +});