diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 2b41a6aa4ab3..1b2d0f10a85b 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -74,9 +74,11 @@ export function getBeaconBlockApi({ const slot = signedBlock.message.slot; const fork = config.getForkName(slot); const blockRoot = toHex(chain.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); + // bodyRoot should be the same to produced block + const bodyRoot = toHex(chain.config.getForkTypes(slot).BeaconBlockBody.hashTreeRoot(signedBlock.message.body)); const blockLocallyProduced = chain.producedBlockRoot.has(blockRoot) || chain.producedBlindedBlockRoot.has(blockRoot); - const valLogMeta = {broadcastValidation, blockRoot, blockLocallyProduced, slot}; + const valLogMeta = {broadcastValidation, blockRoot, bodyRoot, blockLocallyProduced, slot}; switch (broadcastValidation) { case routes.beacon.BroadcastValidation.gossip: { @@ -85,6 +87,11 @@ export function getBeaconBlockApi({ await validateGossipBlock(config, chain, signedBlock, fork); } catch (error) { chain.logger.error("Gossip validations failed while publishing the block", valLogMeta, error as Error); + chain.persistInvalidSszValue( + chain.config.getForkTypes(slot).SignedBeaconBlock, + signedBlock, + "api_reject_gossip_failure" + ); throw error; } } @@ -102,6 +109,11 @@ export function getBeaconBlockApi({ blockInput: blockForImport, peer: IDENTITY_PEER_ID, }); + chain.persistInvalidSszValue( + chain.config.getForkTypes(slot).SignedBeaconBlock, + signedBlock, + "api_reject_parent_unknown" + ); throw new BlockError(signedBlock, { code: BlockErrorCode.PARENT_UNKNOWN, parentRoot: toHexString(signedBlock.message.parentRoot), @@ -123,6 +135,11 @@ export function getBeaconBlockApi({ ); } catch (error) { chain.logger.error("Consensus checks failed while publishing the block", valLogMeta, error as Error); + chain.persistInvalidSszValue( + chain.config.getForkTypes(slot).SignedBeaconBlock, + signedBlock, + "api_reject_consensus_failure" + ); throw error; } } diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 68a7049cf9e9..e41c4c97bc63 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -309,6 +309,9 @@ export function getValidatorApi({ }); const version = config.getForkName(block.slot); + if (chain.opts.persistProducedBlocks) { + void chain.persistBlock(block, "produced_builder_block"); + } if (isForkBlobs(version)) { const blockHash = toHex((block as bellatrix.BlindedBeaconBlock).body.executionPayloadHeader.blockHash); const blindedBlobSidecars = chain.producedBlindedBlobSidecarsCache.get(blockHash); @@ -377,6 +380,9 @@ export function getValidatorApi({ executionPayloadValue, root: toHexString(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block)), }); + if (chain.opts.persistProducedBlocks) { + void chain.persistBlock(block, "produced_engine_block"); + } if (isForkBlobs(version)) { const blockHash = toHex((block as bellatrix.BeaconBlock).body.executionPayload.blockHash); const blobSidecars = chain.producedBlobSidecarsCache.get(blockHash); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index f10eec876b11..56409dc89d21 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -25,6 +25,7 @@ import { deneb, Wei, bellatrix, + isBlindedBeaconBlock, } from "@lodestar/types"; import {CheckpointWithHex, ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; import {ProcessShutdownCallback} from "@lodestar/validator"; @@ -637,21 +638,32 @@ export class BeaconChain implements IBeaconChain { return this.reprocessController.waitForBlockOfAttestation(slot, root); } + persistBlock(data: allForks.BeaconBlock | allForks.BlindedBeaconBlock, suffix?: string): void { + const slot = data.slot; + if (isBlindedBeaconBlock(data)) { + const sszType = this.config.getBlindedForkTypes(slot).BeaconBlock; + void this.persistSszObject("BlindedBeaconBlock", sszType.serialize(data), sszType.hashTreeRoot(data), suffix); + } else { + const sszType = this.config.getForkTypes(slot).BeaconBlock; + void this.persistSszObject("BeaconBlock", sszType.serialize(data), sszType.hashTreeRoot(data), suffix); + } + } + persistInvalidSszValue(type: Type, sszObject: T, suffix?: string): void { if (this.opts.persistInvalidSszObjects) { - void this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); + void this.persistSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); } } persistInvalidSszBytes(typeName: string, sszBytes: Uint8Array, suffix?: string): void { if (this.opts.persistInvalidSszObjects) { - void this.persistInvalidSszObject(typeName, sszBytes, sszBytes, suffix); + void this.persistSszObject(typeName, sszBytes, sszBytes, suffix); } } persistInvalidSszView(view: TreeView, suffix?: string): void { if (this.opts.persistInvalidSszObjects) { - void this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); + void this.persistSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); } } @@ -744,16 +756,12 @@ export class BeaconChain implements IBeaconChain { return {state: blockState, stateId: "block_state_any_epoch", shouldWarn: true}; } - private async persistInvalidSszObject( + private async persistSszObject( typeName: string, bytes: Uint8Array, root: Uint8Array, suffix?: string ): Promise { - if (!this.opts.persistInvalidSszObjects) { - return; - } - const now = new Date(); // yyyy-MM-dd const dateStr = now.toISOString().split("T")[0]; diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 8d6f7f419d7b..59beb122a212 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -156,6 +156,7 @@ export interface IBeaconChain { updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise; + persistBlock(data: allForks.BeaconBlock | allForks.BlindedBeaconBlock, suffix?: string): void; persistInvalidSszValue(type: Type, sszObject: T | Uint8Array, suffix?: string): void; persistInvalidSszBytes(type: string, sszBytes: Uint8Array, suffix?: string): void; /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */ diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 74a9c4430980..d71cd55df673 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -12,6 +12,7 @@ export type IChainOptions = BlockProcessOpts & LightClientServerOpts & { blsVerifyAllMainThread?: boolean; blsVerifyAllMultiThread?: boolean; + persistProducedBlocks?: boolean; persistInvalidSszObjects?: boolean; persistInvalidSszObjectsDir?: string; skipCreateStateCacheIfAvailable?: boolean; diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index 359b77740b00..5aef805f61f2 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -7,6 +7,7 @@ export type ChainArgs = { "chain.blsVerifyAllMultiThread"?: boolean; "chain.blsVerifyAllMainThread"?: boolean; "chain.disableBlsBatchVerify"?: boolean; + "chain.persistProducedBlocks"?: boolean; "chain.persistInvalidSszObjects"?: boolean; // No need to define chain.persistInvalidSszObjects as part of ChainArgs // as this is defined as part of BeaconPaths @@ -32,6 +33,7 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] { blsVerifyAllMultiThread: args["chain.blsVerifyAllMultiThread"], blsVerifyAllMainThread: args["chain.blsVerifyAllMainThread"], disableBlsBatchVerify: args["chain.disableBlsBatchVerify"], + persistProducedBlocks: args["chain.persistProducedBlocks"], persistInvalidSszObjects: args["chain.persistInvalidSszObjects"], // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any persistInvalidSszObjectsDir: undefined as any, @@ -94,6 +96,13 @@ Will double processing times. Use only for debugging purposes.", group: "chain", }, + "chain.persistProducedBlocks": { + hidden: true, + type: "boolean", + description: "Persist produced blocks or not for debugging purpose", + group: "chain", + }, + "chain.persistInvalidSszObjects": { hidden: true, type: "boolean", diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 4f5050d87daf..fe2433bb5030 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -21,6 +21,7 @@ describe("options / beaconNodeOptions", () => { "chain.blsVerifyAllMultiThread": true, "chain.blsVerifyAllMainThread": true, "chain.disableBlsBatchVerify": true, + "chain.persistProducedBlocks": true, "chain.persistInvalidSszObjects": true, "chain.proposerBoostEnabled": false, "chain.disableImportExecutionFcU": false, @@ -121,6 +122,7 @@ describe("options / beaconNodeOptions", () => { blsVerifyAllMultiThread: true, blsVerifyAllMainThread: true, disableBlsBatchVerify: true, + persistProducedBlocks: true, persistInvalidSszObjects: true, proposerBoostEnabled: false, disableImportExecutionFcU: false,