From 10b0ec945bcaadfcaad1b11e13e8e7f0c4173f71 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 27 Mar 2024 14:42:13 -0700 Subject: [PATCH 01/11] consume blobs from miner export to chan Signed-off-by: Pranay Valson --- core/block_replica.go | 80 +++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/core/block_replica.go b/core/block_replica.go index dbabccd15..fedca4375 100644 --- a/core/block_replica.go +++ b/core/block_replica.go @@ -21,8 +21,17 @@ type BlockReplicationEvent struct { } func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *ReplicaConfig, chainConfig *params.ChainConfig, stateSpecimen *types.StateSpecimen) error { + + var blobTxSidecars []*types.BlobTxSidecar + for sidecarData := range types.BlobTxSidecarChan { + if sidecarData.BlockNumber == block.Header().Number { + fmt.Println("Consuming Sidecar From Miner Side Channel", sidecarData.BlockNumber) + blobTxSidecars = append(blobTxSidecars, sidecarData.Sidecar...) + } + } + //block replica - exportBlockReplica, err := bc.createReplica(block, replicaConfig, chainConfig, stateSpecimen) + exportBlockReplica, err := bc.createReplica(block, replicaConfig, chainConfig, stateSpecimen, blobTxSidecars) if err != nil { return err } @@ -50,7 +59,7 @@ func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *Repl } } -func (bc *BlockChain) createReplica(block *types.Block, replicaConfig *ReplicaConfig, chainConfig *params.ChainConfig, stateSpecimen *types.StateSpecimen) (*types.ExportBlockReplica, error) { +func (bc *BlockChain) createReplica(block *types.Block, replicaConfig *ReplicaConfig, chainConfig *params.ChainConfig, stateSpecimen *types.StateSpecimen, blobSpecimen []*types.BlobTxSidecar) (*types.ExportBlockReplica, error) { bHash := block.Hash() bNum := block.NumberU64() @@ -119,48 +128,51 @@ func (bc *BlockChain) createReplica(block *types.Block, replicaConfig *ReplicaCo //block replica export if replicaConfig.EnableSpecimen && replicaConfig.EnableResult { exportBlockReplica := &types.ExportBlockReplica{ - Type: "block-replica", - NetworkId: chainConfig.ChainID.Uint64(), - Hash: bHash, - TotalDiff: td, - Header: header, - Transactions: txsRlp, - Uncles: uncles, - Receipts: receiptsRlp, - Senders: senders, - State: stateSpecimen, - Withdrawals: withdrawalsRlp, + Type: "block-replica", + NetworkId: chainConfig.ChainID.Uint64(), + Hash: bHash, + TotalDiff: td, + Header: header, + Transactions: txsRlp, + Uncles: uncles, + Receipts: receiptsRlp, + Senders: senders, + State: stateSpecimen, + Withdrawals: withdrawalsRlp, + BlobTxSidecars: blobSpecimen, } log.Debug("Exporting full block-replica") return exportBlockReplica, nil } else if replicaConfig.EnableSpecimen && !replicaConfig.EnableResult { exportBlockReplica := &types.ExportBlockReplica{ - Type: "block-specimen", - NetworkId: chainConfig.ChainID.Uint64(), - Hash: bHash, - TotalDiff: td, - Header: header, - Transactions: txsRlp, - Uncles: uncles, - Receipts: []*types.ReceiptExportRLP{}, - Senders: senders, - State: stateSpecimen, - Withdrawals: withdrawalsRlp, + Type: "block-specimen", + NetworkId: chainConfig.ChainID.Uint64(), + Hash: bHash, + TotalDiff: td, + Header: header, + Transactions: txsRlp, + Uncles: uncles, + Receipts: []*types.ReceiptExportRLP{}, + Senders: senders, + State: stateSpecimen, + Withdrawals: withdrawalsRlp, + BlobTxSidecars: blobSpecimen, } log.Debug("Exporting block-specimen only") return exportBlockReplica, nil } else if !replicaConfig.EnableSpecimen && replicaConfig.EnableResult { exportBlockReplica := &types.ExportBlockReplica{ - Type: "block-result", - NetworkId: chainConfig.ChainID.Uint64(), - Hash: bHash, - TotalDiff: td, - Header: header, - Transactions: txsRlp, - Uncles: uncles, - Receipts: receiptsRlp, - Senders: senders, - State: &types.StateSpecimen{}, + Type: "block-result", + NetworkId: chainConfig.ChainID.Uint64(), + Hash: bHash, + TotalDiff: td, + Header: header, + Transactions: txsRlp, + Uncles: uncles, + Receipts: receiptsRlp, + Senders: senders, + State: &types.StateSpecimen{}, + BlobTxSidecars: blobSpecimen, } log.Debug("Exporting block-result only") return exportBlockReplica, nil From 6968553531470cc4025e417352f43ec8b16c6200 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 27 Mar 2024 14:43:36 -0700 Subject: [PATCH 02/11] update types to support blob extraction Signed-off-by: Pranay Valson --- core/types/block_export.go | 91 +++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/core/types/block_export.go b/core/types/block_export.go index 8b9ba670a..aeff579f0 100644 --- a/core/types/block_export.go +++ b/core/types/block_export.go @@ -8,17 +8,18 @@ import ( ) type ExportBlockReplica struct { - Type string - NetworkId uint64 - Hash common.Hash - TotalDiff *big.Int - Header *Header - Transactions []*TransactionExportRLP - Uncles []*Header - Receipts []*ReceiptExportRLP - Senders []common.Address - State *StateSpecimen - Withdrawals []*WithdrawalExportRLP + Type string + NetworkId uint64 + Hash common.Hash + TotalDiff *big.Int + Header *Header + Transactions []*TransactionExportRLP + Uncles []*Header + Receipts []*ReceiptExportRLP + Senders []common.Address + State *StateSpecimen + Withdrawals []*WithdrawalExportRLP + BlobTxSidecars []*BlobTxSidecar } type LogsExportRLP struct { @@ -71,8 +72,18 @@ type TransactionExportRLP struct { V *big.Int `json:"v" rlp:"nil"` R *big.Int `json:"r" rlp:"nil"` S *big.Int `json:"s" rlp:"nil"` + BlobFeeCap *big.Int `json:"blobFeeCap" rlp:"optional"` + BlobHashes []common.Hash `json:"blobHashes" rlp:"optional"` + BlobGas uint64 `json:"blobGas" rlp:"optional"` } +type BlobTxSidecarData struct { + Sidecar []*BlobTxSidecar + BlockNumber *big.Int +} + +var BlobTxSidecarChan = make(chan *BlobTxSidecarData, 1000) + func (r *ReceiptForExport) ExportReceipt() *ReceiptExportRLP { enc := &ReceiptExportRLP{ PostStateOrStatus: (*Receipt)(r).statusEncoding(), @@ -105,21 +116,47 @@ func (tx *TransactionForExport) ExportTx(chainConfig *params.ChainConfig, blockN txData := tx.inner - return &TransactionExportRLP{ - AccountNonce: txData.nonce(), - Price: txData.effectiveGasPrice(&big.Int{}, baseFee), - GasLimit: txData.gas(), - Sender: &from, - Recipient: txData.to(), - Amount: txData.value(), - Payload: txData.data(), - Type: txData.txType(), - ChainId: txData.chainID(), - AccessList: txData.accessList(), - GasTipCap: txData.gasTipCap(), - GasFeeCap: txData.gasFeeCap(), - V: v, - R: r, - S: s, + if inner_tx.Type() == BlobTxType { + return &TransactionExportRLP{ + AccountNonce: txData.nonce(), + Price: txData.effectiveGasPrice(&big.Int{}, baseFee), + GasLimit: txData.gas(), + Sender: &from, + Recipient: txData.to(), + Amount: txData.value(), + Payload: txData.data(), + Type: txData.txType(), + ChainId: txData.chainID(), + AccessList: txData.accessList(), + GasTipCap: txData.gasTipCap(), + GasFeeCap: txData.gasFeeCap(), + V: v, + R: r, + S: s, + BlobFeeCap: inner_tx.BlobGasFeeCap(), + BlobHashes: inner_tx.BlobHashes(), + BlobGas: inner_tx.BlobGas(), + } + } else { + return &TransactionExportRLP{ + AccountNonce: txData.nonce(), + Price: txData.effectiveGasPrice(&big.Int{}, baseFee), + GasLimit: txData.gas(), + Sender: &from, + Recipient: txData.to(), + Amount: txData.value(), + Payload: txData.data(), + Type: txData.txType(), + ChainId: txData.chainID(), + AccessList: txData.accessList(), + GasTipCap: txData.gasTipCap(), + GasFeeCap: txData.gasFeeCap(), + V: v, + R: r, + S: s, + BlobFeeCap: &big.Int{}, + BlobHashes: make([]common.Hash, 0), + BlobGas: 0, + } } } From 935f3fb628187f34f2566d6c5fe67c473e04d934 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 27 Mar 2024 14:45:11 -0700 Subject: [PATCH 03/11] force miner to push blob data into sidecar chan Signed-off-by: Pranay Valson --- miner/worker.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/miner/worker.go b/miner/worker.go index 9a3610623..e2fc8ef0d 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -116,6 +116,13 @@ func (env *environment) copy() *environment { cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) copy(cpy.sidecars, env.sidecars) + go func() { + types.BlobTxSidecarChan <- &types.BlobTxSidecarData{ + Sidecar: env.sidecars, + BlockNumber: env.header.Number, + } + }() + return cpy } From 6f0464ca9f8983ccc373f3d3e10a4549dee4d606 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 27 Mar 2024 14:57:46 -0700 Subject: [PATCH 04/11] close channel after pushing sidecar Signed-off-by: Pranay Valson --- core/block_replica.go | 10 +++++++++- core/types/block_export.go | 5 +++-- miner/worker.go | 5 +++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/core/block_replica.go b/core/block_replica.go index fedca4375..fe82f65de 100644 --- a/core/block_replica.go +++ b/core/block_replica.go @@ -25,9 +25,17 @@ func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *Repl var blobTxSidecars []*types.BlobTxSidecar for sidecarData := range types.BlobTxSidecarChan { if sidecarData.BlockNumber == block.Header().Number { - fmt.Println("Consuming Sidecar From Miner Side Channel", sidecarData.BlockNumber) + log.Info("Consuming Sidecar From Miner Side Channel: ", sidecarData.BlockNumber) blobTxSidecars = append(blobTxSidecars, sidecarData.Sidecar...) + } else { + log.Info("Blob Sidecar did not match block number from Miner Side Channel: ", sidecarData.BlockNumber) } + fmt.Println(sidecarData.BlockNumber, "side car header block number") + fmt.Println(sidecarData.ParentBeaconRoot, "side car parent beacon block root") + } + + for sideCar := range blobTxSidecars { + fmt.Println(sideCar, "full side car") } //block replica diff --git a/core/types/block_export.go b/core/types/block_export.go index aeff579f0..30adcf3e4 100644 --- a/core/types/block_export.go +++ b/core/types/block_export.go @@ -78,8 +78,9 @@ type TransactionExportRLP struct { } type BlobTxSidecarData struct { - Sidecar []*BlobTxSidecar - BlockNumber *big.Int + Sidecar []*BlobTxSidecar + BlockNumber *big.Int + ParentBeaconRoot *common.Hash } var BlobTxSidecarChan = make(chan *BlobTxSidecarData, 1000) diff --git a/miner/worker.go b/miner/worker.go index e2fc8ef0d..d3cc2f732 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -118,8 +118,9 @@ func (env *environment) copy() *environment { go func() { types.BlobTxSidecarChan <- &types.BlobTxSidecarData{ - Sidecar: env.sidecars, - BlockNumber: env.header.Number, + Sidecar: env.sidecars, + BlockNumber: env.header.Number, + ParentBeaconRoot: env.header.ParentBeaconRoot, } }() From 596f7c084d841b1ceec5afc59f4e477d6d4a74f0 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 27 Mar 2024 17:11:56 -0700 Subject: [PATCH 05/11] add blob tx side car chan to core Signed-off-by: Pranay Valson --- core/block_replica.go | 8 ++++---- core/types/block_export.go | 5 ++--- miner/worker.go | 15 +++++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/core/block_replica.go b/core/block_replica.go index fe82f65de..dbed3a4da 100644 --- a/core/block_replica.go +++ b/core/block_replica.go @@ -26,16 +26,16 @@ func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *Repl for sidecarData := range types.BlobTxSidecarChan { if sidecarData.BlockNumber == block.Header().Number { log.Info("Consuming Sidecar From Miner Side Channel: ", sidecarData.BlockNumber) - blobTxSidecars = append(blobTxSidecars, sidecarData.Sidecar...) + blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) } else { log.Info("Blob Sidecar did not match block number from Miner Side Channel: ", sidecarData.BlockNumber) } fmt.Println(sidecarData.BlockNumber, "side car header block number") - fmt.Println(sidecarData.ParentBeaconRoot, "side car parent beacon block root") + fmt.Println("length of sidecar channel", len(types.BlobTxSidecarChan)) } - for sideCar := range blobTxSidecars { - fmt.Println(sideCar, "full side car") + for _, sidecarData := range blobTxSidecars { + fmt.Println(*sidecarData, "full side car") } //block replica diff --git a/core/types/block_export.go b/core/types/block_export.go index 30adcf3e4..483cada73 100644 --- a/core/types/block_export.go +++ b/core/types/block_export.go @@ -78,9 +78,8 @@ type TransactionExportRLP struct { } type BlobTxSidecarData struct { - Sidecar []*BlobTxSidecar - BlockNumber *big.Int - ParentBeaconRoot *common.Hash + Blobs *BlobTxSidecar + BlockNumber *big.Int } var BlobTxSidecarChan = make(chan *BlobTxSidecarData, 1000) diff --git a/miner/worker.go b/miner/worker.go index d3cc2f732..8c391f611 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -80,6 +80,8 @@ var ( errBlockInterruptedByTimeout = errors.New("timeout while building block") ) +var BlobTxSidecarChan = make(chan *types.BlobTxSidecarData, 1000) + // environment is the worker's current environment and holds all // information of the sealing block generation. type environment struct { @@ -116,12 +118,17 @@ func (env *environment) copy() *environment { cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) copy(cpy.sidecars, env.sidecars) + types.BlobTxSidecarChan = make(chan *types.BlobTxSidecarData, 1000) + go func() { - types.BlobTxSidecarChan <- &types.BlobTxSidecarData{ - Sidecar: env.sidecars, - BlockNumber: env.header.Number, - ParentBeaconRoot: env.header.ParentBeaconRoot, + for sidecar := range env.sidecars { + types.BlobTxSidecarChan <- &types.BlobTxSidecarData{ + Blobs: env.sidecars[sidecar], + BlockNumber: env.header.Number, + } } + fmt.Println("closed sidecar channel in miner") + close(types.BlobTxSidecarChan) }() return cpy From 266e19b8342aeaa39595eda943dd8a69ae783548 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 27 Mar 2024 17:59:29 -0700 Subject: [PATCH 06/11] oversee block number check for now (matched from miner) Signed-off-by: Pranay Valson --- core/block_replica.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/core/block_replica.go b/core/block_replica.go index dbed3a4da..82036ae4a 100644 --- a/core/block_replica.go +++ b/core/block_replica.go @@ -24,14 +24,16 @@ func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *Repl var blobTxSidecars []*types.BlobTxSidecar for sidecarData := range types.BlobTxSidecarChan { - if sidecarData.BlockNumber == block.Header().Number { - log.Info("Consuming Sidecar From Miner Side Channel: ", sidecarData.BlockNumber) - blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) - } else { - log.Info("Blob Sidecar did not match block number from Miner Side Channel: ", sidecarData.BlockNumber) - } - fmt.Println(sidecarData.BlockNumber, "side car header block number") - fmt.Println("length of sidecar channel", len(types.BlobTxSidecarChan)) + // if sidecarData.BlockNumber == block.Header().Number { + // log.Info("Consuming Sidecar From Miner Side Channel: ", sidecarData.BlockNumber) + // blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) + // } else { + // log.Info("Blob Sidecar did not match block number from Miner Side Channel: ", sidecarData.BlockNumber) + // } + fmt.Println("side car header block number:", sidecarData.BlockNumber) + fmt.Println("length of sidecar channel:", len(types.BlobTxSidecarChan)) + + blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) } for _, sidecarData := range blobTxSidecars { From 32976491b3e2fd145873de54e8ea5cf9678e18d0 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 27 Mar 2024 18:10:05 -0700 Subject: [PATCH 07/11] make explicit block number check with type Signed-off-by: Pranay Valson --- core/block_replica.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/block_replica.go b/core/block_replica.go index 82036ae4a..93b6a7d6f 100644 --- a/core/block_replica.go +++ b/core/block_replica.go @@ -24,21 +24,21 @@ func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *Repl var blobTxSidecars []*types.BlobTxSidecar for sidecarData := range types.BlobTxSidecarChan { - // if sidecarData.BlockNumber == block.Header().Number { - // log.Info("Consuming Sidecar From Miner Side Channel: ", sidecarData.BlockNumber) - // blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) - // } else { - // log.Info("Blob Sidecar did not match block number from Miner Side Channel: ", sidecarData.BlockNumber) - // } + if sidecarData.BlockNumber.Uint64() == block.NumberU64() { + log.Info("Consuming Sidecar From Miner Side Channel: ", sidecarData.BlockNumber) + blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) + } else { + log.Info("Blob Sidecar did not match block number from Miner Side Channel: ", sidecarData.BlockNumber.Uint64()) + } fmt.Println("side car header block number:", sidecarData.BlockNumber) fmt.Println("length of sidecar channel:", len(types.BlobTxSidecarChan)) - blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) + // blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) } - for _, sidecarData := range blobTxSidecars { - fmt.Println(*sidecarData, "full side car") - } + // for _, sidecarData := range blobTxSidecars { + // fmt.Println(*sidecarData, "full side car") + // } //block replica exportBlockReplica, err := bc.createReplica(block, replicaConfig, chainConfig, stateSpecimen, blobTxSidecars) From 4eaf414c0a9eeea9860c02cd7fdab3bde77e4c41 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Thu, 28 Mar 2024 12:42:02 -0700 Subject: [PATCH 08/11] format info and optimize channel buffers Signed-off-by: Pranay Valson --- core/block_replica.go | 18 ++++++++---------- core/types/block_export.go | 2 +- miner/worker.go | 4 ++-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/core/block_replica.go b/core/block_replica.go index 93b6a7d6f..0cee9236f 100644 --- a/core/block_replica.go +++ b/core/block_replica.go @@ -22,25 +22,23 @@ type BlockReplicationEvent struct { func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *ReplicaConfig, chainConfig *params.ChainConfig, stateSpecimen *types.StateSpecimen) error { + // blobs var blobTxSidecars []*types.BlobTxSidecar for sidecarData := range types.BlobTxSidecarChan { if sidecarData.BlockNumber.Uint64() == block.NumberU64() { - log.Info("Consuming Sidecar From Miner Side Channel: ", sidecarData.BlockNumber) + log.Info("Consuming BlobTxSidecar Match From Chain Sync Channel", "Block Number:", sidecarData.BlockNumber.Uint64()) blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) } else { - log.Info("Blob Sidecar did not match block number from Miner Side Channel: ", sidecarData.BlockNumber.Uint64()) + log.Info("Failing BlobTxSidecar Match from Chain Sync Channel", "Block Number:", sidecarData.BlockNumber.Uint64()) } - fmt.Println("side car header block number:", sidecarData.BlockNumber) - fmt.Println("length of sidecar channel:", len(types.BlobTxSidecarChan)) - // blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) - } + log.Info("BlobTxSidecar Header", "Block Number:", sidecarData.BlockNumber.Uint64()) + + log.Info("Chain Sync Sidecar Channel", "Length:", len(types.BlobTxSidecarChan)) - // for _, sidecarData := range blobTxSidecars { - // fmt.Println(*sidecarData, "full side car") - // } + } - //block replica + //block replica with blobs exportBlockReplica, err := bc.createReplica(block, replicaConfig, chainConfig, stateSpecimen, blobTxSidecars) if err != nil { return err diff --git a/core/types/block_export.go b/core/types/block_export.go index 483cada73..06e90a9ca 100644 --- a/core/types/block_export.go +++ b/core/types/block_export.go @@ -82,7 +82,7 @@ type BlobTxSidecarData struct { BlockNumber *big.Int } -var BlobTxSidecarChan = make(chan *BlobTxSidecarData, 1000) +var BlobTxSidecarChan = make(chan *BlobTxSidecarData, 100) func (r *ReceiptForExport) ExportReceipt() *ReceiptExportRLP { enc := &ReceiptExportRLP{ diff --git a/miner/worker.go b/miner/worker.go index 8c391f611..b9ab71675 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -118,7 +118,7 @@ func (env *environment) copy() *environment { cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) copy(cpy.sidecars, env.sidecars) - types.BlobTxSidecarChan = make(chan *types.BlobTxSidecarData, 1000) + types.BlobTxSidecarChan = make(chan *types.BlobTxSidecarData, 100) go func() { for sidecar := range env.sidecars { @@ -127,7 +127,7 @@ func (env *environment) copy() *environment { BlockNumber: env.header.Number, } } - fmt.Println("closed sidecar channel in miner") + log.Info("Closing Chain Sync BlobTxSidecar Channel For", "Block Number:", env.header.Number.Uint64(), "Length:", len(types.BlobTxSidecarChan)) close(types.BlobTxSidecarChan) }() From 5186db9fe9ed11862cc85f6fe661d3a29b79b548 Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 17 Apr 2024 17:59:53 -0700 Subject: [PATCH 09/11] additional condition for block specimen creation Signed-off-by: Pranay Valson --- .gitignore | 1 + core/blockchain.go | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b117d3da3..72cd02150 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ logs/ coverage.out coverage.txt tests/spec-tests/ +bin/ diff --git a/core/blockchain.go b/core/blockchain.go index 84e51bac3..0d9747988 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1851,7 +1851,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) if !setHead { // Export Block Specimen - bc.createBlockReplica(block, bc.ReplicaConfig, bc.chainConfig, statedb.TakeStateSpecimen()) + if bc.ReplicaConfig.EnableSpecimen || bc.ReplicaConfig.EnableResult { + bc.createBlockReplica(block, bc.ReplicaConfig, bc.chainConfig, statedb.TakeStateSpecimen()) + } // After merge we expect few side chains. Simply count // all blocks the CL gives us for GC processing time bc.gcproc += proctime @@ -1865,7 +1867,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) "elapsed", common.PrettyDuration(time.Since(start)), "root", block.Root()) // Handle creation of block specimen for canonical blocks - bc.createBlockReplica(block, bc.ReplicaConfig, bc.chainConfig, statedb.TakeStateSpecimen()) + if bc.ReplicaConfig.EnableSpecimen || bc.ReplicaConfig.EnableResult { + bc.createBlockReplica(block, bc.ReplicaConfig, bc.chainConfig, statedb.TakeStateSpecimen()) + } lastCanon = block // Only count canonical blocks for GC processing time From 6310ef8c40845e5fbec74eb577ee27d7f66890ae Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Wed, 17 Apr 2024 18:22:55 -0700 Subject: [PATCH 10/11] conditional for blob specimen creation Signed-off-by: Pranay Valson --- cmd/geth/chaincmd.go | 1 + cmd/geth/main.go | 1 + cmd/utils/flags.go | 9 ++++++++- core/block_replica.go | 39 ++++++++++++++++++++------------------- core/blockchain.go | 6 +++++- eth/backend.go | 3 ++- eth/ethconfig/config.go | 1 + miner/worker.go | 34 +++++++++++++++++++--------------- 8 files changed, 57 insertions(+), 37 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 36437c4f7..0d5f2c5ce 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -104,6 +104,7 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.BlockReplicationTargetsFlag, utils.ReplicaEnableSpecimenFlag, utils.ReplicaEnableResultFlag, + utils.ReplicaEnableBlobFlag, }, utils.DatabaseFlags), Description: ` The import command imports blocks from an RLP-encoded form. The form can be one file diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 5e6b750d4..25eba5f49 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -149,6 +149,7 @@ var ( utils.BlockReplicationTargetsFlag, utils.ReplicaEnableResultFlag, utils.ReplicaEnableSpecimenFlag, + utils.ReplicaEnableBlobFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5b97664be..e1185feb9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -694,6 +694,10 @@ var ( Name: "replica.result", Usage: "Enables export of fields that comprise a block-result", } + ReplicaEnableBlobFlag = &cli.BoolFlag{ + Name: "replica.blob", + Usage: "Enables export of fields that comprise a block-blob", + } BatchRequestLimit = &cli.IntFlag{ Name: "rpc.batch-request-limit", Usage: "Maximum number of requests in a batch", @@ -2188,8 +2192,11 @@ func setBlockReplicationTargets(ctx *cli.Context, cfg *eth.Config) { if ctx.Bool(ReplicaEnableResultFlag.Name) { cfg.ReplicaEnableResult = true } + if ctx.Bool(ReplicaEnableBlobFlag.Name) { + cfg.ReplicaEnableBlob = true + } } else { - Fatalf("--replication.targets flag is invalid without --replica.specimen and/or --replica.result") + Fatalf("--replication.targets flag is invalid without --replica.specimen and/or --replica.result, ONLY ADD --replica.blob with both replica.specimen AND replica.result flags for complete unified state capture)") } } diff --git a/core/block_replica.go b/core/block_replica.go index 0cee9236f..9160faf1a 100644 --- a/core/block_replica.go +++ b/core/block_replica.go @@ -24,20 +24,18 @@ func (bc *BlockChain) createBlockReplica(block *types.Block, replicaConfig *Repl // blobs var blobTxSidecars []*types.BlobTxSidecar - for sidecarData := range types.BlobTxSidecarChan { - if sidecarData.BlockNumber.Uint64() == block.NumberU64() { - log.Info("Consuming BlobTxSidecar Match From Chain Sync Channel", "Block Number:", sidecarData.BlockNumber.Uint64()) - blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) - } else { - log.Info("Failing BlobTxSidecar Match from Chain Sync Channel", "Block Number:", sidecarData.BlockNumber.Uint64()) + if replicaConfig.EnableBlob { + for sidecarData := range types.BlobTxSidecarChan { + if sidecarData.BlockNumber.Uint64() == block.NumberU64() { + log.Info("Consuming BlobTxSidecar Match From Chain Sync Channel", "Block Number:", sidecarData.BlockNumber.Uint64()) + blobTxSidecars = append(blobTxSidecars, sidecarData.Blobs) + } else { + log.Info("Failing BlobTxSidecar Match from Chain Sync Channel", "Block Number:", sidecarData.BlockNumber.Uint64()) + } + log.Info("BlobTxSidecar Header", "Block Number:", sidecarData.BlockNumber.Uint64()) + log.Info("Chain Sync Sidecar Channel", "Length:", len(types.BlobTxSidecarChan)) } - - log.Info("BlobTxSidecar Header", "Block Number:", sidecarData.BlockNumber.Uint64()) - - log.Info("Chain Sync Sidecar Channel", "Length:", len(types.BlobTxSidecarChan)) - } - //block replica with blobs exportBlockReplica, err := bc.createReplica(block, replicaConfig, chainConfig, stateSpecimen, blobTxSidecars) if err != nil { @@ -134,7 +132,7 @@ func (bc *BlockChain) createReplica(block *types.Block, replicaConfig *ReplicaCo uncles := block.Uncles() //block replica export - if replicaConfig.EnableSpecimen && replicaConfig.EnableResult { + if replicaConfig.EnableSpecimen && replicaConfig.EnableResult && replicaConfig.EnableBlob { exportBlockReplica := &types.ExportBlockReplica{ Type: "block-replica", NetworkId: chainConfig.ChainID.Uint64(), @@ -149,7 +147,7 @@ func (bc *BlockChain) createReplica(block *types.Block, replicaConfig *ReplicaCo Withdrawals: withdrawalsRlp, BlobTxSidecars: blobSpecimen, } - log.Debug("Exporting full block-replica") + log.Debug("Exporting full block-replica with blob-specimen") return exportBlockReplica, nil } else if replicaConfig.EnableSpecimen && !replicaConfig.EnableResult { exportBlockReplica := &types.ExportBlockReplica{ @@ -164,9 +162,9 @@ func (bc *BlockChain) createReplica(block *types.Block, replicaConfig *ReplicaCo Senders: senders, State: stateSpecimen, Withdrawals: withdrawalsRlp, - BlobTxSidecars: blobSpecimen, + BlobTxSidecars: []*types.BlobTxSidecar{}, } - log.Debug("Exporting block-specimen only") + log.Debug("Exporting block-specimen only (no blob specimens)") return exportBlockReplica, nil } else if !replicaConfig.EnableSpecimen && replicaConfig.EnableResult { exportBlockReplica := &types.ExportBlockReplica{ @@ -180,12 +178,12 @@ func (bc *BlockChain) createReplica(block *types.Block, replicaConfig *ReplicaCo Receipts: receiptsRlp, Senders: senders, State: &types.StateSpecimen{}, - BlobTxSidecars: blobSpecimen, + BlobTxSidecars: []*types.BlobTxSidecar{}, } - log.Debug("Exporting block-result only") + log.Debug("Exporting block-result only (no blob specimens)") return exportBlockReplica, nil } else { - return nil, fmt.Errorf("--replication.targets flag is invalid without --replica.specimen and/or --replica.result") + return nil, fmt.Errorf("--replication.targets flag is invalid without --replica.specimen and/or --replica.result, ADD --replica.blob with both replica.specimen AND replica.result flags for complete unified state capture aka block-replica)") } } @@ -201,5 +199,8 @@ func (bc *BlockChain) SetBlockReplicaExports(replicaConfig *ReplicaConfig) bool if replicaConfig.EnableSpecimen { bc.ReplicaConfig.EnableSpecimen = true } + if replicaConfig.EnableBlob { + bc.ReplicaConfig.EnableBlob = true + } return true } diff --git a/core/blockchain.go b/core/blockchain.go index 0d9747988..3436ef9d7 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -267,6 +267,7 @@ type BlockChain struct { type ReplicaConfig struct { EnableSpecimen bool EnableResult bool + EnableBlob bool HistoricalBlocksSynced *uint32 } @@ -315,6 +316,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis ReplicaConfig: &ReplicaConfig{ EnableSpecimen: false, EnableResult: false, + EnableBlob: false, HistoricalBlocksSynced: new(uint32), // Always set 0 for historical mode at start }, } @@ -1891,7 +1893,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), "root", block.Root()) // Is impossible but keeping in line to be nice to our future selves we add this for now - bc.createBlockReplica(block, bc.ReplicaConfig, bc.chainConfig, statedb.TakeStateSpecimen()) + if bc.ReplicaConfig.EnableSpecimen || bc.ReplicaConfig.EnableResult { + bc.createBlockReplica(block, bc.ReplicaConfig, bc.chainConfig, statedb.TakeStateSpecimen()) + } } } diff --git a/eth/backend.go b/eth/backend.go index d858affbe..62419ffbd 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -177,6 +177,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { ReplicaConfig: &core.ReplicaConfig{ EnableSpecimen: config.ReplicaEnableSpecimen, EnableResult: config.ReplicaEnableResult, + EnableBlob: config.ReplicaEnableBlob, HistoricalBlocksSynced: new(uint32), // Always set 0 for historical mode at start }, } @@ -186,7 +187,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - log.Info("Block replication started", "targets", targets, "network ID", config.NetworkId, "export block-specimen", eth.ReplicaConfig.EnableSpecimen, "export block-result", eth.ReplicaConfig.EnableResult) + log.Info("Block replication started", "targets", targets, "network ID", config.NetworkId, "export block-specimen", eth.ReplicaConfig.EnableSpecimen, "export block-result", eth.ReplicaConfig.EnableResult, "export blob-specimen", eth.ReplicaConfig.EnableBlob) eth.blockReplicators = append(eth.blockReplicators, replicator) } bcVersion := rawdb.ReadDatabaseVersion(chainDb) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 456b4a8c8..c86a00f07 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -160,6 +160,7 @@ type Config struct { // Bools that make explicit types being exported ReplicaEnableResult bool ReplicaEnableSpecimen bool + ReplicaEnableBlob bool // OverrideCancun (TODO: remove after the fork) OverrideCancun *uint64 `toml:",omitempty"` diff --git a/miner/worker.go b/miner/worker.go index b9ab71675..95433153b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -80,7 +80,7 @@ var ( errBlockInterruptedByTimeout = errors.New("timeout while building block") ) -var BlobTxSidecarChan = make(chan *types.BlobTxSidecarData, 1000) +var enableBlobTxSidecar bool // environment is the worker's current environment and holds all // information of the sealing block generation. @@ -115,21 +115,21 @@ func (env *environment) copy() *environment { cpy.txs = make([]*types.Transaction, len(env.txs)) copy(cpy.txs, env.txs) - cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) - copy(cpy.sidecars, env.sidecars) - - types.BlobTxSidecarChan = make(chan *types.BlobTxSidecarData, 100) - - go func() { - for sidecar := range env.sidecars { - types.BlobTxSidecarChan <- &types.BlobTxSidecarData{ - Blobs: env.sidecars[sidecar], - BlockNumber: env.header.Number, + if enableBlobTxSidecar { + cpy.sidecars = make([]*types.BlobTxSidecar, len(env.sidecars)) + copy(cpy.sidecars, env.sidecars) + types.BlobTxSidecarChan = make(chan *types.BlobTxSidecarData, 100) + go func() { + for sidecar := range env.sidecars { + types.BlobTxSidecarChan <- &types.BlobTxSidecarData{ + Blobs: env.sidecars[sidecar], + BlockNumber: env.header.Number, + } } - } - log.Info("Closing Chain Sync BlobTxSidecar Channel For", "Block Number:", env.header.Number.Uint64(), "Length:", len(types.BlobTxSidecarChan)) - close(types.BlobTxSidecarChan) - }() + log.Info("Closing Chain Sync BlobTxSidecar Channel For", "Block Number:", env.header.Number.Uint64(), "Length:", len(types.BlobTxSidecarChan)) + close(types.BlobTxSidecarChan) + }() + } return cpy } @@ -305,6 +305,10 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus } worker.newpayloadTimeout = newpayloadTimeout + if worker.chain.ReplicaConfig.EnableBlob { + enableBlobTxSidecar = true + } + worker.wg.Add(4) go worker.mainLoop() go worker.newWorkLoop(recommit) From b720ac54564c637840b027b66c42a98d927e47ef Mon Sep 17 00:00:00 2001 From: Pranay Valson Date: Tue, 23 Apr 2024 18:17:44 -0700 Subject: [PATCH 11/11] bump bsp-geth minor version Signed-off-by: Pranay Valson --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 52e7ac5a5..f2dded576 100644 --- a/params/version.go +++ b/params/version.go @@ -29,7 +29,7 @@ const ( const ( BspVersionMajor = 1 // Major version component of the current release - BspVersionMinor = 6 // Minor version component of the current release + BspVersionMinor = 8 // Minor version component of the current release BspVersionPatch = 0 // Patch version component of the current release )