From 2f7d3050135718b11785f06417d5a0794112afbe Mon Sep 17 00:00:00 2001 From: GalaIO Date: Mon, 25 Sep 2023 20:11:52 +0800 Subject: [PATCH 1/5] logs: add some logs to trace block miner, justified reorg; (#1894) --- core/blockchain_insert.go | 2 +- core/forkchoice.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 3e913cb87b..ffe2d6501c 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -56,7 +56,7 @@ func (st *insertStats) report(chain []*types.Block, index int, dirty common.Stor // Assemble the log context and send it to the logger context := []interface{}{ - "number", end.Number(), "hash", end.Hash(), + "number", end.Number(), "hash", end.Hash(), "miner", end.Coinbase(), "blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000, "elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed), } diff --git a/core/forkchoice.go b/core/forkchoice.go index d2e939fad2..c42ed24d6f 100644 --- a/core/forkchoice.go +++ b/core/forkchoice.go @@ -137,5 +137,9 @@ func (f *ForkChoice) ReorgNeededWithFastFinality(current *types.Header, header * return f.ReorgNeeded(current, header) } + if justifiedNumber > curJustifiedNumber && header.Number.Cmp(current.Number) <= 0 { + log.Info("Chain find higher justifiedNumber", "fromHeight", current.Number, "fromHash", current.Hash(), "fromMiner", current.Coinbase, "fromJustified", curJustifiedNumber, + "toHeight", header.Number, "toHash", header.Hash(), "toMiner", header.Coinbase, "toJustified", justifiedNumber) + } return justifiedNumber > curJustifiedNumber, nil } From 720ff1fe57810128031d32c202f0437fababa55e Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 11 Aug 2023 03:21:36 +0800 Subject: [PATCH 2/5] all: activate pbss as experimental feature from eth (#26274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * all: activate pbss * core/rawdb: fix compilation error * cma, core, eth, les, trie: address comments * cmd, core, eth, trie: polish code * core, cmd, eth: address comments * cmd, core, eth, les, light, tests: address comment * cmd/utils: shorten log message * trie/triedb/pathdb: limit node buffer size to 1gb * cmd/utils: fix opening non-existing db * cmd/utils: rename flag name * cmd, core: group chain history flags and fix tests * core, eth, trie: fix memory leak in snapshot generation * cmd, eth, internal: deprecate flags * all: enable state tests for pathdb, fixes * cmd, core: polish code * trie/triedb/pathdb: limit the node buffer size to 256mb --------- Co-authored-by: Martin Holst Swende Co-authored-by: Péter Szilágyi --- cmd/evm/blockrunner.go | 3 +- cmd/evm/runner.go | 18 +- cmd/evm/staterunner.go | 35 +- cmd/geth/chaincmd.go | 30 +- cmd/geth/dbcmd.go | 6 +- cmd/geth/genesis_test.go | 4 +- cmd/geth/main.go | 3 + cmd/geth/snapshot.go | 56 +- cmd/utils/flags.go | 187 ++++- cmd/utils/flags_legacy.go | 9 + core/block_validator_test.go | 7 +- core/blockchain.go | 179 +++-- core/blockchain_reader.go | 14 +- core/blockchain_repair_test.go | 57 +- core/blockchain_sethead_test.go | 34 +- core/blockchain_snapshot_test.go | 274 ++++--- core/blockchain_test.go | 746 ++++++++++++++++-- core/chain_makers.go | 16 +- core/chain_makers_test.go | 11 +- core/dao_test.go | 8 +- core/genesis.go | 14 +- core/genesis_test.go | 67 +- core/headerchain_test.go | 2 +- core/rawdb/accessors_chain_test.go | 2 +- core/rawdb/accessors_trie.go | 26 +- core/rawdb/ancient_scheme.go | 8 +- core/rawdb/ancient_utils.go | 54 +- core/rawdb/database.go | 23 +- core/rawdb/schema.go | 29 +- core/state/database.go | 6 +- core/state/iterator_test.go | 15 +- core/state/pruner/pruner.go | 13 +- core/state/snapshot/generate.go | 3 +- core/state/snapshot/generate_test.go | 142 +++- core/state/statedb_fuzz_test.go | 2 +- core/state/statedb_test.go | 57 +- core/state/sync_test.go | 169 ++-- core/types/hashing_test.go | 8 +- eth/api_backend.go | 2 +- eth/api_debug.go | 2 +- eth/backend.go | 14 +- eth/downloader/testchain_test.go | 3 +- eth/ethconfig/config.go | 10 +- eth/ethconfig/gen_config.go | 18 + eth/fetcher/block_fetcher_test.go | 2 +- eth/filters/filter_test.go | 4 +- eth/handler.go | 15 +- eth/protocols/eth/handler_test.go | 2 +- eth/protocols/eth/handlers.go | 7 +- eth/protocols/snap/handler.go | 8 +- eth/protocols/snap/sync_test.go | 204 +++-- eth/state_accessor.go | 91 ++- .../internal/tracetest/calltrace_test.go | 13 +- .../internal/tracetest/flat_calltrace_test.go | 3 +- .../internal/tracetest/prestate_test.go | 4 +- eth/tracers/tracers_test.go | 4 +- internal/flags/categories.go | 1 + les/client.go | 3 +- les/handler_test.go | 8 +- les/odr_test.go | 2 +- les/server_handler.go | 3 +- les/server_requests.go | 5 +- les/test_helper.go | 5 +- light/lightchain_test.go | 5 +- light/odr_test.go | 3 +- light/postprocess.go | 4 +- light/trie.go | 6 +- light/trie_test.go | 2 +- light/txpool_test.go | 3 +- miner/miner_test.go | 5 +- tests/block_test.go | 16 +- tests/block_test_util.go | 27 +- tests/fuzzers/les/les-fuzzer.go | 4 +- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 2 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 4 +- tests/fuzzers/trie/trie-fuzzer.go | 2 +- tests/state_test.go | 56 +- tests/state_test_util.go | 66 +- trie/database.go | 107 ++- trie/database_test.go | 2 +- trie/iterator_test.go | 113 +-- trie/proof_test.go | 20 +- trie/secure_trie_test.go | 4 +- trie/stacktrie_test.go | 12 +- trie/sync_test.go | 4 +- trie/tracer_test.go | 10 +- trie/trie_reader.go | 16 + trie/trie_test.go | 63 +- trie/triedb/hashdb/database.go | 33 +- trie/triedb/pathdb/database.go | 76 +- trie/triedb/pathdb/database_test.go | 5 +- trie/triedb/pathdb/difflayer_test.go | 2 +- trie/triedb/pathdb/disklayer.go | 20 +- trie/triedb/pathdb/history_test.go | 2 +- trie/triedb/pathdb/testutils.go | 4 +- trie/triestate/state.go | 17 +- 96 files changed, 2533 insertions(+), 952 deletions(-) diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index ffd3165f55..0be5f69711 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -22,6 +22,7 @@ import ( "fmt" "os" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/log" @@ -64,7 +65,7 @@ func blockTestCmd(ctx *cli.Context) error { return err } for i, test := range tests { - if err := test.Run(false, tracer); err != nil { + if err := test.Run(false, rawdb.HashScheme, tracer); err != nil { return fmt.Errorf("test %v: %w", i, err) } } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index c4ac4fcea9..73a4169b1d 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -42,6 +42,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/urfave/cli/v2" ) @@ -142,12 +143,23 @@ func runCmd(ctx *cli.Context) error { gen := readGenesis(ctx.String(GenesisFlag.Name)) genesisConfig = gen db := rawdb.NewMemoryDatabase() - genesis := gen.MustCommit(db) - sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: preimages}) + triedb := trie.NewDatabase(db, &trie.Config{ + Preimages: preimages, + HashDB: hashdb.Defaults, + }) + defer triedb.Close() + genesis := gen.MustCommit(db, triedb) + sdb := state.NewDatabaseWithNodeDB(db, triedb) statedb, _ = state.New(genesis.Root(), sdb, nil) chainConfig = gen.Config } else { - sdb := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: preimages}) + db := rawdb.NewMemoryDatabase() + triedb := trie.NewDatabase(db, &trie.Config{ + Preimages: preimages, + HashDB: hashdb.Defaults, + }) + defer triedb.Close() + sdb := state.NewDatabaseWithNodeDB(db, triedb) statedb, _ = state.New(types.EmptyRootHash, sdb, nil) genesisConfig = new(core.Genesis) } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 231830f6d7..a29c4b18f6 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -23,7 +23,9 @@ import ( "os" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/log" @@ -104,25 +106,22 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - _, s, err := test.Run(st, cfg, false) - // print state root for evmlab tracing - if s != nil { - root := s.IntermediateRoot(false) - result.Root = &root - if jsonOut { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + if err != nil { + // Test failed, mark as so and dump any state to aid debugging + result.Pass, result.Error = false, err.Error() + if dump { + dump := state.RawDump(nil) + result.State = &dump + } + } else { + root := state.IntermediateRoot(false) + result.Root = &root + if jsonOut { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) + } } - } - if err != nil { - // Test failed, mark as so and dump any state to aid debugging - result.Pass, result.Error = false, err.Error() - if dump && s != nil { - s, _ = state.New(*result.Root, s.Database(), nil) - dump := s.RawDump(nil) - result.State = &dump - } - } - + }) results = append(results, *result) } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c10a744538..2d22c5a59e 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -44,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/trie" "github.com/urfave/cli/v2" ) @@ -54,7 +53,10 @@ var ( Name: "init", Usage: "Bootstrap and initialize a new genesis block", ArgsUsage: "", - Flags: flags.Merge([]cli.Flag{utils.CachePreimagesFlag}, utils.DatabasePathFlags), + Flags: flags.Merge([]cli.Flag{ + utils.CachePreimagesFlag, + utils.StateSchemeFlag, + }, utils.DatabasePathFlags), Description: ` The init command initializes a new genesis block and definition for the network. This is a destructive action and changes the network in which you will be @@ -116,6 +118,9 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, + utils.TransactionHistoryFlag, + utils.StateSchemeFlag, + utils.StateHistoryFlag, }, utils.DatabasePathFlags), Description: ` The import command imports blocks from an RLP-encoded form. The form can be one file @@ -132,6 +137,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`, Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, + utils.StateSchemeFlag, }, utils.DatabasePathFlags), Description: ` Requires a first argument of the file to write to. @@ -181,6 +187,7 @@ It's deprecated, please use "geth db export" instead. utils.IncludeIncompletesFlag, utils.StartKeyFlag, utils.DumpLimitFlag, + utils.StateSchemeFlag, }, utils.DatabasePathFlags), Description: ` This command dumps out the state for a given block (or latest, if none provided). @@ -217,14 +224,15 @@ func initGenesis(ctx *cli.Context) error { if err != nil { utils.Fatalf("Failed to open database: %v", err) } - triedb := trie.NewDatabaseWithConfig(chaindb, &trie.Config{ - Preimages: ctx.Bool(utils.CachePreimagesFlag.Name), - }) + defer chaindb.Close() + + triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false) + defer triedb.Close() + _, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis) if err != nil { utils.Fatalf("Failed to write genesis block: %v", err) } - chaindb.Close() log.Info("Successfully wrote genesis state", "database", name, "hash", hash) } return nil @@ -422,7 +430,7 @@ func dumpGenesis(ctx *cli.Context) error { if ctx.IsSet(utils.DataDirFlag.Name) { utils.Fatalf("no existing datadir at %s", stack.Config().DataDir) } - utils.Fatalf("no network preset provided. no exisiting genesis in the default datadir") + utils.Fatalf("no network preset provided, no existing genesis in the default datadir") return nil } @@ -646,10 +654,10 @@ func dump(ctx *cli.Context) error { if err != nil { return err } - config := &trie.Config{ - Preimages: true, // always enable preimage lookup - } - state, err := state.New(root, state.NewDatabaseWithConfig(db, config), nil) + triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup + defer triedb.Close() + + state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { return err } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 340bc85faa..b8fa365049 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -154,6 +154,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, ArgsUsage: " ", Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, + utils.StateSchemeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command looks up the specified database key from the database.", } @@ -504,6 +505,9 @@ func dbDumpTrie(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true, false) defer db.Close() + triedb := utils.MakeTrieDatabase(ctx, db, false, true) + defer triedb.Close() + var ( state []byte storage []byte @@ -537,7 +541,7 @@ func dbDumpTrie(ctx *cli.Context) error { } } id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage)) - theTrie, err := trie.New(id, trie.NewDatabase(db)) + theTrie, err := trie.New(id, triedb) if err != nil { return err } diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index fcfaa6e975..5a505b8022 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -176,12 +176,12 @@ func TestCustomBackend(t *testing.T) { { // Can't start pebble on top of leveldb initArgs: []string{"--db.engine", "leveldb"}, execArgs: []string{"--db.engine", "pebble"}, - execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, + execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, }, { // Can't start leveldb on top of pebble initArgs: []string{"--db.engine", "pebble"}, execArgs: []string{"--db.engine", "leveldb"}, - execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, + execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, }, { // Reject invalid backend choice initArgs: []string{"--db.engine", "mssql"}, diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 27f41c216f..b1b08588e2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -94,6 +94,9 @@ var ( utils.GCModeFlag, utils.SnapshotFlag, utils.TxLookupLimitFlag, + utils.TransactionHistoryFlag, + utils.StateSchemeFlag, + utils.StateHistoryFlag, utils.LightServeFlag, utils.LightIngressFlag, utils.LightEgressFlag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 6e4a27007c..ffae740a1f 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -71,10 +71,7 @@ two version states are available: genesis and the specific one. The default pruning target is the HEAD-127 state. -WARNING: It's necessary to delete the trie clean cache after the pruning. -If you specify another directory for the trie clean cache via "--cache.trie.journal" -during the use of Geth, please also specify it here for correct deletion. Otherwise -the trie clean cache with default directory will be deleted. +WARNING: it's only supported in hash mode(--state.scheme=hash)". `, }, { @@ -106,7 +103,9 @@ so it's very necessary to do block data prune, this feature will handle it. Usage: "Recalculate state hash based on the snapshot for verification", ArgsUsage: "", Action: verifyState, - Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge([]cli.Flag{ + utils.StateSchemeFlag, + }, utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot verify-state will traverse the whole accounts and storages set based on the specified @@ -164,7 +163,9 @@ information about the specified address. Usage: "Traverse the state with given root hash and perform quick verification", ArgsUsage: "", Action: traverseState, - Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge([]cli.Flag{ + utils.StateSchemeFlag, + }, utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-state will traverse the whole state from the given state root and will abort if any @@ -179,7 +180,9 @@ It's also usable without snapshot enabled. Usage: "Traverse the state with given root hash and perform detailed verification", ArgsUsage: "", Action: traverseRawState, - Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge([]cli.Flag{ + utils.StateSchemeFlag, + }, utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced @@ -201,6 +204,7 @@ It's also usable without snapshot enabled. utils.StartKeyFlag, utils.DumpLimitFlag, utils.TriesInMemoryFlag, + utils.StateSchemeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: ` This command is semantically equivalent to 'geth dump', but uses the snapshots @@ -235,7 +239,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) { NoBuild: true, AsyncBuild: false, } - snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headBlock.Root(), TriesInMemory, false) + snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb, nil), headBlock.Root(), TriesInMemory, false) if err != nil { log.Error("snaptree error", "err", err) return nil, err // The relevant snapshot(s) might not exist @@ -413,6 +417,9 @@ func pruneState(ctx *cli.Context) error { chaindb := utils.MakeChainDatabase(ctx, stack, false, false) defer chaindb.Close() + if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme { + log.Crit("Offline pruning is not required for path scheme") + } prunerconfig := pruner.Config{ Datadir: stack.ResolvePath(""), BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name), @@ -493,13 +500,16 @@ func verifyState(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - snapconfig := snapshot.Config{ + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + defer triedb.Close() + + snapConfig := snapshot.Config{ CacheSize: 256, Recovery: false, NoBuild: true, AsyncBuild: false, } - snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headBlock.Root(), 128, false) + snaptree, err := snapshot.New(snapConfig, chaindb, triedb, headBlock.Root(), 128, false) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err @@ -541,6 +551,11 @@ func traverseState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, true, false) + defer chaindb.Close() + + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + defer triedb.Close() + headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") @@ -565,7 +580,6 @@ func traverseState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } - triedb := trie.NewDatabase(chaindb) t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -641,6 +655,11 @@ func traverseRawState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, true, false) + defer chaindb.Close() + + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + defer triedb.Close() + headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") @@ -665,7 +684,6 @@ func traverseRawState(ctx *cli.Context) error { root = headBlock.Root() log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } - triedb := trie.NewDatabase(chaindb) t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) @@ -686,6 +704,11 @@ func traverseRawState(ctx *cli.Context) error { log.Error("Failed to open iterator", "root", root, "err", err) return err } + reader, err := triedb.Reader(root) + if err != nil { + log.Error("State is non-existent", "root", root) + return nil + } for accIter.Next(true) { nodes += 1 node := accIter.Hash() @@ -693,7 +716,7 @@ func traverseRawState(ctx *cli.Context) error { // Check the present for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { - blob := rawdb.ReadLegacyTrieNode(chaindb, node) + blob, _ := reader.Node(common.Hash{}, accIter.Path(), node) if len(blob) == 0 { log.Error("Missing trie node(account)", "hash", node) return errors.New("missing account") @@ -734,7 +757,7 @@ func traverseRawState(ctx *cli.Context) error { // Check the presence for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { - blob := rawdb.ReadLegacyTrieNode(chaindb, node) + blob, _ := reader.Node(common.BytesToHash(accIter.LeafKey()), storageIter.Path(), node) if len(blob) == 0 { log.Error("Missing trie node(storage)", "hash", node) return errors.New("missing storage") @@ -794,6 +817,9 @@ func dumpState(ctx *cli.Context) error { if err != nil { return err } + triedb := utils.MakeTrieDatabase(ctx, db, false, true) + defer triedb.Close() + snapConfig := snapshot.Config{ CacheSize: 256, Recovery: false, @@ -801,7 +827,7 @@ func dumpState(ctx *cli.Context) error { AsyncBuild: false, } triesInMemory := ctx.Uint64(utils.TriesInMemoryFlag.Name) - snaptree, err := snapshot.New(snapConfig, db, trie.NewDatabase(db), root, int(triesInMemory), false) + snaptree, err := snapshot.New(snapConfig, db, trie.NewDatabase(db, nil), root, int(triesInMemory), false) if err != nil { return err } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1522a2631a..8bff3d7fa9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -51,6 +51,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" @@ -75,6 +76,9 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // These are all the command line flags we support. @@ -234,30 +238,12 @@ var ( } defaultSyncMode = ethconfig.Defaults.SyncMode - SyncModeFlag = &flags.TextMarshalerFlag{ - Name: "syncmode", - Usage: `Blockchain sync mode ("snap", "full" or "light")`, - Value: &defaultSyncMode, - Category: flags.EthCategory, - } - GCModeFlag = &cli.StringFlag{ - Name: "gcmode", - Usage: `Blockchain garbage collection mode ("full", "archive")`, - Value: "full", - Category: flags.EthCategory, - } - SnapshotFlag = &cli.BoolFlag{ + SnapshotFlag = &cli.BoolFlag{ Name: "snapshot", Usage: `Enables snapshot-database mode (default = enable)`, Value: true, Category: flags.EthCategory, } - TxLookupLimitFlag = &cli.Uint64Flag{ - Name: "txlookuplimit", - Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", - Value: ethconfig.Defaults.TxLookupLimit, - Category: flags.EthCategory, - } LightKDFFlag = &cli.BoolFlag{ Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", @@ -305,6 +291,36 @@ var ( Usage: "Manually specify the Verkle fork timestamp, overriding the bundled setting", Category: flags.EthCategory, } + SyncModeFlag = &flags.TextMarshalerFlag{ + Name: "syncmode", + Usage: `Blockchain sync mode ("snap", "full" or "light")`, + Value: &defaultSyncMode, + Category: flags.StateCategory, + } + GCModeFlag = &cli.StringFlag{ + Name: "gcmode", + Usage: `Blockchain garbage collection mode, only relevant in state.scheme=hash ("full", "archive")`, + Value: "full", + Category: flags.StateCategory, + } + StateSchemeFlag = &cli.StringFlag{ + Name: "state.scheme", + Usage: "Scheme to use for storing ethereum state ('hash' or 'path')", + Value: rawdb.HashScheme, + Category: flags.StateCategory, + } + StateHistoryFlag = &cli.Uint64Flag{ + Name: "history.state", + Usage: "Number of recent blocks to retain state history for (default = 90,000 blocks, 0 = entire chain)", + Value: ethconfig.Defaults.StateHistory, + Category: flags.StateCategory, + } + TransactionHistoryFlag = &cli.Uint64Flag{ + Name: "history.transactions", + Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", + Value: ethconfig.Defaults.TransactionHistory, + Category: flags.StateCategory, + } // Light server and client settings LightServeFlag = &cli.IntFlag{ Name: "light.serve", @@ -1818,13 +1834,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { CheckExclusive(ctx, MainnetFlag, DeveloperFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer - if ctx.String(GCModeFlag.Name) == "archive" && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { - ctx.Set(TxLookupLimitFlag.Name, "0") - log.Warn("Disable transaction unindexing for archive node") - } - if ctx.IsSet(LightServeFlag.Name) && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { - log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") - } + + // Set configurations from CLI flags setEtherbase(ctx, cfg) setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light") setTxPool(ctx, &cfg.TxPool) @@ -1911,8 +1922,36 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") } - if ctx.IsSet(TxLookupLimitFlag.Name) { - cfg.TxLookupLimit = ctx.Uint64(TxLookupLimitFlag.Name) + if ctx.IsSet(StateHistoryFlag.Name) { + cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) + } + // Parse state scheme, abort the process if it's not compatible. + chaindb := tryMakeReadOnlyDatabase(ctx, stack) + scheme, err := ParseStateScheme(ctx, chaindb) + chaindb.Close() + if err != nil { + Fatalf("%v", err) + } + cfg.StateScheme = scheme + + // Parse transaction history flag, if user is still using legacy config + // file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'. + if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit { + log.Warn("The config option 'TxLookupLimit' is deprecated and will be removed, please use 'TransactionHistory'") + cfg.TransactionHistory = cfg.TxLookupLimit + } + if ctx.IsSet(TransactionHistoryFlag.Name) { + cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name) + } else if ctx.IsSet(TxLookupLimitFlag.Name) { + log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions") + cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name) + } + if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 { + cfg.TransactionHistory = 0 + log.Warn("Disabled transaction unindexing for archive node") + } + if ctx.IsSet(LightServeFlag.Name) && cfg.TransactionHistory != 0 { + log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") } if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 @@ -2296,6 +2335,18 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFree return chainDb } +// tryMakeReadOnlyDatabase try to open the chain database in read-only mode, +// or fallback to write mode if the database is not initialized. +func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { + // If datadir doesn't exist we need to open db in write-mode + // so database engine can create files. + readonly := true + if !common.FileExist(stack.ResolvePath("chaindata")) { + readonly = false + } + return MakeChainDatabase(ctx, stack, readonly, false) +} + func IsNetworkPreset(ctx *cli.Context) bool { for _, flag := range NetworkFlags { bFlag, _ := flag.(*cli.BoolFlag) @@ -2358,14 +2409,21 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } + scheme, err := ParseStateScheme(ctx, chainDb) + if err != nil { + Fatalf("%v", err) + } cache := &core.CacheConfig{ - TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, - TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, - TrieDirtyDisabled: ctx.String(GCModeFlag.Name) == "archive", - TrieTimeLimit: ethconfig.Defaults.TrieTimeout, - TriesInMemory: ethconfig.Defaults.TriesInMemory, - SnapshotLimit: ethconfig.Defaults.SnapshotCache, - Preimages: ctx.Bool(CachePreimagesFlag.Name), + TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, + TrieCleanNoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), + TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, + TrieDirtyDisabled: ctx.String(GCModeFlag.Name) == "archive", + TrieTimeLimit: ethconfig.Defaults.TrieTimeout, + TriesInMemory: ethconfig.Defaults.TriesInMemory, + SnapshotLimit: ethconfig.Defaults.SnapshotCache, + Preimages: ctx.Bool(CachePreimagesFlag.Name), + StateScheme: scheme, + StateHistory: ctx.Uint64(StateHistoryFlag.Name), } if cache.TrieDirtyDisabled && !cache.Preimages { cache.Preimages = true @@ -2413,3 +2471,62 @@ func MakeConsolePreloads(ctx *cli.Context) []string { } return preloads } + +// ParseStateScheme resolves scheme identifier from CLI flag. If the provided +// state scheme is not compatible with the one of persistent scheme, an error +// will be returned. +// +// - none: use the scheme consistent with persistent state, or fallback +// to hash-based scheme if state is empty. +// - hash: use hash-based scheme or error out if not compatible with +// persistent state scheme. +// - path: use path-based scheme or error out if not compatible with +// persistent state scheme. +func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) { + // If state scheme is not specified, use the scheme consistent + // with persistent state, or fallback to hash mode if database + // is empty. + stored := rawdb.ReadStateScheme(disk) + if !ctx.IsSet(StateSchemeFlag.Name) { + if stored == "" { + // use default scheme for empty database, flip it when + // path mode is chosen as default + log.Info("State schema set to default", "scheme", "hash") + return rawdb.HashScheme, nil + } + log.Info("State scheme set to already existing", "scheme", stored) + return stored, nil // reuse scheme of persistent scheme + } + // If state scheme is specified, ensure it's compatible with + // persistent state. + scheme := ctx.String(StateSchemeFlag.Name) + if stored == "" || scheme == stored { + log.Info("State scheme set by user", "scheme", scheme) + return scheme, nil + } + return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme) +} + +// MakeTrieDatabase constructs a trie database based on the configured scheme. +func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database { + config := &trie.Config{ + Preimages: preimage, + } + scheme, err := ParseStateScheme(ctx, disk) + if err != nil { + Fatalf("%v", err) + } + if scheme == rawdb.HashScheme { + // Read-only mode is not implemented in hash mode, + // ignore the parameter silently. TODO(rjl493456442) + // please config it if read mode is implemented. + config.HashDB = hashdb.Defaults + return trie.NewDatabase(disk, config) + } + if readOnly { + config.PathDB = pathdb.ReadOnly + } else { + config.PathDB = pathdb.Defaults + } + return trie.NewDatabase(disk, config) +} diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index cf51b51936..6669ff176f 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -19,6 +19,7 @@ package utils import ( "fmt" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/flags" "github.com/urfave/cli/v2" ) @@ -37,6 +38,7 @@ var DeprecatedFlags = []cli.Flag{ CacheTrieJournalFlag, CacheTrieRejournalFlag, LegacyDiscoveryV5Flag, + TxLookupLimitFlag, } var ( @@ -68,6 +70,13 @@ var ( Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism (deprecated, use --discv5 instead)", Category: flags.DeprecatedCategory, } + // Deprecated August 2023 + TxLookupLimitFlag = &cli.Uint64Flag{ + Name: "txlookuplimit", + Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain) (deprecated, use history.transactions instead)", + Value: ethconfig.Defaults.TransactionHistory, + Category: flags.DeprecatedCategory, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. diff --git a/core/block_validator_test.go b/core/block_validator_test.go index b623bfe055..614676c282 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -35,6 +35,11 @@ import ( // Tests that simple header verification works, for both good and bad blocks. func TestHeaderVerification(t *testing.T) { + testHeaderVerification(t, rawdb.HashScheme) + testHeaderVerification(t, rawdb.PathScheme) +} + +func testHeaderVerification(t *testing.T, scheme string) { // Create a simple chain to verify var ( gspec = &Genesis{Config: params.TestChainConfig} @@ -45,7 +50,7 @@ func TestHeaderVerification(t *testing.T) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer chain.Stop() for i := 0; i < len(blocks); i++ { diff --git a/core/blockchain.go b/core/blockchain.go index b278844f4b..08d30872f8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -52,6 +52,8 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "golang.org/x/exp/slices" ) @@ -140,7 +142,7 @@ const ( ) // CacheConfig contains the configuration values for the trie database -// that's resident in a blockchain. +// and state snapshot these are resident in a blockchain. type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks @@ -151,11 +153,31 @@ type CacheConfig struct { Preimages bool // Whether to store preimage of trie key to the disk TriesInMemory uint64 // How many tries keeps in memory NoTries bool // Insecure settings. Do not have any tries in databases if enabled. + StateHistory uint64 // Number of blocks from head whose state histories are reserved. + StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top SnapshotNoBuild bool // Whether the background generation is allowed SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } +// triedbConfig derives the configures for trie database. +func (c *CacheConfig) triedbConfig() *trie.Config { + config := &trie.Config{Preimages: c.Preimages} + if c.StateScheme == rawdb.HashScheme { + config.HashDB = &hashdb.Config{ + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + } + } + if c.StateScheme == rawdb.PathScheme { + config.PathDB = &pathdb.Config{ + StateHistory: c.StateHistory, + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024, + } + } + return config +} + // defaultCacheConfig are the default caching values if none are specified by the // user (also used during testing). var defaultCacheConfig = &CacheConfig{ @@ -165,6 +187,15 @@ var defaultCacheConfig = &CacheConfig{ SnapshotLimit: 256, TriesInMemory: 128, SnapshotWait: true, + StateScheme: rawdb.HashScheme, +} + +// DefaultCacheConfigWithScheme returns a deep copied default cache config with +// a provided trie node scheme. +func DefaultCacheConfigWithScheme(scheme string) *CacheConfig { + config := *defaultCacheConfig + config.StateScheme = scheme + return &config } type BlockChainOption func(*BlockChain) (*BlockChain, error) @@ -278,11 +309,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis diffLayerChanCache, _ := exlru.New(diffLayerCacheLimit) // Open trie database with provided config - triedb := trie.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Preimages: cacheConfig.Preimages, - NoTries: cacheConfig.NoTries, - }) + triedb := trie.NewDatabase(db, cacheConfig.triedbConfig()) // Setup the genesis block, commit the provided genesis specification // to database if the genesis block is not present yet, or load the // stored one from database. @@ -767,6 +794,25 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha pivot := rawdb.ReadLastPivotNumber(bc.db) frozen, _ := bc.db.Ancients() + // resetState resets the persistent state to genesis if it's not available. + resetState := func() { + // Short circuit if the genesis state is already present. + if bc.HasState(bc.genesisBlock.Root()) { + return + } + // Reset the state database to empty for committing genesis state. + // Note, it should only happen in path-based scheme and Reset function + // is also only call-able in this mode. + if bc.triedb.Scheme() == rawdb.PathScheme { + if err := bc.triedb.Reset(types.EmptyRootHash); err != nil { + log.Crit("Failed to clean state", "err", err) // Shouldn't happen + } + } + // Write genesis state into database. + if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil { + log.Crit("Failed to commit genesis state", "err", err) + } + } updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) { // Rewind the blockchain, ensuring we don't end up with a stateless head // block. Note, depth equality is permitted to allow using SetHead as a @@ -777,6 +823,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha if newHeadBlock == nil { log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash()) newHeadBlock = bc.genesisBlock + resetState() } else { // Block exists, keep rewinding until we find one with state, // keeping rewinding until we exceed the optional threshold @@ -790,10 +837,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root { beyondRoot, rootNumber = true, newHeadBlock.NumberU64() } - - enoughBeyondCount = beyondCount > maxBeyondBlocks - - if !bc.HasState(newHeadBlock.Root()) { + if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) { log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) if pivot == nil || newHeadBlock.NumberU64() > *pivot { parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) @@ -823,16 +867,12 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha } } if newHeadBlock.NumberU64() == 0 { - // Recommit the genesis state into disk in case the rewinding destination - // is genesis block and the relevant state is gone. In the future this - // rewinding destination can be the earliest block stored in the chain - // if the historical chain pruning is enabled. In that case the logic - // needs to be improved here. - if !bc.HasState(bc.genesisBlock.Root()) { - if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil { - log.Crit("Failed to commit genesis state", "err", err) - } - log.Debug("Recommitted genesis state to disk") + resetState() + } else if !bc.HasState(newHeadBlock.Root()) { + // Rewind to a block with recoverable state. If the state is + // missing, run the state recovery here. + if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil { + log.Crit("Failed to rollback state", "err", err) // Shouldn't happen } } log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) @@ -939,7 +979,13 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } + // Reset the trie database with the fresh snap synced state. root := block.Root() + if bc.triedb.Scheme() == rawdb.PathScheme { + if err := bc.triedb.Reset(root); err != nil { + return err + } + } if !bc.HasState(root) { return fmt.Errorf("non existent state [%x..]", root[:4]) } @@ -1115,45 +1161,57 @@ func (bc *BlockChain) Stop() { log.Error("Failed to journal state snapshot", "err", err) } } + if bc.triedb.Scheme() == rawdb.PathScheme { + // Ensure that the in-memory trie nodes are journaled to disk properly. + if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil { + log.Info("Failed to journal in-memory trie nodes", "err", err) + } + } else { + // Ensure the state of a recent block is also stored to disk before exiting. + // We're writing three different states to catch different restart scenarios: + // - HEAD: So we don't need to reprocess any blocks in the general case + // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle + // - HEAD-127: So we have a hard limit on the number of blocks reexecuted + if !bc.cacheConfig.TrieDirtyDisabled { + triedb := bc.triedb + + for _, offset := range []uint64{0, 1, TriesInMemory - 1} { + if number := bc.CurrentBlock().Number.Uint64(); number > offset { + recent := bc.GetBlockByNumber(number - offset) + log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) + if err := triedb.Commit(recent.Root(), true); err != nil { + log.Error("Failed to commit recent state trie", "err", err) + } + } + } + if snapBase != (common.Hash{}) { + log.Info("Writing snapshot state to disk", "root", snapBase) + if err := triedb.Commit(snapBase, true); err != nil { + log.Error("Failed to commit recent state trie", "err", err) + } else { + rawdb.WriteSafePointBlockNumber(bc.db, bc.CurrentBlock().Number.Uint64()) + } + } - // Ensure the state of a recent block is also stored to disk before exiting. - // We're writing three different states to catch different restart scenarios: - // - HEAD: So we don't need to reprocess any blocks in the general case - // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle - // - HEAD-127: So we have a hard limit on the number of blocks reexecuted - if !bc.cacheConfig.TrieDirtyDisabled { - triedb := bc.triedb - - for _, offset := range []uint64{0, 1, bc.triesInMemory - 1} { - if number := bc.CurrentBlock().Number.Uint64(); number > offset { - recent := bc.GetBlockByNumber(number - offset) - - log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) - if err := triedb.Commit(recent.Root(), true); err != nil { + if snapBase != (common.Hash{}) { + log.Info("Writing snapshot state to disk", "root", snapBase) + if err := bc.triedb.Commit(snapBase, true); err != nil { log.Error("Failed to commit recent state trie", "err", err) } else { - rawdb.WriteSafePointBlockNumber(bc.db, recent.NumberU64()) + rawdb.WriteSafePointBlockNumber(bc.db, bc.CurrentBlock().Number.Uint64()) } } - } - if snapBase != (common.Hash{}) { - log.Info("Writing snapshot state to disk", "root", snapBase) - if err := triedb.Commit(snapBase, true); err != nil { - log.Error("Failed to commit recent state trie", "err", err) - } else { - rawdb.WriteSafePointBlockNumber(bc.db, bc.CurrentBlock().Number.Uint64()) + for !bc.triegc.Empty() { + go triedb.Dereference(bc.triegc.PopItem()) + } + if size, _ := triedb.Size(); size != 0 { + log.Error("Dangling trie nodes after full cleanup") } - } - for !bc.triegc.Empty() { - go triedb.Dereference(bc.triegc.PopItem()) - } - if size, _ := triedb.Size(); size != 0 { - log.Error("Dangling trie nodes after full cleanup") } } - // Flush the collected preimages to disk - if err := bc.stateCache.TrieDB().Close(); err != nil { - log.Error("Failed to close trie db", "err", err) + // Close the trie database, release all the held resources as the last step. + if err := bc.triedb.Close(); err != nil { + log.Error("Failed to close trie database", "err", err) } log.Info("Blockchain stopped") } @@ -1529,6 +1587,12 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.commitLock.Lock() defer bc.commitLock.Unlock() + // If node is running in path mode, skip explicit gc operation + // which is unnecessary in this mode. + if bc.triedb.Scheme() == rawdb.PathScheme { + return nil + } + triedb := bc.stateCache.TrieDB() // If we're running an archive node, always flush if bc.cacheConfig.TrieDirtyDisabled { @@ -2204,6 +2268,12 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i ) parent := it.previous() for parent != nil && !bc.HasState(parent.Root) { + if bc.stateRecoverable(parent.Root) { + if err := bc.triedb.Recover(parent.Root); err != nil { + return 0, err + } + break + } hashes = append(hashes, parent.Hash()) numbers = append(numbers, parent.Number.Uint64()) @@ -2263,6 +2333,12 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error) parent = block ) for parent != nil && !bc.HasState(parent.Root()) { + if bc.stateRecoverable(parent.Root()) { + if err := bc.triedb.Recover(parent.Root()); err != nil { + return common.Hash{}, err + } + break + } hashes = append(hashes, parent.Hash()) numbers = append(numbers, parent.NumberU64()) parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1) @@ -2776,6 +2852,7 @@ func (bc *BlockChain) maintainTxIndex() { return } defer sub.Unsubscribe() + log.Info("Initialized transaction indexer", "limit", bc.TxLookupLimit()) for { select { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 1cae3a0545..802b979a10 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -319,10 +319,16 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { return bc.HasState(block.Root()) } -// TrieNode retrieves a blob of data associated with a trie node -// either from ephemeral in-memory cache, or from persistent storage. -func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { - return bc.stateCache.TrieDB().Node(hash) +// stateRecoverable checks if the specified state is recoverable. +// Note, this function assumes the state is not present, because +// state is not treated as recoverable if it's available, thus +// false will be returned in this case. +func (bc *BlockChain) stateRecoverable(root common.Hash) bool { + if bc.triedb.Scheme() == rawdb.HashScheme { + return false + } + result, _ := bc.triedb.Recoverable(root) + return result } // ContractCodeWithPrefix retrieves a blob of data associated with a contract diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 439a3d706f..acef2e2d59 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -22,6 +22,7 @@ package core import ( "math/big" + "path" "testing" "time" @@ -1749,16 +1750,23 @@ func testLongReorgedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { } func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + testRepairWithScheme(t, tt, snapshots, scheme) + } +} + +func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(true)) // Create a temporary persistent database datadir := t.TempDir() + ancient := path.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) @@ -1777,6 +1785,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, // Disable snapshot by default + StateScheme: scheme, } ) defer engine.Close() @@ -1807,7 +1816,9 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { t.Fatalf("Failed to import canonical chain start: %v", err) } if tt.commitBlock > 0 { - chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false) + if err := chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false); err != nil { + t.Fatalf("Failed to flush trie state: %v", err) + } if snapshots { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { t.Fatalf("Failed to flatten snapshots: %v", err) @@ -1830,21 +1841,21 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { rawdb.WriteLastPivotNumber(db, *tt.pivotBlock) } // Pull the plug on the database, simulating a hard crash + chain.triedb.Close() db.Close() chain.stopWithoutSaving() // Start a new blockchain back up and see where the repair leads us db, err = rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, }) - if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } defer db.Close() - newChain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + newChain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -1887,17 +1898,22 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { // In this case the snapshot layer of B3 is not created because of existent // state. func TestIssue23496(t *testing.T) { + testIssue23496(t, rawdb.HashScheme) + testIssue23496(t, rawdb.PathScheme) +} + +func testIssue23496(t *testing.T, scheme string) { // It's hard to follow the test case, visualize the input //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // Create a temporary persistent database datadir := t.TempDir() + ancient := path.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, }) - if err != nil { t.Fatalf("Failed to create persistent database: %v", err) } @@ -1910,16 +1926,8 @@ func TestIssue23496(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } engine = ethash.NewFullFaker() - config = &CacheConfig{ - TrieCleanLimit: 256, - TrieDirtyLimit: 256, - TriesInMemory: 128, - TrieTimeLimit: 5 * time.Minute, - SnapshotLimit: 256, - SnapshotWait: true, - } ) - chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } @@ -1932,7 +1940,7 @@ func TestIssue23496(t *testing.T) { if _, err := chain.InsertChain(blocks[:1]); err != nil { t.Fatalf("Failed to import canonical chain start: %v", err) } - chain.stateCache.TrieDB().Commit(blocks[0].Root(), false) + chain.triedb.Commit(blocks[0].Root(), false) // Insert block B2 and commit the snapshot into disk if _, err := chain.InsertChain(blocks[1:2]); err != nil { @@ -1946,7 +1954,7 @@ func TestIssue23496(t *testing.T) { if _, err := chain.InsertChain(blocks[2:3]); err != nil { t.Fatalf("Failed to import canonical chain start: %v", err) } - chain.stateCache.TrieDB().Commit(blocks[2].Root(), false) + chain.triedb.Commit(blocks[2].Root(), false) // Insert the remaining blocks if _, err := chain.InsertChain(blocks[3:]); err != nil { @@ -1954,20 +1962,21 @@ func TestIssue23496(t *testing.T) { } // Pull the plug on the database, simulating a hard crash + chain.triedb.Close() db.Close() chain.stopWithoutSaving() // Start a new blockchain back up and see where the repair leads us db, err = rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, }) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } defer db.Close() - chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -1979,8 +1988,12 @@ func TestIssue23496(t *testing.T) { if head := chain.CurrentSnapBlock(); head.Number.Uint64() != uint64(4) { t.Errorf("Head fast block mismatch: have %d, want %d", head.Number, uint64(4)) } - if head := chain.CurrentBlock(); head.Number.Uint64() != uint64(1) { - t.Errorf("Head block mismatch: have %d, want %d", head.Number, uint64(1)) + expHead := uint64(1) + if scheme == rawdb.PathScheme { + expHead = uint64(2) + } + if head := chain.CurrentBlock(); head.Number.Uint64() != expHead { + t.Errorf("Head block mismatch: have %d, want %d", head.Number, expHead) } // Reinsert B2-B4 diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index b34b14b838..72c1e56db2 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -22,6 +22,7 @@ package core import ( "fmt" "math/big" + "path" "strings" "testing" "time" @@ -29,9 +30,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // rewindTest is a test case for chain rollback upon user request. @@ -1949,16 +1954,23 @@ func testLongReorgedSnapSyncingDeepSetHead(t *testing.T, snapshots bool) { } func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + testSetHeadWithScheme(t, tt, snapshots, scheme) + } +} + +func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(false)) // Create a temporary persistent database datadir := t.TempDir() + ancient := path.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) @@ -1977,6 +1989,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, // Disable snapshot + StateScheme: scheme, } ) if snapshots { @@ -2008,7 +2021,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { t.Fatalf("Failed to import canonical chain start: %v", err) } if tt.commitBlock > 0 { - chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false) + chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false) if snapshots { if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil { t.Fatalf("Failed to flatten snapshots: %v", err) @@ -2018,14 +2031,17 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { if _, err := chain.InsertChain(canonblocks[tt.commitBlock:]); err != nil { t.Fatalf("Failed to import canonical chain tail: %v", err) } - // Manually dereference anything not committed to not have to work with 128+ tries - for _, block := range sideblocks { - chain.stateCache.TrieDB().Dereference(block.Root()) - } - for _, block := range canonblocks { - chain.stateCache.TrieDB().Dereference(block.Root()) + // Reopen the trie database without persisting in-memory dirty nodes. + chain.triedb.Close() + dbconfig := &trie.Config{} + if scheme == rawdb.PathScheme { + dbconfig.PathDB = pathdb.Defaults + } else { + dbconfig.HashDB = hashdb.Defaults } - chain.stateCache.Purge() + chain.triedb = trie.NewDatabase(chain.db, dbconfig) + chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb) + // Force run a freeze cycle type freezer interface { Freeze(threshold uint64) error diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 4f02c43051..f0000eaded 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -24,6 +24,7 @@ import ( "fmt" "math/big" "os" + "path" "strings" "testing" "time" @@ -39,6 +40,7 @@ import ( // snapshotTestBasic wraps the common testing fields in the snapshot tests. type snapshotTestBasic struct { + scheme string // Disk scheme used for storing trie nodes chainBlocks int // Number of blocks to generate for the canonical chain snapshotBlock uint64 // Block number of the relevant snapshot disk layer commitBlock uint64 // Block number for which to commit the state to disk @@ -51,6 +53,7 @@ type snapshotTestBasic struct { // share fields, set in runtime datadir string + ancient string db ethdb.Database genDb ethdb.Database engine consensus.Engine @@ -60,10 +63,11 @@ type snapshotTestBasic struct { func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { // Create a temporary persistent database datadir := t.TempDir() + ancient := path.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, - AncientsDirectory: datadir, + AncientsDirectory: ancient, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) @@ -75,13 +79,8 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo Config: params.AllEthashProtocolChanges, } engine = ethash.NewFullFaker() - - // Snapshot is enabled, the first snapshot is created from the Genesis. - // The snapshot memory allowance is 256MB, it means no snapshot flush - // will happen during the block insertion. - cacheConfig = defaultCacheConfig ) - chain, err := NewBlockChain(db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(basic.scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } @@ -102,7 +101,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo startPoint = point if basic.commitBlock > 0 && basic.commitBlock == point { - chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), false) + chain.TrieDB().Commit(blocks[point-1].Root(), false) } if basic.snapshotBlock > 0 && basic.snapshotBlock == point { // Flushing the entire snap tree into the disk, the @@ -121,6 +120,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo // Set runtime fields basic.datadir = datadir + basic.ancient = ancient basic.db = db basic.genDb = genDb basic.engine = engine @@ -210,6 +210,7 @@ func (basic *snapshotTestBasic) teardown() { basic.db.Close() basic.genDb.Close() os.RemoveAll(basic.datadir) + os.RemoveAll(basic.ancient) } // snapshotTest is a test case type for normal snapshot recovery. @@ -226,7 +227,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { // Restart the chain normally chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -235,7 +236,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { snaptest.verify(t, newchain, blocks) } -// crashSnapshotTest is a test case type for innormal snapshot recovery. +// crashSnapshotTest is a test case type for irregular snapshot recovery. // It can be used for testing that restart Geth after the crash. type crashSnapshotTest struct { snapshotTestBasic @@ -251,13 +252,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { db := chain.db db.Close() chain.stopWithoutSaving() + chain.triedb.Close() // Start a new blockchain back up and see where the repair leads us newdb, err := rawdb.Open(rawdb.OpenOptions{ Directory: snaptest.datadir, - AncientsDirectory: snaptest.datadir, + AncientsDirectory: snaptest.ancient, }) - if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) } @@ -267,13 +268,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { // the crash, we do restart twice here: one after the crash and one // after the normal stop. It's used to ensure the broken snapshot // can be detected all the time. - newchain, err := NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } newchain.Stop() - newchain, err = NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -300,7 +301,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { // Insert blocks without enabling snapshot if gapping is required. chain.Stop() - gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {}) + gappedBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {}) // Insert a few more blocks without enabling snapshot var cacheConfig = &CacheConfig{ @@ -309,6 +310,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, TriesInMemory: 128, SnapshotLimit: 0, + StateScheme: snaptest.scheme, } newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { @@ -318,7 +320,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { newchain.Stop() // Restart the chain with enabling the snapshot - newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -346,7 +348,7 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) { chain.SetHead(snaptest.setHead) chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -381,16 +383,17 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, TriesInMemory: 128, + StateScheme: snaptest.scheme, } newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {}) + newBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {}) newchain.InsertChain(newBlocks) newchain.Stop() - // Restart the chain, the wiper should starts working + // Restart the chain, the wiper should start working config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, @@ -398,6 +401,7 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { SnapshotLimit: 256, SnapshotWait: false, // Don't wait rebuild TriesInMemory: 128, + StateScheme: snaptest.scheme, } tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { @@ -405,14 +409,15 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { } // Simulate the blockchain crash. + tmp.triedb.Close() tmp.stopWithoutSaving() - newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } - defer newchain.Stop() snaptest.verify(t, newchain, blocks) + newchain.Stop() } // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot @@ -436,20 +441,23 @@ func TestRestartWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C8 // Expected snapshot disk : G - test := &snapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 8, - expSnapshotBottom: 0, // Initial disk layer built from genesis - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &snapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 8, + expSnapshotBottom: 0, // Initial disk layer built from genesis + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -475,20 +483,23 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 0, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 0, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -514,20 +525,23 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C2 // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 2, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 2, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 2, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: 2, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was crashed and restarts with a broken snapshot. In this case @@ -553,20 +567,27 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 6, - expCanonicalBlocks: 8, - expHeadHeader: 8, - expHeadFastBlock: 8, - expHeadBlock: 0, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + expHead := uint64(0) + if scheme == rawdb.PathScheme { + expHead = uint64(4) + } + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 6, + expCanonicalBlocks: 8, + expHeadHeader: 8, + expHeadFastBlock: 8, + expHeadBlock: expHead, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() + } } // Tests a Geth was running with snapshot enabled. Then restarts without @@ -590,21 +611,24 @@ func TestGappedNewSnapshot(t *testing.T) { // Expected head fast block: C10 // Expected head block : C10 // Expected snapshot disk : C10 - test := &gappedSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD - }, - gapped: 2, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &gappedSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 10, + expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD + }, + gapped: 2, + } + test.test(t) + test.teardown() + } } // Tests the Geth was running with snapshot enabled and resetHead is applied. @@ -628,21 +652,24 @@ func TestSetHeadWithNewSnapshot(t *testing.T) { // Expected head fast block: C4 // Expected head block : C4 // Expected snapshot disk : G - test := &setHeadSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 0, - commitBlock: 0, - expCanonicalBlocks: 4, - expHeadHeader: 4, - expHeadFastBlock: 4, - expHeadBlock: 4, - expSnapshotBottom: 0, // The initial disk layer is built from the genesis - }, - setHead: 4, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &setHeadSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 0, + commitBlock: 0, + expCanonicalBlocks: 4, + expHeadHeader: 4, + expHeadFastBlock: 4, + expHeadBlock: 4, + expSnapshotBottom: 0, // The initial disk layer is built from the genesis + }, + setHead: 4, + } + test.test(t) + test.teardown() + } } // Tests the Geth was running with a complete snapshot and then imports a few @@ -666,19 +693,22 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) { // Expected head fast block: C10 // Expected head block : C8 // Expected snapshot disk : C10 - test := &wipeCrashSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - commitBlock: 0, - expCanonicalBlocks: 10, - expHeadHeader: 10, - expHeadFastBlock: 10, - expHeadBlock: 10, - expSnapshotBottom: 10, - }, - newBlocks: 2, - } - test.test(t) - test.teardown() + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &wipeCrashSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + commitBlock: 0, + expCanonicalBlocks: 10, + expHeadHeader: 10, + expHeadFastBlock: 10, + expHeadBlock: 10, + expSnapshotBottom: 10, + }, + newBlocks: 2, + } + test.test(t) + test.teardown() + } } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 56a5c7763e..56b67eedaf 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -55,13 +55,22 @@ var ( // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. The database and genesis specification for block generation // are also returned in case more test blocks are needed later. +<<<<<<< HEAD func newCanonical(engine consensus.Engine, n int, full bool, pipeline bool) (ethdb.Database, *Genesis, *BlockChain, error) { +======= +func newCanonical(engine consensus.Engine, n int, full bool, scheme string) (ethdb.Database, *Genesis, *BlockChain, error) { +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) var ( genesis = &Genesis{ BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges, } ) +<<<<<<< HEAD +======= + // Initialize a fresh chain with only a genesis block + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Initialize a fresh chain with only a genesis block var ops []BlockChainOption @@ -90,6 +99,7 @@ func newGwei(n int64) *big.Int { } // Test fork of length N starting from block i +<<<<<<< HEAD func testInvalidStateRootBlockImport(t *testing.T, blockchain *BlockChain, i, n int, pipeline bool) { // Copy old chain up to #i into a new db db, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, true, pipeline) @@ -137,6 +147,11 @@ func testInvalidStateRootBlockImport(t *testing.T, blockchain *BlockChain, i, n func testFork(t *testing.T, blockchain *BlockChain, i, n int, full, pipeline bool, comparator func(td1, td2 *big.Int)) { // Copy old chain up to #i into a new db genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, pipeline) +======= +func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int), scheme string) { + // Copy old chain up to #i into a new db + genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatal("could not make new canonical in testFork", err) } @@ -255,6 +270,7 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error } return nil } +<<<<<<< HEAD func TestBlockImportVerification(t *testing.T) { length := 5 @@ -272,6 +288,15 @@ func TestBlockImportVerification(t *testing.T) { func TestLastBlock(t *testing.T) { genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, false) +======= +func TestLastBlock(t *testing.T) { + testLastBlock(t, rawdb.HashScheme) + testLastBlock(t, rawdb.PathScheme) +} + +func testLastBlock(t *testing.T, scheme string) { + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -288,9 +313,13 @@ func TestLastBlock(t *testing.T) { // Test inserts the blocks/headers after the fork choice rule is changed. // The chain is reorged to whatever specified. -func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full bool) { +func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full bool, scheme string) { // Copy old chain up to #i into a new db +<<<<<<< HEAD genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, false) +======= + genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatal("could not make new canonical in testFork", err) } @@ -338,6 +367,7 @@ func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full b // Tests that given a starting canonical chain of a given size, it can be extended // with various length chains. func TestExtendCanonicalHeaders(t *testing.T) { +<<<<<<< HEAD testExtendCanonical(t, false, false) } @@ -351,6 +381,21 @@ func testExtendCanonical(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) +======= + testExtendCanonical(t, false, rawdb.HashScheme) + testExtendCanonical(t, false, rawdb.PathScheme) +} +func TestExtendCanonicalBlocks(t *testing.T) { + testExtendCanonical(t, true, rawdb.HashScheme) + testExtendCanonical(t, true, rawdb.PathScheme) +} + +func testExtendCanonical(t *testing.T, full bool, scheme string) { + length := 5 + + // Make first chain starting from genesis + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -363,33 +408,51 @@ func testExtendCanonical(t *testing.T, full, pipeline bool) { } } // Start fork from current height +<<<<<<< HEAD testFork(t, processor, length, 1, full, pipeline, better) testFork(t, processor, length, 2, full, pipeline, better) testFork(t, processor, length, 5, full, pipeline, better) testFork(t, processor, length, 10, full, pipeline, better) +======= + testFork(t, processor, length, 1, full, better, scheme) + testFork(t, processor, length, 2, full, better, scheme) + testFork(t, processor, length, 5, full, better, scheme) + testFork(t, processor, length, 10, full, better, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that given a starting canonical chain of a given size, it can be extended // with various length chains. -func TestExtendCanonicalHeadersAfterMerge(t *testing.T) { testExtendCanonicalAfterMerge(t, false) } -func TestExtendCanonicalBlocksAfterMerge(t *testing.T) { testExtendCanonicalAfterMerge(t, true) } +func TestExtendCanonicalHeadersAfterMerge(t *testing.T) { + testExtendCanonicalAfterMerge(t, false, rawdb.HashScheme) + testExtendCanonicalAfterMerge(t, false, rawdb.PathScheme) +} +func TestExtendCanonicalBlocksAfterMerge(t *testing.T) { + testExtendCanonicalAfterMerge(t, true, rawdb.HashScheme) + testExtendCanonicalAfterMerge(t, true, rawdb.PathScheme) +} -func testExtendCanonicalAfterMerge(t *testing.T, full bool) { +func testExtendCanonicalAfterMerge(t *testing.T, full bool, scheme string) { length := 5 // Make first chain starting from genesis +<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) +======= + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, length, 1, full) - testInsertAfterMerge(t, processor, length, 10, full) + testInsertAfterMerge(t, processor, length, 1, full, scheme) + testInsertAfterMerge(t, processor, length, 10, full, scheme) } // Tests that given a starting canonical chain of a given size, creating shorter // forks do not take canonical ownership. +<<<<<<< HEAD func TestShorterForkHeaders(t *testing.T) { testShorterFork(t, false, false) } func TestShorterForkBlocks(t *testing.T) { testShorterFork(t, true, false) @@ -401,6 +464,22 @@ func testShorterFork(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) +======= +func TestShorterForkHeaders(t *testing.T) { + testShorterFork(t, false, rawdb.HashScheme) + testShorterFork(t, false, rawdb.PathScheme) +} +func TestShorterForkBlocks(t *testing.T) { + testShorterFork(t, true, rawdb.HashScheme) + testShorterFork(t, true, rawdb.PathScheme) +} + +func testShorterFork(t *testing.T, full bool, scheme string) { + length := 10 + + // Make first chain starting from genesis + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -413,40 +492,60 @@ func testShorterFork(t *testing.T, full, pipeline bool) { } } // Sum of numbers must be less than `length` for this to be a shorter fork +<<<<<<< HEAD testFork(t, processor, 0, 3, full, pipeline, worse) testFork(t, processor, 0, 7, full, pipeline, worse) testFork(t, processor, 1, 1, full, pipeline, worse) testFork(t, processor, 1, 7, full, pipeline, worse) testFork(t, processor, 5, 3, full, pipeline, worse) testFork(t, processor, 5, 4, full, pipeline, worse) +======= + testFork(t, processor, 0, 3, full, worse, scheme) + testFork(t, processor, 0, 7, full, worse, scheme) + testFork(t, processor, 1, 1, full, worse, scheme) + testFork(t, processor, 1, 7, full, worse, scheme) + testFork(t, processor, 5, 3, full, worse, scheme) + testFork(t, processor, 5, 4, full, worse, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that given a starting canonical chain of a given size, creating shorter // forks do not take canonical ownership. -func TestShorterForkHeadersAfterMerge(t *testing.T) { testShorterForkAfterMerge(t, false) } -func TestShorterForkBlocksAfterMerge(t *testing.T) { testShorterForkAfterMerge(t, true) } +func TestShorterForkHeadersAfterMerge(t *testing.T) { + testShorterForkAfterMerge(t, false, rawdb.HashScheme) + testShorterForkAfterMerge(t, false, rawdb.PathScheme) +} +func TestShorterForkBlocksAfterMerge(t *testing.T) { + testShorterForkAfterMerge(t, true, rawdb.HashScheme) + testShorterForkAfterMerge(t, true, rawdb.PathScheme) +} -func testShorterForkAfterMerge(t *testing.T, full bool) { +func testShorterForkAfterMerge(t *testing.T, full bool, scheme string) { length := 10 // Make first chain starting from genesis +<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) +======= + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 3, full) - testInsertAfterMerge(t, processor, 0, 7, full) - testInsertAfterMerge(t, processor, 1, 1, full) - testInsertAfterMerge(t, processor, 1, 7, full) - testInsertAfterMerge(t, processor, 5, 3, full) - testInsertAfterMerge(t, processor, 5, 4, full) + testInsertAfterMerge(t, processor, 0, 3, full, scheme) + testInsertAfterMerge(t, processor, 0, 7, full, scheme) + testInsertAfterMerge(t, processor, 1, 1, full, scheme) + testInsertAfterMerge(t, processor, 1, 7, full, scheme) + testInsertAfterMerge(t, processor, 5, 3, full, scheme) + testInsertAfterMerge(t, processor, 5, 4, full, scheme) } // Tests that given a starting canonical chain of a given size, creating longer // forks do take canonical ownership. func TestLongerForkHeaders(t *testing.T) { +<<<<<<< HEAD testLongerFork(t, false, false) } func TestLongerForkBlocks(t *testing.T) { @@ -459,44 +558,70 @@ func testLongerFork(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) +======= + testLongerFork(t, false, rawdb.HashScheme) + testLongerFork(t, false, rawdb.PathScheme) +} +func TestLongerForkBlocks(t *testing.T) { + testLongerFork(t, true, rawdb.HashScheme) + testLongerFork(t, true, rawdb.PathScheme) +} + +func testLongerFork(t *testing.T, full bool, scheme string) { + length := 10 + + // Make first chain starting from genesis + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 11, full) - testInsertAfterMerge(t, processor, 0, 15, full) - testInsertAfterMerge(t, processor, 1, 10, full) - testInsertAfterMerge(t, processor, 1, 12, full) - testInsertAfterMerge(t, processor, 5, 6, full) - testInsertAfterMerge(t, processor, 5, 8, full) + testInsertAfterMerge(t, processor, 0, 11, full, scheme) + testInsertAfterMerge(t, processor, 0, 15, full, scheme) + testInsertAfterMerge(t, processor, 1, 10, full, scheme) + testInsertAfterMerge(t, processor, 1, 12, full, scheme) + testInsertAfterMerge(t, processor, 5, 6, full, scheme) + testInsertAfterMerge(t, processor, 5, 8, full, scheme) } // Tests that given a starting canonical chain of a given size, creating longer // forks do take canonical ownership. -func TestLongerForkHeadersAfterMerge(t *testing.T) { testLongerForkAfterMerge(t, false) } -func TestLongerForkBlocksAfterMerge(t *testing.T) { testLongerForkAfterMerge(t, true) } +func TestLongerForkHeadersAfterMerge(t *testing.T) { + testLongerForkAfterMerge(t, false, rawdb.HashScheme) + testLongerForkAfterMerge(t, false, rawdb.PathScheme) +} +func TestLongerForkBlocksAfterMerge(t *testing.T) { + testLongerForkAfterMerge(t, true, rawdb.HashScheme) + testLongerForkAfterMerge(t, true, rawdb.PathScheme) +} -func testLongerForkAfterMerge(t *testing.T, full bool) { +func testLongerForkAfterMerge(t *testing.T, full bool, scheme string) { length := 10 // Make first chain starting from genesis +<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) +======= + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 11, full) - testInsertAfterMerge(t, processor, 0, 15, full) - testInsertAfterMerge(t, processor, 1, 10, full) - testInsertAfterMerge(t, processor, 1, 12, full) - testInsertAfterMerge(t, processor, 5, 6, full) - testInsertAfterMerge(t, processor, 5, 8, full) + testInsertAfterMerge(t, processor, 0, 11, full, scheme) + testInsertAfterMerge(t, processor, 0, 15, full, scheme) + testInsertAfterMerge(t, processor, 1, 10, full, scheme) + testInsertAfterMerge(t, processor, 1, 12, full, scheme) + testInsertAfterMerge(t, processor, 5, 6, full, scheme) + testInsertAfterMerge(t, processor, 5, 8, full, scheme) } // Tests that given a starting canonical chain of a given size, creating equal // forks do take canonical ownership. +<<<<<<< HEAD func TestEqualForkHeaders(t *testing.T) { testEqualFork(t, false, false) } func TestEqualForkBlocks(t *testing.T) { testEqualFork(t, true, true) @@ -508,6 +633,22 @@ func testEqualFork(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) +======= +func TestEqualForkHeaders(t *testing.T) { + testEqualFork(t, false, rawdb.HashScheme) + testEqualFork(t, false, rawdb.PathScheme) +} +func TestEqualForkBlocks(t *testing.T) { + testEqualFork(t, true, rawdb.HashScheme) + testEqualFork(t, true, rawdb.PathScheme) +} + +func testEqualFork(t *testing.T, full bool, scheme string) { + length := 10 + + // Make first chain starting from genesis + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -520,38 +661,58 @@ func testEqualFork(t *testing.T, full, pipeline bool) { } } // Sum of numbers must be equal to `length` for this to be an equal fork +<<<<<<< HEAD testFork(t, processor, 0, 10, full, pipeline, equal) testFork(t, processor, 1, 9, full, pipeline, equal) testFork(t, processor, 2, 8, full, pipeline, equal) testFork(t, processor, 5, 5, full, pipeline, equal) testFork(t, processor, 6, 4, full, pipeline, equal) testFork(t, processor, 9, 1, full, pipeline, equal) +======= + testFork(t, processor, 0, 10, full, equal, scheme) + testFork(t, processor, 1, 9, full, equal, scheme) + testFork(t, processor, 2, 8, full, equal, scheme) + testFork(t, processor, 5, 5, full, equal, scheme) + testFork(t, processor, 6, 4, full, equal, scheme) + testFork(t, processor, 9, 1, full, equal, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that given a starting canonical chain of a given size, creating equal // forks do take canonical ownership. -func TestEqualForkHeadersAfterMerge(t *testing.T) { testEqualForkAfterMerge(t, false) } -func TestEqualForkBlocksAfterMerge(t *testing.T) { testEqualForkAfterMerge(t, true) } +func TestEqualForkHeadersAfterMerge(t *testing.T) { + testEqualForkAfterMerge(t, false, rawdb.HashScheme) + testEqualForkAfterMerge(t, false, rawdb.PathScheme) +} +func TestEqualForkBlocksAfterMerge(t *testing.T) { + testEqualForkAfterMerge(t, true, rawdb.HashScheme) + testEqualForkAfterMerge(t, true, rawdb.PathScheme) +} -func testEqualForkAfterMerge(t *testing.T, full bool) { +func testEqualForkAfterMerge(t *testing.T, full bool, scheme string) { length := 10 // Make first chain starting from genesis +<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) +======= + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 10, full) - testInsertAfterMerge(t, processor, 1, 9, full) - testInsertAfterMerge(t, processor, 2, 8, full) - testInsertAfterMerge(t, processor, 5, 5, full) - testInsertAfterMerge(t, processor, 6, 4, full) - testInsertAfterMerge(t, processor, 9, 1, full) + testInsertAfterMerge(t, processor, 0, 10, full, scheme) + testInsertAfterMerge(t, processor, 1, 9, full, scheme) + testInsertAfterMerge(t, processor, 2, 8, full, scheme) + testInsertAfterMerge(t, processor, 5, 5, full, scheme) + testInsertAfterMerge(t, processor, 6, 4, full, scheme) + testInsertAfterMerge(t, processor, 9, 1, full, scheme) } // Tests that chains missing links do not get accepted by the processor. +<<<<<<< HEAD func TestBrokenHeaderChain(t *testing.T) { testBrokenChain(t, false, false) } func TestBrokenBlockChain(t *testing.T) { testBrokenChain(t, true, false) @@ -561,6 +722,20 @@ func TestBrokenBlockChain(t *testing.T) { func testBrokenChain(t *testing.T, full, pipeline bool) { // Make chain starting from genesis genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 10, full, pipeline) +======= +func TestBrokenHeaderChain(t *testing.T) { + testBrokenChain(t, false, rawdb.HashScheme) + testBrokenChain(t, false, rawdb.PathScheme) +} +func TestBrokenBlockChain(t *testing.T) { + testBrokenChain(t, true, rawdb.HashScheme) + testBrokenChain(t, true, rawdb.PathScheme) +} + +func testBrokenChain(t *testing.T, full bool, scheme string) { + // Make chain starting from genesis + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 10, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -582,6 +757,7 @@ func testBrokenChain(t *testing.T, full, pipeline bool) { // Tests that reorganising a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. +<<<<<<< HEAD func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false, false) } func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true, false) @@ -590,10 +766,24 @@ func TestReorgLongBlocks(t *testing.T) { func testReorgLong(t *testing.T, full, pipeline bool) { testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280+params.GenesisDifficulty.Int64(), full, pipeline) +======= +func TestReorgLongHeaders(t *testing.T) { + testReorgLong(t, false, rawdb.HashScheme) + testReorgLong(t, false, rawdb.PathScheme) +} +func TestReorgLongBlocks(t *testing.T) { + testReorgLong(t, true, rawdb.HashScheme) + testReorgLong(t, true, rawdb.PathScheme) +} + +func testReorgLong(t *testing.T, full bool, scheme string) { + testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280+params.GenesisDifficulty.Int64(), full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that reorganising a short difficult chain after a long easy one // overwrites the canonical numbers and links in the database. +<<<<<<< HEAD func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false, false) } func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true, false) @@ -601,6 +791,18 @@ func TestReorgShortBlocks(t *testing.T) { } func testReorgShort(t *testing.T, full, pipeline bool) { +======= +func TestReorgShortHeaders(t *testing.T) { + testReorgShort(t, false, rawdb.HashScheme) + testReorgShort(t, false, rawdb.PathScheme) +} +func TestReorgShortBlocks(t *testing.T) { + testReorgShort(t, true, rawdb.HashScheme) + testReorgShort(t, true, rawdb.PathScheme) +} + +func testReorgShort(t *testing.T, full bool, scheme string) { +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment // we need a fairly long chain of blocks with different difficulties for a short // one to become heavier than a long one. The 96 is an empirical value. @@ -612,12 +814,21 @@ func testReorgShort(t *testing.T, full, pipeline bool) { for i := 0; i < len(diff); i++ { diff[i] = -9 } +<<<<<<< HEAD testReorg(t, easy, diff, 12615120+params.GenesisDifficulty.Int64(), full, pipeline) } func testReorg(t *testing.T, first, second []int64, td int64, full, pipeline bool) { // Create a pristine chain and database genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) +======= + testReorg(t, easy, diff, 12615120+params.GenesisDifficulty.Int64(), full, scheme) +} + +func testReorg(t *testing.T, first, second []int64, td int64, full bool, scheme string) { + // Create a pristine chain and database + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -685,6 +896,7 @@ func testReorg(t *testing.T, first, second []int64, td int64, full, pipeline boo } // Tests that the insertion functions detect banned hashes. +<<<<<<< HEAD func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false, false) } func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true, true) @@ -694,6 +906,20 @@ func TestBadBlockHashes(t *testing.T) { func testBadHashes(t *testing.T, full, pipeline bool) { // Create a pristine chain and database genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) +======= +func TestBadHeaderHashes(t *testing.T) { + testBadHashes(t, false, rawdb.HashScheme) + testBadHashes(t, false, rawdb.PathScheme) +} +func TestBadBlockHashes(t *testing.T) { + testBadHashes(t, true, rawdb.HashScheme) + testBadHashes(t, true, rawdb.PathScheme) +} + +func testBadHashes(t *testing.T, full bool, scheme string) { + // Create a pristine chain and database + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -722,6 +948,7 @@ func testBadHashes(t *testing.T, full, pipeline bool) { // Tests that bad hashes are detected on boot, and the chain rolled back to a // good state prior to the bad hash. +<<<<<<< HEAD func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false, false) } func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true, false) @@ -731,6 +958,20 @@ func TestReorgBadBlockHashes(t *testing.T) { func testReorgBadHashes(t *testing.T, full, pipeline bool) { // Create a pristine chain and database genDb, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) +======= +func TestReorgBadHeaderHashes(t *testing.T) { + testReorgBadHashes(t, false, rawdb.HashScheme) + testReorgBadHashes(t, false, rawdb.PathScheme) +} +func TestReorgBadBlockHashes(t *testing.T) { + testReorgBadHashes(t, true, rawdb.HashScheme) + testReorgBadHashes(t, true, rawdb.PathScheme) +} + +func testReorgBadHashes(t *testing.T, full bool, scheme string) { + // Create a pristine chain and database + genDb, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -760,7 +1001,7 @@ func testReorgBadHashes(t *testing.T, full, pipeline bool) { blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(blockchain.db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ncm, err := NewBlockChain(blockchain.db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -780,6 +1021,7 @@ func testReorgBadHashes(t *testing.T, full, pipeline bool) { } // Tests chain insertions in the face of one entity containing an invalid nonce. +<<<<<<< HEAD func TestHeadersInsertNonceError(t *testing.T) { testInsertNonceError(t, false, false) } func TestBlocksInsertNonceError(t *testing.T) { testInsertNonceError(t, true, false) @@ -790,6 +1032,21 @@ func testInsertNonceError(t *testing.T, full bool, pipeline bool) { doTest := func(i int) { // Create a pristine chain and database genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) +======= +func TestHeadersInsertNonceError(t *testing.T) { + testInsertNonceError(t, false, rawdb.HashScheme) + testInsertNonceError(t, false, rawdb.PathScheme) +} +func TestBlocksInsertNonceError(t *testing.T) { + testInsertNonceError(t, true, rawdb.HashScheme) + testInsertNonceError(t, true, rawdb.PathScheme) +} + +func testInsertNonceError(t *testing.T, full bool, scheme string) { + doTest := func(i int) { + // Create a pristine chain and database + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -844,6 +1101,11 @@ func testInsertNonceError(t *testing.T, full bool, pipeline bool) { // Tests that fast importing a block chain produces the same chain data as the // classical full block processing. func TestFastVsFullChains(t *testing.T) { + testFastVsFullChains(t, rawdb.HashScheme) + testFastVsFullChains(t, rawdb.PathScheme) +} + +func testFastVsFullChains(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -876,7 +1138,7 @@ func TestFastVsFullChains(t *testing.T) { }) // Import the chain as an archive node for the comparison baseline archiveDb := rawdb.NewMemoryDatabase() - archive, _ := NewBlockChain(archiveDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + archive, _ := NewBlockChain(archiveDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer archive.Stop() if n, err := archive.InsertChain(blocks); err != nil { @@ -884,7 +1146,7 @@ func TestFastVsFullChains(t *testing.T) { } // Fast import the chain as a non-archive node to test fastDb := rawdb.NewMemoryDatabase() - fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -903,7 +1165,8 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + + ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers); err != nil { @@ -971,6 +1234,11 @@ func TestFastVsFullChains(t *testing.T) { // Tests that various import methods move the chain head pointers to the correct // positions. func TestLightVsFastVsFullChainHeads(t *testing.T) { + testLightVsFastVsFullChainHeads(t, rawdb.HashScheme) + testLightVsFastVsFullChainHeads(t, rawdb.PathScheme) +} + +func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1016,6 +1284,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { archiveCaching := *defaultCacheConfig archiveCaching.TrieDirtyDisabled = true + archiveCaching.StateScheme = scheme archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := archive.InsertChain(blocks); err != nil { @@ -1030,7 +1299,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb := makeDb() defer fastDb.Close() - fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -1050,7 +1319,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a ancient-first node and ensure all pointers are updated ancientDb := makeDb() defer ancientDb.Close() - ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers); err != nil { @@ -1069,7 +1338,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb := makeDb() defer lightDb.Close() - light, _ := NewBlockChain(lightDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + light, _ := NewBlockChain(lightDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := light.InsertHeaderChain(headers); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) } @@ -1082,6 +1351,11 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Tests that chain reorganisations handle transaction removals and reinsertions. func TestChainTxReorgs(t *testing.T) { + testChainTxReorgs(t, rawdb.HashScheme) + testChainTxReorgs(t, rawdb.PathScheme) +} + +func testChainTxReorgs(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") @@ -1137,7 +1411,7 @@ func TestChainTxReorgs(t *testing.T) { }) // Import the chain. This runs all block validation rules. db := rawdb.NewMemoryDatabase() - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -1196,6 +1470,11 @@ func TestChainTxReorgs(t *testing.T) { } func TestLogReorgs(t *testing.T) { + testLogReorgs(t, rawdb.HashScheme) + testLogReorgs(t, rawdb.PathScheme) +} + +func testLogReorgs(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -1206,7 +1485,7 @@ func TestLogReorgs(t *testing.T) { signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() rmLogsCh := make(chan RemovedLogsEvent) @@ -1251,13 +1530,18 @@ var logCode = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd // This test checks that log events and RemovedLogsEvent are sent // when the chain reorganizes. func TestLogRebirth(t *testing.T) { + testLogRebirth(t, rawdb.HashScheme) + testLogRebirth(t, rawdb.PathScheme) +} + +func testLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() - blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1328,12 +1612,17 @@ func TestLogRebirth(t *testing.T) { // This test is a variation of TestLogRebirth. It verifies that log events are emitted // when a side chain containing log events overtakes the canonical chain. func TestSideLogRebirth(t *testing.T) { + testSideLogRebirth(t, rawdb.HashScheme) + testSideLogRebirth(t, rawdb.PathScheme) +} + +func testSideLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) - blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1418,6 +1707,11 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan Re } func TestReorgSideEvent(t *testing.T) { + testReorgSideEvent(t, rawdb.HashScheme) + testReorgSideEvent(t, rawdb.PathScheme) +} + +func testReorgSideEvent(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -1427,7 +1721,7 @@ func TestReorgSideEvent(t *testing.T) { } signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, gen *BlockGen) {}) @@ -1498,7 +1792,16 @@ done: // Tests if the canonical block can be fetched from the database during chain insertion. func TestCanonicalBlockRetrieval(t *testing.T) { +<<<<<<< HEAD _, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, false) +======= + testCanonicalBlockRetrieval(t, rawdb.HashScheme) + testCanonicalBlockRetrieval(t, rawdb.PathScheme) +} + +func testCanonicalBlockRetrieval(t *testing.T, scheme string) { + _, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, scheme) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -1542,8 +1845,12 @@ func TestCanonicalBlockRetrieval(t *testing.T) { } pend.Wait() } - func TestEIP155Transition(t *testing.T) { + testEIP155Transition(t, rawdb.HashScheme) + testEIP155Transition(t, rawdb.PathScheme) +} + +func testEIP155Transition(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1602,7 +1909,7 @@ func TestEIP155Transition(t *testing.T) { } }) - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if _, err := blockchain.InsertChain(blocks); err != nil { @@ -1652,8 +1959,12 @@ func TestEIP155Transition(t *testing.T) { t.Errorf("have %v, want %v", have, want) } } - func TestEIP161AccountRemoval(t *testing.T) { + testEIP161AccountRemoval(t, rawdb.HashScheme) + testEIP161AccountRemoval(t, rawdb.PathScheme) +} + +func testEIP161AccountRemoval(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1691,7 +2002,7 @@ func TestEIP161AccountRemoval(t *testing.T) { block.AddTx(tx) }) // account must exist pre eip 161 - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if _, err := blockchain.InsertChain(types.Blocks{blocks[0]}); err != nil { @@ -1724,6 +2035,11 @@ func TestEIP161AccountRemoval(t *testing.T) { // // https://github.com/ethereum/go-ethereum/pull/15941 func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { + testBlockchainHeaderchainReorgConsistency(t, rawdb.HashScheme) + testBlockchainHeaderchainReorgConsistency(t, rawdb.PathScheme) +} + +func testBlockchainHeaderchainReorgConsistency(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -1744,7 +2060,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { } // Import the canonical and fork chain side by side, verifying the current block // and current header consistency - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1803,9 +2119,15 @@ func TestTrieForkGC(t *testing.T) { } } // Dereference all the recent tries and ensure no past trie is left in +<<<<<<< HEAD for i := 0; i < TestTriesInMemory; i++ { chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) +======= + for i := 0; i < TriesInMemory; i++ { + chain.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) + chain.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } if nodes, _ := chain.TrieDB().Size(); nodes > 0 { t.Fatalf("stale tries still alive after garbase collection") @@ -1815,6 +2137,11 @@ func TestTrieForkGC(t *testing.T) { // Tests that doing large reorgs works even if the state associated with the // forking point is not available any more. func TestLargeReorgTrieGC(t *testing.T) { + testLargeReorgTrieGC(t, rawdb.HashScheme) + testLargeReorgTrieGC(t, rawdb.PathScheme) +} + +func testLargeReorgTrieGC(t *testing.T, scheme string) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() genesis := &Genesis{ @@ -1826,7 +2153,10 @@ func TestLargeReorgTrieGC(t *testing.T) { competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + defer db.Close() + + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1839,7 +2169,7 @@ func TestLargeReorgTrieGC(t *testing.T) { t.Fatalf("failed to insert original chain: %v", err) } // Ensure that the state associated with the forking point is pruned away - if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil { + if chain.HasState(shared[len(shared)-1].Root()) { t.Fatalf("common-but-old ancestor still cache") } // Import the competitor chain without exceeding the canonical's TD and ensure @@ -1848,7 +2178,7 @@ func TestLargeReorgTrieGC(t *testing.T) { t.Fatalf("failed to insert competitor chain: %v", err) } for i, block := range competitor[:len(competitor)-2] { - if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { + if chain.HasState(block.Root()) { t.Fatalf("competitor %d: low TD chain became processed", i) } } @@ -1857,14 +2187,36 @@ func TestLargeReorgTrieGC(t *testing.T) { if _, err := chain.InsertChain(competitor[len(competitor)-2:]); err != nil { t.Fatalf("failed to finalize competitor chain: %v", err) } +<<<<<<< HEAD for i, block := range competitor[:len(competitor)-TestTriesInMemory] { if node, err := chain.stateCache.TrieDB().Node(block.Root()); node != nil { t.Fatalf("competitor %d: competing chain state missing, err: %v", i, err) +======= + // In path-based trie database implementation, it will keep 128 diff + 1 disk + // layers, totally 129 latest states available. In hash-based it's 128. + states := TriesInMemory + if scheme == rawdb.PathScheme { + states = states + 1 + } + for i, block := range competitor[:len(competitor)-states] { + if chain.HasState(block.Root()) { + t.Fatalf("competitor %d: unexpected competing chain state", i) + } + } + for i, block := range competitor[len(competitor)-states:] { + if !chain.HasState(block.Root()) { + t.Fatalf("competitor %d: competing chain state missing", i) +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } } } func TestBlockchainRecovery(t *testing.T) { + testBlockchainRecovery(t, rawdb.HashScheme) + testBlockchainRecovery(t, rawdb.PathScheme) +} + +func testBlockchainRecovery(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1881,7 +2233,7 @@ func TestBlockchainRecovery(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -1901,7 +2253,7 @@ func TestBlockchainRecovery(t *testing.T) { rawdb.WriteHeadFastBlockHash(ancientDb, midBlock.Hash()) // Reopen broken blockchain again - ancient, _ = NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ = NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if num := ancient.CurrentBlock().Number.Uint64(); num != 0 { t.Errorf("head block mismatch: have #%v, want #%v", num, 0) @@ -1916,8 +2268,13 @@ func TestBlockchainRecovery(t *testing.T) { // This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain. func TestInsertReceiptChainRollback(t *testing.T) { + testInsertReceiptChainRollback(t, rawdb.HashScheme) + testInsertReceiptChainRollback(t, rawdb.PathScheme) +} + +func testInsertReceiptChainRollback(t *testing.T, scheme string) { // Generate forked chain. The returned BlockChain object is used to process the side chain blocks. - tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains() + tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains(scheme) if err != nil { t.Fatal(err) } @@ -1948,7 +2305,7 @@ func TestInsertReceiptChainRollback(t *testing.T) { } defer ancientDb.Close() - ancientChain, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancientChain, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancientChain.Stop() // Import the canonical header chain. @@ -1993,6 +2350,11 @@ func TestInsertReceiptChainRollback(t *testing.T) { // - https://github.com/ethereum/go-ethereum/issues/18977 // - https://github.com/ethereum/go-ethereum/pull/18988 func TestLowDiffLongChain(t *testing.T) { + testLowDiffLongChain(t, rawdb.HashScheme) + testLowDiffLongChain(t, rawdb.PathScheme) +} + +func testLowDiffLongChain(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -2007,11 +2369,14 @@ func TestLowDiffLongChain(t *testing.T) { }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) + diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + defer diskdb.Close() + + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } - defer chain.stopWithoutSaving() + defer chain.Stop() if n, err := chain.InsertChain(blocks); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", n, err) @@ -2191,11 +2556,20 @@ func TestPrunedImportSideWithMerging(t *testing.T) { testSideImport(t, 1, -10, 1) } -func TestInsertKnownHeaders(t *testing.T) { testInsertKnownChainData(t, "headers") } -func TestInsertKnownReceiptChain(t *testing.T) { testInsertKnownChainData(t, "receipts") } -func TestInsertKnownBlocks(t *testing.T) { testInsertKnownChainData(t, "blocks") } +func TestInsertKnownHeaders(t *testing.T) { + testInsertKnownChainData(t, "headers", rawdb.HashScheme) + testInsertKnownChainData(t, "headers", rawdb.PathScheme) +} +func TestInsertKnownReceiptChain(t *testing.T) { + testInsertKnownChainData(t, "receipts", rawdb.HashScheme) + testInsertKnownChainData(t, "receipts", rawdb.PathScheme) +} +func TestInsertKnownBlocks(t *testing.T) { + testInsertKnownChainData(t, "blocks", rawdb.HashScheme) + testInsertKnownChainData(t, "blocks", rawdb.PathScheme) +} -func testInsertKnownChainData(t *testing.T, typ string) { +func testInsertKnownChainData(t *testing.T, typ string, scheme string) { engine := ethash.NewFaker() genesis := &Genesis{ Config: params.TestChainConfig, @@ -2218,7 +2592,7 @@ func testInsertKnownChainData(t *testing.T, typ string) { } defer chaindb.Close() - chain, err := NewBlockChain(chaindb, nil, genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(chaindb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2491,7 +2865,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i } // getLongAndShortChains returns two chains: A is longer, B is heavier. -func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, *Genesis, error) { +func getLongAndShortChains(scheme string) (*BlockChain, []*types.Block, []*types.Block, *Genesis, error) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -2503,7 +2877,7 @@ func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, *Gene genDb, longChain, _ := GenerateChainWithGenesis(genesis, engine, 80, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { return nil, nil, nil, nil, fmt.Errorf("failed to create tester chain: %v", err) } @@ -2549,7 +2923,12 @@ func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, *Gene // 3. Then there should be no canon mapping for the block at height X // 4. The forked block should still be retrievable by hash func TestReorgToShorterRemovesCanonMapping(t *testing.T) { - chain, canonblocks, sideblocks, _, err := getLongAndShortChains() + testReorgToShorterRemovesCanonMapping(t, rawdb.HashScheme) + testReorgToShorterRemovesCanonMapping(t, rawdb.PathScheme) +} + +func testReorgToShorterRemovesCanonMapping(t *testing.T, scheme string) { + chain, canonblocks, sideblocks, _, err := getLongAndShortChains(scheme) if err != nil { t.Fatal(err) } @@ -2587,7 +2966,12 @@ func TestReorgToShorterRemovesCanonMapping(t *testing.T) { // as TestReorgToShorterRemovesCanonMapping, but applied on headerchain // imports -- that is, for fast sync func TestReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T) { - chain, canonblocks, sideblocks, _, err := getLongAndShortChains() + testReorgToShorterRemovesCanonMappingHeaderChain(t, rawdb.HashScheme) + testReorgToShorterRemovesCanonMappingHeaderChain(t, rawdb.PathScheme) +} + +func testReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T, scheme string) { + chain, canonblocks, sideblocks, _, err := getLongAndShortChains(scheme) if err != nil { t.Fatal(err) } @@ -2736,6 +3120,11 @@ func TestTransactionIndices(t *testing.T) { } func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { + testSkipStaleTxIndicesInSnapSync(t, rawdb.HashScheme) + testSkipStaleTxIndicesInSnapSync(t, rawdb.PathScheme) +} + +func testSkipStaleTxIndicesInSnapSync(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -2794,7 +3183,7 @@ func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { // Import all blocks into ancient db, only HEAD-32 indices are kept. l := uint64(32) - chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2928,6 +3317,11 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { // 2. Downloader starts to sync again // 3. The blocks fetched are all known and canonical blocks func TestSideImportPrunedBlocks(t *testing.T) { + testSideImportPrunedBlocks(t, rawdb.HashScheme) + testSideImportPrunedBlocks(t, rawdb.PathScheme) +} + +func testSideImportPrunedBlocks(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -2937,7 +3331,7 @@ func TestSideImportPrunedBlocks(t *testing.T) { // Generate and import the canonical chain _, blocks, _ := GenerateChainWithGenesis(genesis, engine, 2*TriesInMemory, nil) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2946,15 +3340,30 @@ func TestSideImportPrunedBlocks(t *testing.T) { if n, err := chain.InsertChain(blocks); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } +<<<<<<< HEAD lastPrunedIndex := len(blocks) - TestTriesInMemory - 1 lastPrunedBlock := blocks[lastPrunedIndex-1] +======= + // In path-based trie database implementation, it will keep 128 diff + 1 disk + // layers, totally 129 latest states available. In hash-based it's 128. + states := TriesInMemory + if scheme == rawdb.PathScheme { + states = TriesInMemory + 1 + } + lastPrunedIndex := len(blocks) - states - 1 + lastPrunedBlock := blocks[lastPrunedIndex] +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Verify pruning of lastPrunedBlock if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64()) } +<<<<<<< HEAD firstNonPrunedBlock := blocks[len(blocks)-TestTriesInMemory] +======= + firstNonPrunedBlock := blocks[len(blocks)-states] +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Verify firstNonPrunedBlock is not pruned if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) { t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64()) @@ -2976,6 +3385,11 @@ func TestSideImportPrunedBlocks(t *testing.T) { // each transaction, so this works ok. The rework accumulated writes in memory // first, but the journal wiped the entire state object on create-revert. func TestDeleteCreateRevert(t *testing.T) { + testDeleteCreateRevert(t, rawdb.HashScheme) + testDeleteCreateRevert(t, rawdb.PathScheme) +} + +func testDeleteCreateRevert(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb") @@ -3027,7 +3441,7 @@ func TestDeleteCreateRevert(t *testing.T) { b.AddTx(tx) }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3046,6 +3460,11 @@ func TestDeleteCreateRevert(t *testing.T) { // Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct, // and then the new slots exist func TestDeleteRecreateSlots(t *testing.T) { + testDeleteRecreateSlots(t, rawdb.HashScheme) + testDeleteRecreateSlots(t, rawdb.PathScheme) +} + +func testDeleteRecreateSlots(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3135,7 +3554,7 @@ func TestDeleteRecreateSlots(t *testing.T) { b.AddTx(tx) }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) if err != nil { @@ -3169,6 +3588,11 @@ func TestDeleteRecreateSlots(t *testing.T) { // regular value-transfer // Expected outcome is that _all_ slots are cleared from A func TestDeleteRecreateAccount(t *testing.T) { + testDeleteRecreateAccount(t, rawdb.HashScheme) + testDeleteRecreateAccount(t, rawdb.PathScheme) +} + +func testDeleteRecreateAccount(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3212,7 +3636,7 @@ func TestDeleteRecreateAccount(t *testing.T) { b.AddTx(tx) }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) if err != nil { @@ -3242,6 +3666,11 @@ func TestDeleteRecreateAccount(t *testing.T) { // Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct, // and then the new slots exist func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { + testDeleteRecreateSlotsAcrossManyBlocks(t, rawdb.HashScheme) + testDeleteRecreateSlotsAcrossManyBlocks(t, rawdb.PathScheme) +} + +func testDeleteRecreateSlotsAcrossManyBlocks(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3382,7 +3811,7 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { current = exp }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3440,7 +3869,14 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { // to the destructset in case something is created "onto" an existing item. // We need to either roll back the snapDestructs, or not place it into snapDestructs // in the first place. +// + func TestInitThenFailCreateContract(t *testing.T) { + testInitThenFailCreateContract(t, rawdb.HashScheme) + testInitThenFailCreateContract(t, rawdb.PathScheme) +} + +func testInitThenFailCreateContract(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3513,7 +3949,7 @@ func TestInitThenFailCreateContract(t *testing.T) { }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3550,6 +3986,7 @@ func TestInitThenFailCreateContract(t *testing.T) { // access list transaction, which specifies a single slot access, and then // checking that the gas usage of a hot SLOAD and a cold SLOAD are calculated // correctly. +<<<<<<< HEAD // TestEIP2718TransitionWithTestChainConfig tests EIP-2718 with TestChainConfig. func TestEIP2718TransitionWithTestChainConfig(t *testing.T) { @@ -3563,6 +4000,14 @@ func TestEIP2718TransitionWithParliaConfig(t *testing.T) { // testEIP2718TransitionWithConfig tests EIP02718 with given ChainConfig. func testEIP2718TransitionWithConfig(t *testing.T, config *params.ChainConfig) { +======= +func TestEIP2718Transition(t *testing.T) { + testEIP2718Transition(t, rawdb.HashScheme) + testEIP2718Transition(t, rawdb.PathScheme) +} + +func testEIP2718Transition(t *testing.T, scheme string) { +>>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") engine = ethash.NewFaker() @@ -3610,7 +4055,7 @@ func testEIP2718TransitionWithConfig(t *testing.T, config *params.ChainConfig) { }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3640,6 +4085,11 @@ func testEIP2718TransitionWithConfig(t *testing.T, config *params.ChainConfig) { // gasFeeCap - gasTipCap < baseFee. // 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). func TestEIP1559Transition(t *testing.T) { + testEIP1559Transition(t, rawdb.HashScheme) + testEIP1559Transition(t, rawdb.PathScheme) +} + +func testEIP1559Transition(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") engine = ethash.NewFaker() @@ -3650,8 +4100,9 @@ func TestEIP1559Transition(t *testing.T) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) funds = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) + config = *params.AllEthashProtocolChanges gspec = &Genesis{ - Config: params.AllEthashProtocolChanges, + Config: &config, Alloc: GenesisAlloc{ addr1: {Balance: funds}, addr2: {Balance: funds}, @@ -3698,7 +4149,7 @@ func TestEIP1559Transition(t *testing.T) { b.AddTx(tx) }) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3780,6 +4231,11 @@ func TestEIP1559Transition(t *testing.T) { // Tests the scenario the chain is requested to another point with the missing state. // It expects the state is recovered and all relevant chain markers are set correctly. func TestSetCanonical(t *testing.T) { + testSetCanonical(t, rawdb.HashScheme) + testSetCanonical(t, rawdb.PathScheme) +} + +func testSetCanonical(t *testing.T, scheme string) { //log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) var ( @@ -3802,7 +4258,10 @@ func TestSetCanonical(t *testing.T) { } gen.AddTx(tx) }) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) + defer diskdb.Close() + + chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3859,6 +4318,11 @@ func TestSetCanonical(t *testing.T) { // TestCanonicalHashMarker tests all the canonical hash markers are updated/deleted // correctly in case reorg is called. func TestCanonicalHashMarker(t *testing.T) { + testCanonicalHashMarker(t, rawdb.HashScheme) + testCanonicalHashMarker(t, rawdb.PathScheme) +} + +func testCanonicalHashMarker(t *testing.T, scheme string) { var cases = []struct { forkA int forkB int @@ -3906,7 +4370,7 @@ func TestCanonicalHashMarker(t *testing.T) { _, forkB, _ := GenerateChainWithGenesis(gspec, engine, c.forkB, func(i int, gen *BlockGen) {}) // Initialize test chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -4262,6 +4726,116 @@ func testCreateThenDelete(t *testing.T, config *params.ChainConfig) { } } +func TestDeleteThenCreate(t *testing.T) { + var ( + engine = ethash.NewFaker() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + factoryAddr = crypto.CreateAddress(address, 0) + funds = big.NewInt(1000000000000000) + ) + /* + contract Factory { + function deploy(bytes memory code) public { + address addr; + assembly { + addr := create2(0, add(code, 0x20), mload(code), 0) + if iszero(extcodesize(addr)) { + revert(0, 0) + } + } + } + } + */ + factoryBIN := common.Hex2Bytes("608060405234801561001057600080fd5b50610241806100206000396000f3fe608060405234801561001057600080fd5b506004361061002a5760003560e01c80627743601461002f575b600080fd5b610049600480360381019061004491906100d8565b61004b565b005b6000808251602084016000f59050803b61006457600080fd5b5050565b600061007b61007684610146565b610121565b905082815260208101848484011115610097576100966101eb565b5b6100a2848285610177565b509392505050565b600082601f8301126100bf576100be6101e6565b5b81356100cf848260208601610068565b91505092915050565b6000602082840312156100ee576100ed6101f5565b5b600082013567ffffffffffffffff81111561010c5761010b6101f0565b5b610118848285016100aa565b91505092915050565b600061012b61013c565b90506101378282610186565b919050565b6000604051905090565b600067ffffffffffffffff821115610161576101606101b7565b5b61016a826101fa565b9050602081019050919050565b82818337600083830152505050565b61018f826101fa565b810181811067ffffffffffffffff821117156101ae576101ad6101b7565b5b80604052505050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600080fd5b600080fd5b600080fd5b600080fd5b6000601f19601f830116905091905056fea2646970667358221220ea8b35ed310d03b6b3deef166941140b4d9e90ea2c92f6b41eb441daf49a59c364736f6c63430008070033") + + /* + contract C { + uint256 value; + constructor() { + value = 100; + } + function destruct() public payable { + selfdestruct(payable(msg.sender)); + } + receive() payable external {} + } + */ + contractABI := common.Hex2Bytes("6080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c63430008070033") + contractAddr := crypto.CreateAddress2(factoryAddr, [32]byte{}, crypto.Keccak256(contractABI)) + + gspec := &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{ + address: {Balance: funds}, + }, + } + nonce := uint64(0) + signer := types.HomesteadSigner{} + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 2, func(i int, b *BlockGen) { + fee := big.NewInt(1) + if b.header.BaseFee != nil { + fee = b.header.BaseFee + } + b.SetCoinbase(common.Address{1}) + + // Block 1 + if i == 0 { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + Data: factoryBIN, + }) + nonce++ + b.AddTx(tx) + + data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &factoryAddr, + Data: data, + }) + b.AddTx(tx) + nonce++ + } else { + // Block 2 + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &contractAddr, + Data: common.Hex2Bytes("2b68b9c6"), // destruct + }) + nonce++ + b.AddTx(tx) + + data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &factoryAddr, // re-creation + Data: data, + }) + b.AddTx(tx) + nonce++ + } + }) + // Import the canonical chain + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + for _, block := range blocks { + if _, err := chain.InsertChain([]*types.Block{block}); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) + } + } +} + // TestTransientStorageReset ensures the transient storage is wiped correctly // between transactions. func TestTransientStorageReset(t *testing.T) { diff --git a/core/chain_makers.go b/core/chain_makers.go index 4686aed47c..f0026089ac 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -280,7 +280,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) chainreader := &fakeChainReader{config: config} - genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { + genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} b.header = makeHeader(chainreader, parent, statedb, b.engine) @@ -324,19 +324,23 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } - if err := statedb.Database().TrieDB().Commit(root, false); err != nil { + if err = triedb.Commit(root, false); err != nil { panic(fmt.Sprintf("trie write error: %v", err)) } return block, b.receipts } return nil, nil } + // Forcibly use hash-based state scheme for retaining all nodes in disk. + triedb := trie.NewDatabase(db, trie.HashDefaults) + defer triedb.Close() + for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil) + statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { panic(err) } - block, receipt := genblock(i, parent, statedb) + block, receipt := genblock(i, parent, triedb, statedb) blocks[i] = block receipts[i] = receipt parent = block @@ -349,7 +353,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { db := rawdb.NewMemoryDatabase() - _, err := genesis.Commit(db, trie.NewDatabase(db)) + triedb := trie.NewDatabase(db, trie.HashDefaults) + defer triedb.Close() + _, err := genesis.Commit(db, triedb) if err != nil { panic(err) } diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 4f73555275..db220cf24a 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) func TestGenerateWithdrawalChain(t *testing.T) { @@ -74,8 +75,7 @@ func TestGenerateWithdrawalChain(t *testing.T) { Storage: storage, Code: common.Hex2Bytes("600154600354"), } - - genesis := gspec.MustCommit(gendb) + genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults)) chain, _ := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) { tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(address), address, big.NewInt(1000), params.TxGas, new(big.Int).Add(gen.BaseFee(), common.Big1), nil), signer, key) @@ -146,6 +146,7 @@ func ExampleGenerateChain() { addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) db = rawdb.NewMemoryDatabase() + genDb = rawdb.NewMemoryDatabase() ) // Ensure that key1 has some funds in the genesis block. @@ -153,13 +154,13 @@ func ExampleGenerateChain() { Config: ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, } - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults)) // This call generates a chain of 5 blocks. The function runs for // each block and adds different features to gen based on the // block index. signer := types.HomesteadSigner{} - chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 5, func(i int, gen *BlockGen) { + chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), genDb, 5, func(i int, gen *BlockGen) { switch i { case 0: // In block 1, addr1 sends addr2 some ether. @@ -188,7 +189,7 @@ func ExampleGenerateChain() { }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.HashScheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if i, err := blockchain.InsertChain(chain); err != nil { diff --git a/core/dao_test.go b/core/dao_test.go index f2e8dfe8f9..b9a899ef2f 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -83,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } bc.Stop() @@ -106,7 +106,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } bc.Stop() @@ -131,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&proConf, conBc.GetBlockByHash(conBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) @@ -149,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { + if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&conConf, proBc.GetBlockByHash(proBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) diff --git a/core/genesis.go b/core/genesis.go index 9454b97620..8fa107ac66 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -330,10 +330,12 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } - // We have the genesis block in database(perhaps in ancient database) - // but the corresponding state is missing. + // The genesis block is present(perhaps in ancient database) while the + // state database is not initialized yet. It can happen that the node + // is initialized with an external ancient store. Commit genesis state + // in this case. header := rawdb.ReadHeader(db, stored, 0) - if header.Root != types.EmptyRootHash && !rawdb.HasLegacyTrieNode(db, header.Root) { + if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) { if genesis == nil { genesis = DefaultGenesisBlock() } @@ -586,10 +588,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block // MustCommit writes the genesis block and state to db, panicking on error. // The block is committed as the canonical head block. -// Note the state changes will be committed in hash-based scheme, use Commit -// if path-scheme is preferred. -func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { - block, err := g.Commit(db, trie.NewDatabase(db)) +func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block { + block, err := g.Commit(db, triedb) if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index d07ccaf7dd..deba7e265f 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -30,9 +30,24 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) +func TestInvalidCliqueConfig(t *testing.T) { + block := DefaultGoerliGenesisBlock() + block.ExtraData = []byte{} + db := rawdb.NewMemoryDatabase() + if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil { + t.Fatal("Expected error on invalid clique config") + } +} + func TestSetupGenesis(t *testing.T) { + testSetupGenesis(t, rawdb.HashScheme) + testSetupGenesis(t, rawdb.PathScheme) +} + +func testSetupGenesis(t *testing.T, scheme string) { var ( customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50") customg = Genesis{ @@ -44,6 +59,7 @@ func TestSetupGenesis(t *testing.T) { oldcustomg = customg ) oldcustomg.Config = ¶ms.ChainConfig{HomesteadBlock: big.NewInt(2)} + tests := []struct { name string fn func(ethdb.Database) (*params.ChainConfig, common.Hash, error) @@ -54,7 +70,7 @@ func TestSetupGenesis(t *testing.T) { { name: "genesis without ChainConfig", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db), new(Genesis)) + return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis)) }, wantErr: errGenesisNoConfig, wantConfig: params.AllEthashProtocolChanges, @@ -62,7 +78,7 @@ func TestSetupGenesis(t *testing.T) { { name: "no block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlock(db, trie.NewDatabase(db), nil) + return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -70,8 +86,8 @@ func TestSetupGenesis(t *testing.T) { { name: "mainnet block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - DefaultGenesisBlock().MustCommit(db) - return SetupGenesisBlock(db, trie.NewDatabase(db), nil) + DefaultGenesisBlock().MustCommit(db, trie.NewDatabase(db, newDbConfig(scheme))) + return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil) }, wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, @@ -79,17 +95,30 @@ func TestSetupGenesis(t *testing.T) { { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - customg.MustCommit(db) - return SetupGenesisBlock(db, trie.NewDatabase(db), nil) + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + customg.Commit(db, tdb) + return SetupGenesisBlock(db, tdb, nil) }, wantHash: customghash, wantConfig: customg.Config, }, + { + name: "custom block in DB, genesis == goerli", + fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + customg.Commit(db, tdb) + return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock()) + }, + wantErr: &GenesisMismatchError{Stored: customghash, New: params.GoerliGenesisHash}, + wantHash: params.GoerliGenesisHash, + wantConfig: params.GoerliChainConfig, + }, { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - oldcustomg.MustCommit(db) - return SetupGenesisBlock(db, trie.NewDatabase(db), &customg) + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + oldcustomg.Commit(db, tdb) + return SetupGenesisBlock(db, tdb, &customg) }, wantHash: customghash, wantConfig: customg.Config, @@ -99,16 +128,17 @@ func TestSetupGenesis(t *testing.T) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. - genesis := oldcustomg.MustCommit(db) + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + oldcustomg.Commit(db, tdb) - bc, _ := NewBlockChain(db, nil, &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) defer bc.Stop() - blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) + _, blocks, _ := GenerateChainWithGenesis(&oldcustomg, ethash.NewFaker(), 4, nil) bc.InsertChain(blocks) // This should return a compatibility error. - return SetupGenesisBlock(db, trie.NewDatabase(db), &customg) + return SetupGenesisBlock(db, tdb, &customg) }, wantHash: customghash, wantConfig: customg.Config, @@ -154,7 +184,8 @@ func TestGenesisHashes(t *testing.T) { {DefaultGenesisBlock(), params.MainnetGenesisHash}, } { // Test via MustCommit - if have := c.genesis.MustCommit(rawdb.NewMemoryDatabase()).Hash(); have != c.want { + db := rawdb.NewMemoryDatabase() + if have := c.genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)).Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock @@ -172,7 +203,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock := genesis.MustCommit(db) + genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") @@ -221,6 +252,7 @@ func TestReadWriteGenesisAlloc(t *testing.T) { } } } + func TestConfigOrDefault(t *testing.T) { defaultGenesis := DefaultGenesisBlock() if defaultGenesis.Config.PlanckBlock != nil { @@ -273,3 +305,10 @@ func TestSetDefaultBlockValues(t *testing.T) { t.Errorf("ChainID should not have been modified. ChainID = %v", genesis.Config.ChainID) } } + +func newDbConfig(scheme string) *trie.Config { + if scheme == rawdb.HashScheme { + return trie.HashDefaults + } + return &trie.Config{PathDB: pathdb.Defaults} +} diff --git a/core/headerchain_test.go b/core/headerchain_test.go index 08d19f6950..2c0323e6f7 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -73,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} ) - gspec.Commit(db, trie.NewDatabase(db)) + gspec.Commit(db, trie.NewDatabase(db, nil)) hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) if err != nil { t.Fatal(err) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index f607dd534e..ae49932fc2 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -435,12 +435,12 @@ func checkReceiptsRLP(have, want types.Receipts) error { func TestAncientStorage(t *testing.T) { // Freezer style fast import the chain. frdir := t.TempDir() - db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false) if err != nil { t.Fatalf("failed to create database with ancient backend") } defer db.Close() + // Create a test block block := types.NewBlockWithHeader(&types.Header{ Number: big.NewInt(0), diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 12f1ecdf83..f5c2f8899a 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -36,7 +36,7 @@ import ( // // Now this scheme is still kept for backward compatibility, and it will be used // for archive node and some other tries(e.g. light trie). -const HashScheme = "hashScheme" +const HashScheme = "hash" // PathScheme is the new path-based state scheme with which trie nodes are stored // in the disk with node path as the database key. This scheme will only store one @@ -44,7 +44,7 @@ const HashScheme = "hashScheme" // is native. At the same time, this scheme will put adjacent trie nodes in the same // area of the disk with good data locality property. But this scheme needs to rely // on extra state diffs to survive deep reorg. -const PathScheme = "pathScheme" +const PathScheme = "path" // hasher is used to compute the sha256 hash of the provided data. type hasher struct{ sha crypto.KeccakState } @@ -263,3 +263,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has panic(fmt.Sprintf("Unknown scheme %v", scheme)) } } + +// ReadStateScheme reads the state scheme of persistent state, or none +// if the state is not present in database. +func ReadStateScheme(db ethdb.Reader) string { + // Check if state in path-based scheme is present + blob, _ := ReadAccountTrieNode(db, nil) + if len(blob) != 0 { + return PathScheme + } + // In a hash-based scheme, the genesis state is consistently stored + // on the disk. To assess the scheme of the persistent state, it + // suffices to inspect the scheme of the genesis state. + header := ReadHeader(db, ReadCanonicalHash(db, 0), 0) + if header == nil { + return "" // empty datadir + } + blob = ReadLegacyTrieNode(db, header.Root) + if len(blob) == 0 { + return "" // no state in disk + } + return HashScheme +} diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go index 6432a3486b..1f22a5c342 100644 --- a/core/rawdb/ancient_scheme.go +++ b/core/rawdb/ancient_scheme.go @@ -58,7 +58,7 @@ const ( stateHistoryStorageData = "storage.data" ) -var stateHistoryFreezerNoSnappy = map[string]bool{ +var stateFreezerNoSnappy = map[string]bool{ stateHistoryMeta: true, stateHistoryAccountIndex: false, stateHistoryStorageIndex: false, @@ -75,7 +75,7 @@ var ( // freezers the collections of all builtin freezers. var freezers = []string{chainFreezerName, stateFreezerName} -// NewStateHistoryFreezer initializes the freezer for state history. -func NewStateHistoryFreezer(ancientDir string, readOnly bool, offset uint64) (*ResettableFreezer, error) { - return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, offset, stateHistoryTableSize, stateHistoryFreezerNoSnappy) +// NewStateFreezer initializes the freezer for state history. +func NewStateFreezer(ancientDir string, readOnly bool, offset uint64) (*ResettableFreezer, error) { + return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, offset, stateHistoryTableSize, stateFreezerNoSnappy) } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 363a911aee..7b4e6e7f12 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -50,36 +50,58 @@ func (info *freezerInfo) size() common.StorageSize { return total } +func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) { + info := freezerInfo{name: name} + for t := range order { + size, err := reader.AncientSize(t) + if err != nil { + return freezerInfo{}, err + } + info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)}) + } + // Retrieve the number of last stored item + ancients, err := reader.Ancients() + if err != nil { + return freezerInfo{}, err + } + info.head = ancients - 1 + + // Retrieve the number of first stored item + tail, err := reader.Tail() + if err != nil { + return freezerInfo{}, err + } + info.tail = tail + return info, nil +} + // inspectFreezers inspects all freezers registered in the system. func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { var infos []freezerInfo for _, freezer := range freezers { switch freezer { case chainFreezerName: - // Chain ancient store is a bit special. It's always opened along - // with the key-value store, inspect the chain store directly. - info := freezerInfo{name: freezer} - // Retrieve storage size of every contained table. - for table := range chainFreezerNoSnappy { - size, err := db.AncientSize(table) - if err != nil { - return nil, err - } - info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)}) + info, err := inspect(chainFreezerName, chainFreezerNoSnappy, db) + if err != nil { + return nil, err + } + infos = append(infos, info) + + case stateFreezerName: + datadir, err := db.AncientDatadir() + if err != nil { + return nil, err } - // Retrieve the number of last stored item - ancients, err := db.Ancients() + f, err := NewStateFreezer(datadir, true, 0) if err != nil { return nil, err } - info.head = ancients - 1 + defer f.Close() - // Retrieve the number of first stored item - tail, err := db.Tail() + info, err := inspect(stateFreezerName, stateFreezerNoSnappy, f) if err != nil { return nil, err } - info.tail = tail infos = append(infos, info) default: diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 3416b79b55..7188efa6f3 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -599,7 +599,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { tds stat numHashPairings stat hashNumPairings stat - tries stat + legacyTries stat + stateLookups stat + accountTries stat + storageTries stat codes stat txLookups stat accountSnaps stat @@ -640,8 +643,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { numHashPairings.Add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.Add(size) - case len(key) == common.HashLength: - tries.Add(size) + case IsLegacyTrieNode(key, it.Value()): + legacyTries.Add(size) + case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: + stateLookups.Add(size) + case IsAccountTrieNode(key): + accountTries.Add(size) + case IsStorageTrieNode(key): + storageTries.Add(size) case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: codes.Add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): @@ -678,7 +687,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, - uncleanShutdownKey, badBlockKey, transitionStatusKey, + uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, + persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, } { if bytes.Equal(key, meta) { metadata.Add(size) @@ -707,7 +717,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, - {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, + {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, + {"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, + {"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()}, + {"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()}, {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 1de8500ace..13a00d795a 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -67,6 +67,9 @@ var ( // snapshotSyncStatusKey tracks the snapshot sync status across restarts. snapshotSyncStatusKey = []byte("SnapshotSyncStatus") + // skeletonSyncStatusKey tracks the skeleton sync status across restarts. + skeletonSyncStatusKey = []byte("SkeletonSyncStatus") + // trieJournalKey tracks the in-memory trie node layers across restarts. trieJournalKey = []byte("TrieJournal") @@ -285,9 +288,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { return bytes.Equal(key, crypto.Keccak256(val)) } -// IsAccountTrieNode reports whether a provided database entry is an account -// trie node in path-based state scheme. -func IsAccountTrieNode(key []byte) (bool, []byte) { +// ResolveAccountTrieNodeKey reports whether a provided database entry is an +// account trie node in path-based state scheme, and returns the resolved +// node path if so. +func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { if !bytes.HasPrefix(key, trieNodeAccountPrefix) { return false, nil } @@ -300,9 +304,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) { return true, key[len(trieNodeAccountPrefix):] } -// IsStorageTrieNode reports whether a provided database entry is a storage +// IsAccountTrieNode reports whether a provided database entry is an account // trie node in path-based state scheme. -func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) { +func IsAccountTrieNode(key []byte) bool { + ok, _ := ResolveAccountTrieNodeKey(key) + return ok +} + +// ResolveStorageTrieNode reports whether a provided database entry is a storage +// trie node in path-based state scheme, and returns the resolved account hash +// and node path if so. +func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { if !bytes.HasPrefix(key, trieNodeStoragePrefix) { return false, common.Hash{}, nil } @@ -318,3 +330,10 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) { accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] } + +// IsStorageTrieNode reports whether a provided database entry is a storage +// trie node in path-based state scheme. +func IsStorageTrieNode(key []byte) bool { + ok, _, _ := ResolveStorageTrieNode(key) + return ok +} diff --git a/core/state/database.go b/core/state/database.go index a7081b717f..51f0940cc5 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -70,7 +70,7 @@ type Database interface { // DiskDB returns the underlying key-value disk database. DiskDB() ethdb.KeyValueStore - // TrieDB retrieves the low level trie database used for data storage. + // TrieDB returns the underlying trie database for managing trie nodes. TrieDB() *trie.Database // Cache the account trie tree @@ -173,7 +173,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { disk: db, codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - triedb: trie.NewDatabaseWithConfig(db, config), + triedb: trie.NewDatabase(db, config), noTries: noTries, } } @@ -200,7 +200,7 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab disk: db, codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - triedb: trie.NewDatabaseWithConfig(db, config), + triedb: trie.NewDatabase(db, config), accountTrieCache: atc, storageTrieCache: stc, noTries: noTries, diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index c8e1a181eb..73cc22490b 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -26,9 +26,14 @@ import ( // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { + testNodeIteratorCoverage(t, rawdb.HashScheme) + testNodeIteratorCoverage(t, rawdb.PathScheme) +} + +func testNodeIteratorCoverage(t *testing.T, scheme string) { // Create some arbitrary test state to iterate - db, sdb, root, _ := makeTestState() - sdb.TrieDB().Commit(root, false) + db, sdb, ndb, root, _ := makeTestState(scheme) + ndb.Commit(root, false) state, err := New(root, sdb, nil) if err != nil { @@ -48,7 +53,7 @@ func TestNodeIteratorCoverage(t *testing.T) { ) it := db.NewIterator(nil, nil) for it.Next() { - ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) + ok, hash := isTrieNode(scheme, it.Key(), it.Value()) if !ok { continue } @@ -90,11 +95,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) { return true, common.BytesToHash(key) } } else { - ok, _ := rawdb.IsAccountTrieNode(key) + ok := rawdb.IsAccountTrieNode(key) if ok { return true, crypto.Keccak256Hash(val) } - ok, _, _ = rawdb.IsStorageTrieNode(key) + ok = rawdb.IsStorageTrieNode(key) if ok { return true, crypto.Keccak256Hash(val) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 09d87945cc..1555852d03 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -100,13 +100,16 @@ func NewPruner(db ethdb.Database, config Config, triesInMemory uint64) (*Pruner, if headBlock == nil { return nil, errors.New("failed to load head block") } + // Offline pruning is only supported in legacy hash based scheme. + triedb := trie.NewDatabase(db, trie.HashDefaults) + snapconfig := snapshot.Config{ CacheSize: 256, Recovery: false, NoBuild: true, AsyncBuild: false, } - snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root(), int(triesInMemory), false) + snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root(), int(triesInMemory), false) if err != nil { return nil, err // The relevant snapshot(s) might not exist } @@ -684,7 +687,9 @@ func RecoverPruning(datadir string, db ethdb.Database, triesInMemory uint64) err NoBuild: true, AsyncBuild: false, } - snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root(), int(triesInMemory), false) + // Offline pruning is only supported in legacy hash based scheme. + triedb := trie.NewDatabase(db, trie.HashDefaults) + snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root(), int(triesInMemory), false) if err != nil { return err // The relevant snapshot(s) might not exist } @@ -726,7 +731,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults)) if err != nil { return err } @@ -750,7 +755,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { } if acc.Root != types.EmptyRootHash { id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root) - storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db)) + storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults)) if err != nil { return err } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index b1ae987581..fbd83e3b52 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -356,7 +356,8 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi var resolver trie.NodeResolver if len(result.keys) > 0 { mdb := rawdb.NewMemoryDatabase() - tdb := trie.NewDatabase(mdb) + tdb := trie.NewDatabase(mdb, trie.HashDefaults) + defer tdb.Close() snapTrie := trie.NewEmpty(tdb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index c503676241..07016b675c 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" ) @@ -45,10 +47,15 @@ func hashData(input []byte) common.Hash { // Tests that snapshot generation from an empty database. func TestGeneration(t *testing.T) { + testGeneration(t, rawdb.HashScheme) + testGeneration(t, rawdb.PathScheme) +} + +func testGeneration(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var helper = newHelper() + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -79,10 +86,15 @@ func TestGeneration(t *testing.T) { // Tests that snapshot generation with existent flat state. func TestGenerateExistentState(t *testing.T) { + testGenerateExistentState(t, rawdb.HashScheme) + testGenerateExistentState(t, rawdb.PathScheme) +} + +func testGenerateExistentState(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var helper = newHelper() + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -148,9 +160,15 @@ type testHelper struct { nodes *trienode.MergedNodeSet } -func newHelper() *testHelper { +func newHelper(scheme string) *testHelper { diskdb := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(diskdb) + config := &trie.Config{} + if scheme == rawdb.PathScheme { + config.PathDB = &pathdb.Config{} // disable caching + } else { + config.HashDB = &hashdb.Config{} // disable caching + } + triedb := trie.NewDatabase(diskdb, config) accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb) return &testHelper{ diskdb: diskdb, @@ -233,7 +251,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { // - extra slots in the middle // - extra slots in the end func TestGenerateExistentStateWithWrongStorage(t *testing.T) { - helper := newHelper() + testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme) + testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme) +} + +func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { + helper := newHelper(scheme) // Account one, empty root but non-empty database helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -325,7 +348,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // - wrong accounts // - extra accounts func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { - helper := newHelper() + testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme) + testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme) +} + +func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) { + helper := newHelper(scheme) helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -380,10 +408,15 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { // Tests that snapshot generation errors out correctly in case of a missing trie // node in the account trie. func TestGenerateCorruptAccountTrie(t *testing.T) { + testGenerateCorruptAccountTrie(t, rawdb.HashScheme) + testGenerateCorruptAccountTrie(t, rawdb.PathScheme) +} + +func testGenerateCorruptAccountTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // without any storage slots to keep the test smaller. - helper := newHelper() + helper := newHelper(scheme) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 @@ -391,9 +424,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 - // Delete an account trie leaf and ensure the generator chokes - helper.triedb.Commit(root, false) - helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) + // Delete an account trie node and ensure the generator chokes + targetPath := []byte{0xc} + targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7") + + rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { @@ -414,11 +449,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // trie node for a storage trie. It's similar to internal corruption but it is // handled differently inside the generator. func TestGenerateMissingStorageTrie(t *testing.T) { + testGenerateMissingStorageTrie(t, rawdb.HashScheme) + testGenerateMissingStorageTrie(t, rawdb.PathScheme) +} + +func testGenerateMissingStorageTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - helper := newHelper() - + var ( + acc1 = hashData([]byte("acc-1")) + acc3 = hashData([]byte("acc-3")) + helper = newHelper(scheme) + ) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 @@ -427,8 +470,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) { root := helper.Commit() - // Delete a storage trie root and ensure the generator chokes - helper.diskdb.Delete(stRoot.Bytes()) + // Delete storage trie root of account one and three. + rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme) + rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { @@ -448,10 +492,15 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // Tests that snapshot generation errors out correctly in case of a missing trie // node in a storage trie. func TestGenerateCorruptStorageTrie(t *testing.T) { + testGenerateCorruptStorageTrie(t, rawdb.HashScheme) + testGenerateCorruptStorageTrie(t, rawdb.PathScheme) +} + +func testGenerateCorruptStorageTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - helper := newHelper() + helper := newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -461,8 +510,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { root := helper.Commit() - // Delete a storage trie leaf and ensure the generator chokes - helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) + // Delete a node in the storage trie. + targetPath := []byte{0x4} + targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371") + rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme) + rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { @@ -481,7 +533,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithExtraAccounts(t *testing.T) { - helper := newHelper() + testGenerateWithExtraAccounts(t, rawdb.HashScheme) + testGenerateWithExtraAccounts(t, rawdb.PathScheme) +} + +func testGenerateWithExtraAccounts(t *testing.T, scheme string) { + helper := newHelper(scheme) { // Account one in the trie stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), @@ -549,10 +606,15 @@ func enableLogging() { // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithManyExtraAccounts(t *testing.T) { + testGenerateWithManyExtraAccounts(t, rawdb.HashScheme) + testGenerateWithManyExtraAccounts(t, rawdb.PathScheme) +} + +func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { // Account one in the trie stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), @@ -605,11 +667,16 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { // So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted. // But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished. func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { + testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme) + testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme) +} + +func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) { accountCheckRange = 3 if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) @@ -642,11 +709,16 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { // TestGenerateWithMalformedSnapdata tests what happes if we have some junk // in the snapshot database, which cannot be parsed back to an account func TestGenerateWithMalformedSnapdata(t *testing.T) { + testGenerateWithMalformedSnapdata(t, rawdb.HashScheme) + testGenerateWithMalformedSnapdata(t, rawdb.PathScheme) +} + +func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) { accountCheckRange = 3 if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) @@ -679,10 +751,15 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { } func TestGenerateFromEmptySnap(t *testing.T) { + testGenerateFromEmptySnap(t, rawdb.HashScheme) + testGenerateFromEmptySnap(t, rawdb.PathScheme) +} + +func testGenerateFromEmptySnap(t *testing.T, scheme string) { //enableLogging() accountCheckRange = 10 storageCheckRange = 20 - helper := newHelper() + helper := newHelper(scheme) // Add 1K accounts to the trie for i := 0; i < 400; i++ { stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -714,8 +791,13 @@ func TestGenerateFromEmptySnap(t *testing.T) { // This hits a case where the snap verification passes, but there are more elements in the trie // which we must also add. func TestGenerateWithIncompleteStorage(t *testing.T) { + testGenerateWithIncompleteStorage(t, rawdb.HashScheme) + testGenerateWithIncompleteStorage(t, rawdb.PathScheme) +} + +func testGenerateWithIncompleteStorage(t *testing.T, scheme string) { storageCheckRange = 4 - helper := newHelper() + helper := newHelper(scheme) stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} // We add 8 accounts, each one is missing exactly one of the storage slots. This means @@ -813,7 +895,12 @@ func populateDangling(disk ethdb.KeyValueStore) { // // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { - var helper = newHelper() + testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.HashScheme) + testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.PathScheme) +} + +func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string) { + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -848,7 +935,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { // // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { - var helper = newHelper() + testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.HashScheme) + testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.PathScheme) +} + +func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) { + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 9b65a342ba..602b728b28 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -179,7 +179,7 @@ func (test *stateTest) run() bool { storageList = append(storageList, copy2DSet(states.Storages)) } disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit}) + tdb = trie.NewDatabase(disk, &trie.Config{OnCommit: onCommit}) sdb = NewDatabaseWithNodeDB(disk, tdb) byzantium = rand.Intn(2) == 0 ) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 6c90b1cbec..50d485dae3 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -36,14 +36,19 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // Tests that updating a state trie does not leak any database writes prior to // actually committing the state. func TestUpdateLeaks(t *testing.T) { // Create an empty state database - db := rawdb.NewMemoryDatabase() - state, _ := New(types.EmptyRootHash, NewDatabase(db), nil) + var ( + db = rawdb.NewMemoryDatabase() + tdb = trie.NewDatabase(db, nil) + ) + state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil) // Update it with some accounts for i := byte(0); i < 255; i++ { @@ -59,7 +64,7 @@ func TestUpdateLeaks(t *testing.T) { } root := state.IntermediateRoot(false) - if err := state.Database().TrieDB().Commit(root, false); err != nil { + if err := tdb.Commit(root, false); err != nil { t.Errorf("can not commit trie %v to persistent database", root.Hex()) } @@ -77,8 +82,10 @@ func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning transDb := rawdb.NewMemoryDatabase() finalDb := rawdb.NewMemoryDatabase() - transState, _ := New(types.EmptyRootHash, NewDatabase(transDb), nil) - finalState, _ := New(types.EmptyRootHash, NewDatabase(finalDb), nil) + transNdb := trie.NewDatabase(transDb, nil) + finalNdb := trie.NewDatabase(finalDb, nil) + transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil) + finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil) modify := func(state *StateDB, addr common.Address, i, tweak byte) { state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak))) @@ -112,7 +119,7 @@ func TestIntermediateLeaks(t *testing.T) { if err != nil { t.Fatalf("failed to commit transition state: %v", err) } - if err = transState.Database().TrieDB().Commit(transRoot, false); err != nil { + if err = transNdb.Commit(transRoot, false); err != nil { t.Errorf("can not commit trie %v to persistent database", transRoot.Hex()) } @@ -122,7 +129,7 @@ func TestIntermediateLeaks(t *testing.T) { if err != nil { t.Fatalf("failed to commit final state: %v", err) } - if err = finalState.Database().TrieDB().Commit(finalRoot, false); err != nil { + if err = finalNdb.Commit(finalRoot, false); err != nil { t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex()) } @@ -759,9 +766,28 @@ func TestDeleteCreateRevert(t *testing.T) { // the Commit operation fails with an error // If we are missing trie nodes, we should not continue writing to the trie func TestMissingTrieNodes(t *testing.T) { + testMissingTrieNodes(t, rawdb.HashScheme) + testMissingTrieNodes(t, rawdb.PathScheme) +} + +func testMissingTrieNodes(t *testing.T, scheme string) { // Create an initial state with a few accounts - memDb := rawdb.NewMemoryDatabase() - db := NewDatabase(memDb) + var ( + triedb *trie.Database + memDb = rawdb.NewMemoryDatabase() + ) + if scheme == rawdb.PathScheme { + triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{ + CleanCacheSize: 0, + DirtyCacheSize: 0, + }}) // disable caching + } else { + triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{ + CleanCacheSize: 0, + }}) // disable caching + } + db := NewDatabaseWithNodeDB(memDb, triedb) + var root common.Hash state, _ := New(types.EmptyRootHash, db, nil) addr := common.BytesToAddress([]byte("so")) @@ -776,7 +802,7 @@ func TestMissingTrieNodes(t *testing.T) { root, _, _ = state.Commit(0, nil) t.Logf("root: %x", root) // force-flush - state.Database().TrieDB().Cap(0) + triedb.Commit(root, false) } // Create a new state on the old root state, _ = New(root, db, nil) @@ -985,7 +1011,8 @@ func TestFlushOrderDataLoss(t *testing.T) { // Create a state trie with many accounts and slots var ( memdb = rawdb.NewMemoryDatabase() - statedb = NewDatabase(memdb) + triedb = trie.NewDatabase(memdb, nil) + statedb = NewDatabaseWithNodeDB(memdb, triedb) state, _ = New(types.EmptyRootHash, statedb, nil) ) for a := byte(0); a < 10; a++ { @@ -999,11 +1026,11 @@ func TestFlushOrderDataLoss(t *testing.T) { if err != nil { t.Fatalf("failed to commit state trie: %v", err) } - statedb.TrieDB().Reference(root, common.Hash{}) - if err := statedb.TrieDB().Cap(1024); err != nil { + triedb.Reference(root, common.Hash{}) + if err := triedb.Cap(1024); err != nil { t.Fatalf("failed to cap trie dirty cache: %v", err) } - if err := statedb.TrieDB().Commit(root, false); err != nil { + if err := triedb.Commit(root, false); err != nil { t.Fatalf("failed to commit state trie: %v", err) } // Reopen the state trie from flushed disk and verify it @@ -1057,7 +1084,7 @@ func TestStateDBTransientStorage(t *testing.T) { func TestResetObject(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk) + tdb = trie.NewDatabase(disk, nil) db = NewDatabaseWithNodeDB(disk, tdb) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash, 128, false) state, _ = New(types.EmptyRootHash, db, snaps) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index c70876fb85..c935c02ae4 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -28,6 +28,8 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // testAccount is the data associated with an account used by the state tests. @@ -39,10 +41,17 @@ type testAccount struct { } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { +func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) { // Create an empty state + config := &trie.Config{Preimages: true} + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } else { + config.HashDB = hashdb.Defaults + } db := rawdb.NewMemoryDatabase() - sdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + nodeDb := trie.NewDatabase(db, config) + sdb := NewDatabaseWithNodeDB(db, nodeDb) state, _ := New(types.EmptyRootHash, sdb, nil) // Fill it with some arbitrary data @@ -67,7 +76,6 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { obj.SetState(hash, hash) } } - state.updateStateObject(obj) accounts = append(accounts, acc) } state.Finalise(false) @@ -75,18 +83,22 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { root, _, _ := state.Commit(0, nil) // Return the generated state - return db, sdb, root, accounts + return db, sdb, nodeDb, root, accounts } // checkStateAccounts cross references a reconstructed state with an expected // account array. -func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { +func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root common.Hash, accounts []*testAccount) { + var config trie.Config + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } // Check root availability and state contents - state, err := New(root, NewDatabase(db), nil) + state, err := New(root, NewDatabaseWithConfig(db, &config), nil) if err != nil { t.Fatalf("failed to create state trie at %x: %v", root, err) } - if err := checkStateConsistency(db, root); err != nil { + if err := checkStateConsistency(db, scheme, root); err != nil { t.Fatalf("inconsistent state trie at %x: %v", root, err) } for i, acc := range accounts { @@ -103,8 +115,12 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou } // checkStateConsistency checks that all data of a state root is present. -func checkStateConsistency(db ethdb.Database, root common.Hash) error { - state, err := New(root, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) +func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) error { + config := &trie.Config{Preimages: true} + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } + state, err := New(root, NewDatabaseWithConfig(db, config), nil) if err != nil { return err } @@ -116,8 +132,14 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Tests that an empty state is not scheduled for syncing. func TestEmptyStateSync(t *testing.T) { - db := trie.NewDatabase(rawdb.NewMemoryDatabase()) - sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, db.Scheme()) + dbA := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) + dbB := trie.NewDatabase(rawdb.NewMemoryDatabase(), &trie.Config{PathDB: pathdb.Defaults}) + + sync := NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbA.Scheme()) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes) + } + sync = NewStateSync(types.EmptyRootHash, rawdb.NewMemoryDatabase(), nil, dbB.Scheme()) if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes) } @@ -126,22 +148,28 @@ func TestEmptyStateSync(t *testing.T) { // Tests that given a root hash, a state can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go. func TestIterativeStateSyncIndividual(t *testing.T) { - testIterativeStateSync(t, 1, false, false) + testIterativeStateSync(t, 1, false, false, rawdb.HashScheme) + testIterativeStateSync(t, 1, false, false, rawdb.PathScheme) } func TestIterativeStateSyncBatched(t *testing.T) { - testIterativeStateSync(t, 100, false, false) + testIterativeStateSync(t, 100, false, false, rawdb.HashScheme) + testIterativeStateSync(t, 100, false, false, rawdb.PathScheme) } func TestIterativeStateSyncIndividualFromDisk(t *testing.T) { - testIterativeStateSync(t, 1, true, false) + testIterativeStateSync(t, 1, true, false, rawdb.HashScheme) + testIterativeStateSync(t, 1, true, false, rawdb.PathScheme) } func TestIterativeStateSyncBatchedFromDisk(t *testing.T) { - testIterativeStateSync(t, 100, true, false) + testIterativeStateSync(t, 100, true, false, rawdb.HashScheme) + testIterativeStateSync(t, 100, true, false, rawdb.PathScheme) } func TestIterativeStateSyncIndividualByPath(t *testing.T) { - testIterativeStateSync(t, 1, false, true) + testIterativeStateSync(t, 1, false, true, rawdb.HashScheme) + testIterativeStateSync(t, 1, false, true, rawdb.PathScheme) } func TestIterativeStateSyncBatchedByPath(t *testing.T) { - testIterativeStateSync(t, 100, false, true) + testIterativeStateSync(t, 100, false, true, rawdb.HashScheme) + testIterativeStateSync(t, 100, false, true, rawdb.PathScheme) } // stateElement represents the element in the state trie(bytecode or trie node). @@ -152,17 +180,17 @@ type stateElement struct { syncPath trie.SyncPath } -func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { +func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, scheme string) { // Create a random state to copy - srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) if commit { - srcDb.TrieDB().Commit(srcRoot, false) + ndb.Commit(srcRoot, false) } - srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB()) + srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), ndb) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme()) var ( nodeElements []stateElement @@ -177,9 +205,11 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { }) } for i := 0; i < len(codes); i++ { - codeElements = append(codeElements, stateElement{ - code: codes[i], - }) + codeElements = append(codeElements, stateElement{code: codes[i]}) + } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) } for len(nodeElements)+len(codeElements) > 0 { var ( @@ -207,7 +237,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) } id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root) - stTrie, err := trie.New(id, srcDb.TrieDB()) + stTrie, err := trie.New(id, ndb) if err != nil { t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) } @@ -218,7 +248,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } } else { - data, err := srcDb.TrieDB().Node(node.hash) + owner, inner := trie.ResolvePath([]byte(node.path)) + data, err := reader.Node(owner, inner, node.hash) if err != nil { t.Fatalf("failed to retrieve node data for key %v", []byte(node.path)) } @@ -262,18 +293,23 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { copyPreimages(srcDisk, dstDb) // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned, and the others sent only later. func TestIterativeDelayedStateSync(t *testing.T) { + testIterativeDelayedStateSync(t, rawdb.HashScheme) + testIterativeDelayedStateSync(t, rawdb.PathScheme) +} + +func testIterativeDelayedStateSync(t *testing.T, scheme string) { // Create a random state to copy - srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme()) var ( nodeElements []stateElement @@ -288,9 +324,11 @@ func TestIterativeDelayedStateSync(t *testing.T) { }) } for i := 0; i < len(codes); i++ { - codeElements = append(codeElements, stateElement{ - code: codes[i], - }) + codeElements = append(codeElements, stateElement{code: codes[i]}) + } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) } for len(nodeElements)+len(codeElements) > 0 { // Sync only half of the scheduled nodes @@ -315,7 +353,8 @@ func TestIterativeDelayedStateSync(t *testing.T) { if len(nodeElements) > 0 { nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1) for i, element := range nodeElements[:len(nodeResults)] { - data, err := srcDb.TrieDB().Node(element.hash) + owner, inner := trie.ResolvePath([]byte(element.path)) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve contract bytecode for %x", element.code) } @@ -355,22 +394,28 @@ func TestIterativeDelayedStateSync(t *testing.T) { copyPreimages(srcDisk, dstDb) // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that given a root hash, a trie can sync iteratively on a single thread, // requesting retrieval tasks and returning all of them in one go, however in a // random order. -func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) } -func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) } +func TestIterativeRandomStateSyncIndividual(t *testing.T) { + testIterativeRandomStateSync(t, 1, rawdb.HashScheme) + testIterativeRandomStateSync(t, 1, rawdb.PathScheme) +} +func TestIterativeRandomStateSyncBatched(t *testing.T) { + testIterativeRandomStateSync(t, 100, rawdb.HashScheme) + testIterativeRandomStateSync(t, 100, rawdb.PathScheme) +} -func testIterativeRandomStateSync(t *testing.T, count int) { +func testIterativeRandomStateSync(t *testing.T, count int, scheme string) { // Create a random state to copy - srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme()) nodeQueue := make(map[string]stateElement) codeQueue := make(map[common.Hash]struct{}) @@ -385,6 +430,10 @@ func testIterativeRandomStateSync(t *testing.T, count int) { for _, hash := range codes { codeQueue[hash] = struct{}{} } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) + } for len(nodeQueue)+len(codeQueue) > 0 { // Fetch all the queued nodes in a random order if len(codeQueue) > 0 { @@ -405,7 +454,8 @@ func testIterativeRandomStateSync(t *testing.T, count int) { if len(nodeQueue) > 0 { results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) for path, element := range nodeQueue { - data, err := srcDb.TrieDB().Node(element.hash) + owner, inner := trie.ResolvePath([]byte(element.path)) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path) } @@ -417,7 +467,6 @@ func testIterativeRandomStateSync(t *testing.T, count int) { } } } - // Feed the retrieved results back and queue new tasks batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { t.Fatalf("failed to commit data: %v", err) @@ -443,18 +492,23 @@ func testIterativeRandomStateSync(t *testing.T, count int) { copyPreimages(srcDisk, dstDb) // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that the trie scheduler can correctly reconstruct the state even if only // partial results are returned (Even those randomly), others sent only later. func TestIterativeRandomDelayedStateSync(t *testing.T) { + testIterativeRandomDelayedStateSync(t, rawdb.HashScheme) + testIterativeRandomDelayedStateSync(t, rawdb.PathScheme) +} + +func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) { // Create a random state to copy - srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme()) nodeQueue := make(map[string]stateElement) codeQueue := make(map[common.Hash]struct{}) @@ -469,6 +523,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { for _, hash := range codes { codeQueue[hash] = struct{}{} } + reader, err := ndb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) + } for len(nodeQueue)+len(codeQueue) > 0 { // Sync only half of the scheduled nodes, even those in random order if len(codeQueue) > 0 { @@ -497,7 +555,8 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { for path, element := range nodeQueue { delete(nodeQueue, path) - data, err := srcDb.TrieDB().Node(element.hash) + owner, inner := trie.ResolvePath([]byte(element.path)) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x", element.hash) } @@ -537,14 +596,19 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { copyPreimages(srcDisk, dstDb) // Cross check that the two states are in sync - checkStateAccounts(t, dstDb, srcRoot, srcAccounts) + checkStateAccounts(t, dstDb, ndb.Scheme(), srcRoot, srcAccounts) } // Tests that at any point in time during a sync, only complete sub-tries are in // the database. func TestIncompleteStateSync(t *testing.T) { + testIncompleteStateSync(t, rawdb.HashScheme) + testIncompleteStateSync(t, rawdb.PathScheme) +} + +func testIncompleteStateSync(t *testing.T, scheme string) { // Create a random state to copy - db, srcDb, srcRoot, srcAccounts := makeTestState() + db, srcDb, ndb, srcRoot, srcAccounts := makeTestState(scheme) // isCodeLookup to save some hashing var isCode = make(map[common.Hash]struct{}) @@ -557,14 +621,14 @@ func TestIncompleteStateSync(t *testing.T) { // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() - sched := NewStateSync(srcRoot, dstDb, nil, srcDb.TrieDB().Scheme()) + sched := NewStateSync(srcRoot, dstDb, nil, ndb.Scheme()) var ( addedCodes []common.Hash addedPaths []string addedHashes []common.Hash ) - reader, err := srcDb.TrieDB().Reader(srcRoot) + reader, err := ndb.Reader(srcRoot) if err != nil { t.Fatalf("state is not available %x", srcRoot) } @@ -651,12 +715,11 @@ func TestIncompleteStateSync(t *testing.T) { for _, node := range addedCodes { val := rawdb.ReadCode(dstDb, node) rawdb.DeleteCode(dstDb, node) - if err := checkStateConsistency(dstDb, srcRoot); err == nil { + if err := checkStateConsistency(dstDb, ndb.Scheme(), srcRoot); err == nil { t.Errorf("trie inconsistency not caught, missing: %x", node) } rawdb.WriteCode(dstDb, node, val) } - scheme := srcDb.TrieDB().Scheme() for i, path := range addedPaths { owner, inner := trie.ResolvePath([]byte(path)) hash := addedHashes[i] @@ -665,7 +728,7 @@ func TestIncompleteStateSync(t *testing.T) { t.Error("missing trie node") } rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme) - if err := checkStateConsistency(dstDb, srcRoot); err == nil { + if err := checkStateConsistency(dstDb, scheme, srcRoot); err == nil { t.Errorf("trie inconsistency not caught, missing: %v", path) } rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme) diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index c5b9f690d8..d2a98ed7bf 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -39,7 +39,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -86,7 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) } }) @@ -107,7 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -135,7 +135,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) diff --git a/eth/api_backend.go b/eth/api_backend.go index 30dc98f084..3192823148 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -426,7 +426,7 @@ func (b *EthAPIBackend) StartMining() error { } func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { - return b.eth.StateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) + return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) } func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { diff --git a/eth/api_debug.go b/eth/api_debug.go index 9cfa9103fb..6afa046787 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -322,7 +322,7 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } - triedb := api.eth.BlockChain().StateCache().TrieDB() + triedb := api.eth.BlockChain().TrieDB() oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb) if err != nil { diff --git a/eth/backend.go b/eth/backend.go index 012586968d..cf593fbb71 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -146,8 +146,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, config.TriesInMemory); err != nil { - log.Error("Failed to recover state", "error", err) + if config.StateScheme == rawdb.HashScheme { + if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, config.TriesInMemory); err != nil { + log.Error("Failed to recover state", "error", err) + } } chainConfig, genesisHash, err := core.LoadChainConfig(chainDb, config.Genesis) if err != nil { @@ -211,6 +213,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { SnapshotLimit: config.SnapshotCache, TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, + StateHistory: config.StateHistory, + StateScheme: config.StateScheme, } ) bcOps := make([]core.BlockChainOption, 0) @@ -234,7 +238,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.OverrideVerkle != nil { overrides.OverrideVerkle = config.OverrideVerkle } - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit, bcOps...) + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory, bcOps...) if err != nil { return nil, err } @@ -532,7 +536,7 @@ func (s *Ethereum) StartMining() error { } // If mining is started, we can disable the transaction rejection mechanism // introduced to speed sync times. - s.handler.acceptTxs.Store(true) + s.handler.enableSyncedFeatures() go s.miner.Start() } @@ -566,7 +570,7 @@ func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader } func (s *Ethereum) Synced() bool { return s.handler.acceptTxs.Load() } -func (s *Ethereum) SetSynced() { s.handler.acceptTxs.Store(true) } +func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } func (s *Ethereum) Merger() *consensus.Merger { return s.merger } diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index d791723b05..1bf03411d1 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // Test chain parameters. @@ -43,7 +44,7 @@ var ( Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - testGenesis = testGspec.MustCommit(testDB) + testGenesis = testGspec.MustCommit(testDB, trie.NewDatabase(testDB, trie.HashDefaults)) ) // The common prefix of all test chains: diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 499b181d28..384996e7fc 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/parlia" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/downloader" @@ -60,6 +61,9 @@ var Defaults = Config{ SyncMode: downloader.SnapSync, NetworkId: 1, TxLookupLimit: 2350000, + TransactionHistory: 2350000, + StateHistory: params.FullImmutabilityThreshold, + StateScheme: rawdb.HashScheme, LightPeers: 100, DatabaseCache: 512, TrieCleanCache: 154, @@ -114,7 +118,11 @@ type Config struct { PipeCommit bool RangeLimit bool - TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + // Deprecated, use 'TransactionHistory' instead. + TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. + StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top // RequiredBlocks is a set of block number -> hash mappings which must be in the // canonical chain of all remote peers. Setting the option makes geth verify the diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 88ded21384..9752d6fd8a 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -33,6 +33,9 @@ func (c Config) MarshalTOML() (interface{}, error) { PipeCommit bool RangeLimit bool TxLookupLimit uint64 `toml:",omitempty"` + TransactionHistory uint64 `toml:",omitempty"` + StateHistory uint64 `toml:",omitempty"` + StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` LightIngress int `toml:",omitempty"` @@ -85,6 +88,9 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.PipeCommit = c.PipeCommit enc.RangeLimit = c.RangeLimit enc.TxLookupLimit = c.TxLookupLimit + enc.TransactionHistory = c.TransactionHistory + enc.StateHistory = c.StateHistory + enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks enc.LightServ = c.LightServ enc.LightIngress = c.LightIngress @@ -141,6 +147,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { PipeCommit *bool RangeLimit *bool TxLookupLimit *uint64 `toml:",omitempty"` + TransactionHistory *uint64 `toml:",omitempty"` + StateHistory *uint64 `toml:",omitempty"` + StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` LightIngress *int `toml:",omitempty"` @@ -228,6 +237,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } + if dec.TransactionHistory != nil { + c.TransactionHistory = *dec.TransactionHistory + } + if dec.StateHistory != nil { + c.StateHistory = *dec.StateHistory + } + if dec.StateScheme != nil { + c.StateScheme = *dec.StateScheme + } if dec.RequiredBlocks != nil { c.RequiredBlocks = dec.RequiredBlocks } diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index e620d2b608..bc0f308c0e 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -44,7 +44,7 @@ var ( Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(testdb) + genesis = gspec.MustCommit(testdb, trie.NewDatabase(testdb, trie.HashDefaults)) unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index bf1ff3804f..dfd988e04e 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -86,7 +86,7 @@ func BenchmarkFilters(b *testing.B) { // The test txs are not properly signed, can't simply create a chain // and then import blocks. TODO(rjl493456442) try to get rid of the // manual database writes. - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) for i, block := range chain { rawdb.WriteBlock(db, block) @@ -184,7 +184,7 @@ func TestFilters(t *testing.T) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = gspec.Commit(db, trie.NewDatabase(db)) + _, err = gspec.Commit(db, trie.NewDatabase(db, nil)) if err != nil { t.Fatal(err) } diff --git a/eth/handler.go b/eth/handler.go index 782c930c3c..1ff9823e2d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -45,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) const ( @@ -237,8 +238,7 @@ func newHandler(config *handlerConfig) (*handler, error) { // round on next sync cycle if h.snapSync.Load() { log.Info("Snap sync complete, auto disabling") - h.snapSync.Store(false) - } + h.snapSync.Store(false) } // If we've successfully finished a sync cycle, accept transactions from // the network h.acceptTxs.Store(true) @@ -327,7 +327,7 @@ func newHandler(config *handlerConfig) (*handler, error) { } n, err := h.chain.InsertChain(blocks) if err == nil { - h.acceptTxs.Store(true) // Mark initial sync done on any fetcher import + h.enableSyncedFeatures() // Mark initial sync done on any fetcher import } return n, err } @@ -956,3 +956,12 @@ func (h *handler) voteBroadcastLoop() { } } } + +// enableSyncedFeatures enables the post-sync functionalities when the initial +// sync is finished. +func (h *handler) enableSyncedFeatures() { + h.acceptTxs.Store(true) + if h.chain.TrieDB().Scheme() == rawdb.PathScheme { + h.chain.TrieDB().SetBufferSize(pathdb.DefaultBufferSize) + } +} diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 6272ccd845..f2f8ee2d2b 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -112,7 +112,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int, panic(err) } for _, block := range bs { - chain.StateCache().TrieDB().Commit(block.Root(), false) + chain.TrieDB().Commit(block.Root(), false) } txconfig := legacypool.DefaultConfig txconfig.Journal = "" // Don't litter the disk with test journals diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index f9fbf72b7b..cdac0da21a 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -246,6 +247,10 @@ func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { // ServiceGetNodeDataQuery assembles the response to a node data query. It is // exposed to allow external packages to test protocol behavior. func ServiceGetNodeDataQuery(chain *core.BlockChain, query GetNodeDataPacket) [][]byte { + // Request nodes by hash is not supported in path-based scheme. + if chain.TrieDB().Scheme() == rawdb.PathScheme { + return nil + } // Gather state data until the fetch or network limits is reached var ( bytes int @@ -257,7 +262,7 @@ func ServiceGetNodeDataQuery(chain *core.BlockChain, query GetNodeDataPacket) [] break } // Retrieve the requested state entry - entry, err := chain.TrieNode(hash) + entry, err := chain.TrieDB().Node(hash) if len(entry) == 0 || err != nil { // Read the contract code with prefix only to save unnecessary lookups. entry, err = chain.ContractCodeWithPrefix(hash) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 1c6d80d354..b2fd03766e 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -284,7 +284,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac req.Bytes = softResponseLimit } // Retrieve the requested state and bail out if non existent - tr, err := trie.New(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) + tr, err := trie.New(trie.StateTrieID(req.Root), chain.TrieDB()) if err != nil { return nil, nil } @@ -414,7 +414,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) + accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.TrieDB()) if err != nil { return nil, nil } @@ -423,7 +423,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP return nil, nil } id := trie.StorageTrieID(req.Root, account, acc.Root) - stTrie, err := trie.NewStateTrie(id, chain.StateCache().TrieDB()) + stTrie, err := trie.NewStateTrie(id, chain.TrieDB()) if err != nil { return nil, nil } @@ -487,7 +487,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s req.Bytes = softResponseLimit } // Make sure we have the state associated with the request - triedb := chain.StateCache().TrieDB() + triedb := chain.TrieDB() accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), triedb) if err != nil { diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 0aa6fd8730..1514ad4e13 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" "golang.org/x/exp/slices" @@ -561,6 +562,11 @@ func noProofStorageRequestHandler(t *testPeer, requestId uint64, root common.Has func TestSyncBloatedProof(t *testing.T) { t.Parallel() + testSyncBloatedProof(t, rawdb.HashScheme) + testSyncBloatedProof(t, rawdb.PathScheme) +} + +func testSyncBloatedProof(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -570,7 +576,7 @@ func TestSyncBloatedProof(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme) source := newTestPeer("source", t, term) source.accountTrie = sourceAccountTrie.Copy() source.accountValues = elems @@ -638,6 +644,11 @@ func setupSyncer(scheme string, peers ...*testPeer) *Syncer { func TestSync(t *testing.T) { t.Parallel() + testSync(t, rawdb.HashScheme) + testSync(t, rawdb.PathScheme) +} + +func testSync(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -647,7 +658,7 @@ func TestSync(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -659,7 +670,7 @@ func TestSync(t *testing.T) { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncTinyTriePanic tests a basic sync with one peer, and a tiny trie. This caused a @@ -667,6 +678,11 @@ func TestSync(t *testing.T) { func TestSyncTinyTriePanic(t *testing.T) { t.Parallel() + testSyncTinyTriePanic(t, rawdb.HashScheme) + testSyncTinyTriePanic(t, rawdb.PathScheme) +} + +func testSyncTinyTriePanic(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -676,7 +692,7 @@ func TestSyncTinyTriePanic(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(1) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(1, scheme) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -690,13 +706,18 @@ func TestSyncTinyTriePanic(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSync tests a basic sync with multiple peers func TestMultiSync(t *testing.T) { t.Parallel() + testMultiSync(t, rawdb.HashScheme) + testMultiSync(t, rawdb.PathScheme) +} + +func testMultiSync(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -706,7 +727,7 @@ func TestMultiSync(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -720,13 +741,18 @@ func TestMultiSync(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorage tests basic sync using accounts + storage + code func TestSyncWithStorage(t *testing.T) { t.Parallel() + testSyncWithStorage(t, rawdb.HashScheme) + testSyncWithStorage(t, rawdb.PathScheme) +} + +func testSyncWithStorage(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -736,7 +762,7 @@ func TestSyncWithStorage(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(3, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 3, 3000, true, false) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -752,13 +778,18 @@ func TestSyncWithStorage(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all func TestMultiSyncManyUseless(t *testing.T) { t.Parallel() + testMultiSyncManyUseless(t, rawdb.HashScheme) + testMultiSyncManyUseless(t, rawdb.PathScheme) +} + +func testMultiSyncManyUseless(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -768,7 +799,7 @@ func TestMultiSyncManyUseless(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -801,11 +832,18 @@ func TestMultiSyncManyUseless(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUseless contains one good peer, and many which doesn't return anything valuable at all func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { + t.Parallel() + + testMultiSyncManyUselessWithLowTimeout(t, rawdb.HashScheme) + testMultiSyncManyUselessWithLowTimeout(t, rawdb.PathScheme) +} + +func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -815,7 +853,7 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -853,11 +891,18 @@ func TestMultiSyncManyUselessWithLowTimeout(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestMultiSyncManyUnresponsive contains one good peer, and many which doesn't respond at all func TestMultiSyncManyUnresponsive(t *testing.T) { + t.Parallel() + + testMultiSyncManyUnresponsive(t, rawdb.HashScheme) + testMultiSyncManyUnresponsive(t, rawdb.PathScheme) +} + +func testMultiSyncManyUnresponsive(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -867,7 +912,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -903,7 +948,7 @@ func TestMultiSyncManyUnresponsive(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } func checkStall(t *testing.T, term func()) chan struct{} { @@ -925,6 +970,11 @@ func checkStall(t *testing.T, term func()) chan struct{} { func TestSyncBoundaryAccountTrie(t *testing.T) { t.Parallel() + testSyncBoundaryAccountTrie(t, rawdb.HashScheme) + testSyncBoundaryAccountTrie(t, rawdb.PathScheme) +} + +func testSyncBoundaryAccountTrie(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -934,7 +984,7 @@ func TestSyncBoundaryAccountTrie(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(3000) + nodeScheme, sourceAccountTrie, elems := makeBoundaryAccountTrie(scheme, 3000) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -952,7 +1002,7 @@ func TestSyncBoundaryAccountTrie(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCappedPeer tests sync using accounts and no storage, where one peer is @@ -960,6 +1010,11 @@ func TestSyncBoundaryAccountTrie(t *testing.T) { func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneCappedPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneCappedPeer(t, rawdb.PathScheme) +} + +func testSyncNoStorageAndOneCappedPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -969,7 +1024,7 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme) mkSource := func(name string, slow bool) *testPeer { source := newTestPeer(name, t, term) @@ -994,7 +1049,7 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCodeCorruptPeer has one peer which doesn't deliver @@ -1002,6 +1057,11 @@ func TestSyncNoStorageAndOneCappedPeer(t *testing.T) { func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneCodeCorruptPeer(t, rawdb.PathScheme) +} + +func testSyncNoStorageAndOneCodeCorruptPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1011,7 +1071,7 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme) mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1034,12 +1094,17 @@ func TestSyncNoStorageAndOneCodeCorruptPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneAccountCorruptPeer(t, rawdb.PathScheme) +} + +func testSyncNoStorageAndOneAccountCorruptPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1049,7 +1114,7 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme) mkSource := func(name string, accFn accountHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1072,7 +1137,7 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncNoStorageAndOneCodeCappedPeer has one peer which delivers code hashes @@ -1080,6 +1145,11 @@ func TestSyncNoStorageAndOneAccountCorruptPeer(t *testing.T) { func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { t.Parallel() + testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.HashScheme) + testSyncNoStorageAndOneCodeCappedPeer(t, rawdb.PathScheme) +} + +func testSyncNoStorageAndOneCodeCappedPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1089,7 +1159,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(3000, scheme) mkSource := func(name string, codeFn codeHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1123,7 +1193,7 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { if threshold := 100; counter > threshold { t.Logf("Error, expected < %d invocations, got %d", threshold, counter) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncBoundaryStorageTrie tests sync against a few normal peers, but the @@ -1131,6 +1201,11 @@ func TestSyncNoStorageAndOneCodeCappedPeer(t *testing.T) { func TestSyncBoundaryStorageTrie(t *testing.T) { t.Parallel() + testSyncBoundaryStorageTrie(t, rawdb.HashScheme) + testSyncBoundaryStorageTrie(t, rawdb.PathScheme) +} + +func testSyncBoundaryStorageTrie(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1140,7 +1215,7 @@ func TestSyncBoundaryStorageTrie(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(10, 1000, false, true) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 10, 1000, false, true) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -1160,7 +1235,7 @@ func TestSyncBoundaryStorageTrie(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorageAndOneCappedPeer tests sync using accounts + storage, where one peer is @@ -1168,6 +1243,11 @@ func TestSyncBoundaryStorageTrie(t *testing.T) { func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { t.Parallel() + testSyncWithStorageAndOneCappedPeer(t, rawdb.HashScheme) + testSyncWithStorageAndOneCappedPeer(t, rawdb.PathScheme) +} + +func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1177,7 +1257,7 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(300, 1000, false, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 300, 1000, false, false) mkSource := func(name string, slow bool) *testPeer { source := newTestPeer(name, t, term) @@ -1202,7 +1282,7 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorageAndCorruptPeer tests sync using accounts + storage, where one peer is @@ -1210,6 +1290,11 @@ func TestSyncWithStorageAndOneCappedPeer(t *testing.T) { func TestSyncWithStorageAndCorruptPeer(t *testing.T) { t.Parallel() + testSyncWithStorageAndCorruptPeer(t, rawdb.HashScheme) + testSyncWithStorageAndCorruptPeer(t, rawdb.PathScheme) +} + +func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1219,7 +1304,7 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1243,12 +1328,17 @@ func TestSyncWithStorageAndCorruptPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { t.Parallel() + testSyncWithStorageAndNonProvingPeer(t, rawdb.HashScheme) + testSyncWithStorageAndNonProvingPeer(t, rawdb.PathScheme) +} + +func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1258,7 +1348,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(100, 3000, true, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1281,7 +1371,7 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { t.Fatalf("sync failed: %v", err) } close(done) - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } // TestSyncWithStorage tests basic sync using accounts + storage + code, against @@ -1290,6 +1380,12 @@ func TestSyncWithStorageAndNonProvingPeer(t *testing.T) { // did not mark the account for healing. func TestSyncWithStorageMisbehavingProve(t *testing.T) { t.Parallel() + + testSyncWithStorageMisbehavingProve(t, rawdb.HashScheme) + testSyncWithStorageMisbehavingProve(t, rawdb.PathScheme) +} + +func testSyncWithStorageMisbehavingProve(t *testing.T, scheme string) { var ( once sync.Once cancel = make(chan struct{}) @@ -1299,7 +1395,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(10, 30, false) + nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorageWithUniqueStorage(scheme, 10, 30, false) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -1314,7 +1410,7 @@ func TestSyncWithStorageMisbehavingProve(t *testing.T) { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } type kv struct { @@ -1364,9 +1460,9 @@ func getCodeByHash(hash common.Hash) []byte { } // makeAccountTrieNoStorage spits out a trie, along with the leafs -func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { +func makeAccountTrieNoStorage(n int, scheme string) (string, *trie.Trie, []*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv ) @@ -1396,12 +1492,12 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { // makeBoundaryAccountTrie constructs an account trie. Instead of filling // accounts normally, this function will fill a few accounts which have // boundary hash. -func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { +func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { var ( entries []*kv boundaries []common.Hash - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) ) // Initialize boundaries @@ -1457,9 +1553,9 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts // has a unique storage set. -func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { +func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv storageRoots = make(map[common.Hash]common.Hash) @@ -1512,9 +1608,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) } // makeAccountTrieWithStorage spits out a trie, along with the leafs -func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { +func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) entries []*kv storageRoots = make(map[common.Hash]common.Hash) @@ -1656,9 +1752,9 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo return root, nodes, entries } -func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { +func verifyTrie(scheme string, db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() - triedb := trie.NewDatabase(rawdb.NewDatabase(db)) + triedb := trie.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme)) accTrie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { t.Fatal(err) @@ -1700,6 +1796,13 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { // TestSyncAccountPerformance tests how efficient the snap algo is at minimizing // state healing func TestSyncAccountPerformance(t *testing.T) { + t.Parallel() + + testSyncAccountPerformance(t, rawdb.HashScheme) + testSyncAccountPerformance(t, rawdb.PathScheme) +} + +func testSyncAccountPerformance(t *testing.T, scheme string) { // Set the account concurrency to 1. This _should_ result in the // range root to become correct, and there should be no healing needed defer func(old int) { accountConcurrency = old }(accountConcurrency) @@ -1714,7 +1817,7 @@ func TestSyncAccountPerformance(t *testing.T) { }) } ) - nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100) + nodeScheme, sourceAccountTrie, elems := makeAccountTrieNoStorage(100, scheme) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -1727,7 +1830,7 @@ func TestSyncAccountPerformance(t *testing.T) { if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) } - verifyTrie(syncer.db, sourceAccountTrie.Hash(), t) + verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) // The trie root will always be requested, since it is added when the snap // sync cycle starts. When popping the queue, we do not look it up again. // Doing so would bring this number down to zero in this artificial testcase, @@ -1787,3 +1890,10 @@ func TestSlotEstimation(t *testing.T) { } } } + +func newDbConfig(scheme string) *trie.Config { + if scheme == rawdb.HashScheme { + return &trie.Config{} + } + return &trie.Config{PathDB: pathdb.Defaults} +} diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 503d63e4af..71a38253ef 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -38,31 +39,11 @@ import ( // for releasing state. var noopReleaser = tracers.StateReleaseFunc(func() {}) -// StateAtBlock retrieves the state database associated with a certain block. -// If no state is locally available for the given block, a number of blocks -// are attempted to be reexecuted to generate the desired state. The optional -// base layer statedb can be provided which is regarded as the statedb of the -// parent block. -// -// An additional release function will be returned if the requested state is -// available. Release is expected to be invoked when the returned state is no longer needed. -// Its purpose is to prevent resource leaking. Though it can be noop in some cases. -// -// Parameters: -// - block: The block for which we want the state(state = block.Root) -// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state -// - base: If the caller is tracing multiple blocks, the caller can provide the parent -// state continuously from the callsite. -// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should -// be made from caller, e.g. perform Commit or other 'save-to-disk' changes. -// Otherwise, the trash generated by caller may be persisted permanently. -// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is -// provided, it would be preferable to start from a fresh state, if we have it -// on disk. -func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { +func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { var ( current *types.Block database state.Database + triedb *trie.Database report = true origin = block.NumberU64() ) @@ -73,9 +54,9 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe // on top to prevent garbage collection and return a release // function to deref it. if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil { - statedb.Database().TrieDB().Reference(block.Root(), common.Hash{}) + eth.blockchain.TrieDB().Reference(block.Root(), common.Hash{}) return statedb, func() { - statedb.Database().TrieDB().Dereference(block.Root()) + eth.blockchain.TrieDB().Dereference(block.Root()) }, nil } } @@ -86,14 +67,16 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe if preferDisk { // Create an ephemeral trie.Database for isolating the live one. Otherwise // the internal junks created by tracing will be persisted into the disk. - database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) + // TODO(rjl493456442), clean cache is disabled to prevent memory leak, + // please re-enable it for better performance. + database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults) if statedb, err = state.New(block.Root(), database, nil); err == nil { log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) return statedb, noopReleaser, nil } } // The optional base statedb is given, mark the start point as parent block - statedb, database, report = base, base.Database(), false + statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if current == nil { return nil, nil, fmt.Errorf("missing parent block %v %d", block.ParentHash(), block.NumberU64()-1) @@ -104,7 +87,10 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe // Create an ephemeral trie.Database for isolating the live one. Otherwise // the internal junks created by tracing will be persisted into the disk. - database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) + // TODO(rjl493456442), clean cache is disabled to prevent memory leak, + // please re-enable it for better performance. + triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults) + database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb) // If we didn't check the live database, do check state over ephemeral database, // otherwise we would rewind past a persisted block (specific corner case is @@ -182,17 +168,58 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe } // Hold the state reference and also drop the parent state // to prevent accumulating too many nodes in memory. - database.TrieDB().Reference(root, common.Hash{}) + triedb.Reference(root, common.Hash{}) if parent != (common.Hash{}) { - database.TrieDB().Dereference(parent) + triedb.Dereference(parent) } parent = root } if report { - nodes, imgs := database.TrieDB().Size() + nodes, imgs := triedb.Size() log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) } - return statedb, func() { database.TrieDB().Dereference(block.Root()) }, nil + return statedb, func() { triedb.Dereference(block.Root()) }, nil +} + +func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) { + // Check if the requested state is available in the live chain. + statedb, err := eth.blockchain.StateAt(block.Root()) + if err == nil { + return statedb, noopReleaser, nil + } + // TODO historic state is not supported in path-based scheme. + // Fully archive node in pbss will be implemented by relying + // on state history, but needs more work on top. + return nil, nil, errors.New("historical state not available in path scheme yet") +} + +// stateAtBlock retrieves the state database associated with a certain block. +// If no state is locally available for the given block, a number of blocks +// are attempted to be reexecuted to generate the desired state. The optional +// base layer statedb can be provided which is regarded as the statedb of the +// parent block. +// +// An additional release function will be returned if the requested state is +// available. Release is expected to be invoked when the returned state is no +// longer needed. Its purpose is to prevent resource leaking. Though it can be +// noop in some cases. +// +// Parameters: +// - block: The block for which we want the state(state = block.Root) +// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state +// - base: If the caller is tracing multiple blocks, the caller can provide the parent +// state continuously from the callsite. +// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should +// be made from caller, e.g. perform Commit or other 'save-to-disk' changes. +// Otherwise, the trash generated by caller may be persisted permanently. +// - preferDisk: This arg can be used by the caller to signal that even though the 'base' is +// provided, it would be preferable to start from a fresh state, if we have it +// on disk. +func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { + if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme { + return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk) + } + return eth.pathState(block) } // stateAtTransaction returns the execution environment of a certain transaction. @@ -208,7 +235,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, } // Lookup the statedb of parent block from the live database, // otherwise regenerate it on the flight. - statedb, release, err := eth.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := eth.stateAtBlock(ctx, parent, reexec, nil, true, false) if err != nil { return nil, vm.BlockContext{}, nil, nil, err } diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 26c4455e30..6df49a90c1 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -137,8 +137,10 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) + triedb.Close() + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) @@ -237,7 +239,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer triedb.Close() b.ReportAllocs() b.ResetTimer() @@ -363,7 +366,7 @@ func TestInternals(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), core.GenesisAlloc{ to: core.GenesisAccount{ Code: tc.code, @@ -371,7 +374,9 @@ func TestInternals(t *testing.T) { origin: core.GenesisAccount{ Balance: big.NewInt(500000000000000), }, - }, false) + }, false, rawdb.HashScheme) + defer triedb.Close() + evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer}) msg := &core.Message{ To: &to, diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 85e95401a2..423167b13c 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -100,7 +100,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer triedb.Close() // Create the tracer, the EVM environment and run it tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 991da10b3a..b4fa5b6272 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -108,8 +108,10 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) + defer triedb.Close() + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 759e3a4dd3..b4989ec984 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -79,7 +79,9 @@ func BenchmarkTransactionTrace(b *testing.B) { Code: []byte{}, Balance: big.NewInt(500000000000000), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) + defer triedb.Close() + // Create the tracer, the EVM environment and run it tracer := logger.NewStructLogger(&logger.Config{ Debug: false, diff --git a/internal/flags/categories.go b/internal/flags/categories.go index 610180cdf2..7a6b8d374c 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -22,6 +22,7 @@ const ( EthCategory = "ETHEREUM" LightCategory = "LIGHT CLIENT" DevCategory = "DEVELOPER CHAIN" + StateCategory = "STATE HISTORY MANAGEMENT" EthashCategory = "ETHASH" TxPoolCategory = "TRANSACTION POOL (EVM)" BlobPoolCategory = "TRANSACTION POOL (BLOB)" diff --git a/les/client.go b/les/client.go index d0f08beb4c..d95c0ff159 100644 --- a/les/client.go +++ b/les/client.go @@ -98,7 +98,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if config.OverrideVerkle != nil { overrides.OverrideVerkle = config.OverrideVerkle } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, trie.NewDatabase(chainDb), config.Genesis, &overrides) + triedb := trie.NewDatabase(chainDb, trie.HashDefaults) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, triedb, config.Genesis, &overrides) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } diff --git a/les/handler_test.go b/les/handler_test.go index 564d3d2b4b..3b7e39d996 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -406,7 +406,7 @@ func testGetProofs(t *testing.T, protocol int) { accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().Number.Uint64(); i++ { header := bc.GetHeaderByNumber(i) - trie, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) + trie, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB()) for _, acc := range accounts { req := ProofReq{ @@ -457,7 +457,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { proofsV2 := light.NewNodeSet() - t, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) + t, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB()) t.Prove(account, proofsV2) expected = proofsV2.NodeList() } @@ -513,7 +513,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxData: [][]byte{rlp}, } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) - trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.ChtTablePrefix)))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.ChtTablePrefix)), trie.HashDefaults)) trie.Prove(key, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ @@ -578,7 +578,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.BloomTrieTablePrefix)))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.BloomTrieTablePrefix)), trie.HashDefaults)) trie.Prove(key, &proofs.Proofs) // Send the proof request and verify the response diff --git a/les/odr_test.go b/les/odr_test.go index 853b4ab565..69824a92dd 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -104,7 +104,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon for _, addr := range acc { if bc != nil { header := bc.GetHeaderByHash(bhash) - st, err = state.New(header.Root, state.NewDatabase(db), nil) + st, err = state.New(header.Root, bc.StateCache(), nil) } else { header := lc.GetHeaderByHash(bhash) st = light.NewState(ctx, header, lc.Odr()) diff --git a/les/server_handler.go b/les/server_handler.go index ad18d3f4e4..c31cc69cd5 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -390,7 +390,8 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { if root == (common.Hash{}) { return nil } - trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + triedb := trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix), trie.HashDefaults) + trie, _ := trie.New(trie.TrieID(root), triedb) return trie } diff --git a/les/server_requests.go b/les/server_requests.go index 6641fab117..f2a747d2d2 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -303,9 +303,8 @@ func handleGetCode(msg Decoder) (serveRequestFn, uint64, uint64, error) { p.bumpInvalid() continue } - triedb := bc.StateCache().TrieDB() address := common.BytesToAddress(request.AccountAddress) - account, err := getAccount(triedb, header.Root, address) + account, err := getAccount(bc.TrieDB(), header.Root, address) if err != nil { p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", address, "err", err) p.bumpInvalid() @@ -424,7 +423,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { default: // Account key specified, open a storage trie address := common.BytesToAddress(request.AccountAddress) - account, err := getAccount(statedb.TrieDB(), root, address) + account, err := getAccount(bc.TrieDB(), root, address) if err != nil { p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", address, "err", err) p.bumpInvalid() diff --git a/les/test_helper.go b/les/test_helper.go index e5d85024e1..63f1b47b9b 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -49,6 +49,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -188,7 +189,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index BaseFee: big.NewInt(params.InitialBaseFee), } ) - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) chain, _ := light.NewLightChain(odr, gspec.Config, engine) client := &LightEthereum{ @@ -226,7 +227,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da BaseFee: big.NewInt(params.InitialBaseFee), } ) - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) // create a simulation backend and pre-commit several customized block to the database. simulation := backends.NewSimulatedBackendWithDatabase(db, gspec.Alloc, 100000000) diff --git a/light/lightchain_test.go b/light/lightchain_test.go index e3d756f801..5694ca72c2 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // So we can deterministically seed different blockchains @@ -55,7 +56,7 @@ func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) [ func newCanonical(n int) (ethdb.Database, *LightChain, error) { db := rawdb.NewMemoryDatabase() gspec := core.Genesis{Config: params.TestChainConfig} - genesis := gspec.MustCommit(db) + genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker()) // Create and inject the requested chain @@ -75,7 +76,7 @@ func newTestLightChain() *LightChain { Difficulty: big.NewInt(1), Config: params.TestChainConfig, } - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker()) if err != nil { panic(err) diff --git a/light/odr_test.go b/light/odr_test.go index 79f901bbdb..d8a7f10675 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -282,7 +283,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { t.Fatal(err) } - gspec.MustCommit(ldb) + gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults)) odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} lightchain, err := NewLightChain(odr, gspec.Config, ethash.NewFullFaker()) if err != nil { diff --git a/light/postprocess.go b/light/postprocess.go index 567814e2bf..13d75f8617 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -145,7 +145,7 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, dis diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down + triedb: trie.NewDatabase(trieTable, trie.HashDefaults), sectionSize: size, disablePruning: disablePruning, } @@ -348,7 +348,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin diskdb: db, odr: odr, trieTable: trieTable, - triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down + triedb: trie.NewDatabase(trieTable, trie.HashDefaults), parentSize: parentSize, size: size, disablePruning: disablePruning, diff --git a/light/trie.go b/light/trie.go index 306fe392a6..529f1e5d89 100644 --- a/light/trie.go +++ b/light/trie.go @@ -224,7 +224,8 @@ func (t *odrTrie) do(key []byte, fn func() error) error { } else { id = trie.StateTrieID(t.id.StateRoot) } - t.trie, err = trie.New(id, trie.NewDatabase(t.db.backend.Database())) + triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults) + t.trie, err = trie.New(id, triedb) } if err == nil { err = fn() @@ -260,7 +261,8 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { } else { id = trie.StateTrieID(t.id.StateRoot) } - t, err := trie.New(id, trie.NewDatabase(t.db.backend.Database())) + triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults) + t, err := trie.New(id, triedb) if err == nil { it.t.trie = t } diff --git a/light/trie_test.go b/light/trie_test.go index ad7d769c84..fe724e9eea 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -50,7 +50,7 @@ func TestNodeIterator(t *testing.T) { panic(err) } - gspec.MustCommit(lightdb) + gspec.MustCommit(lightdb, trie.NewDatabase(lightdb, trie.HashDefaults)) ctx := context.Background() odr := &testOdr{sdb: fulldb, ldb: lightdb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} head := blockchain.CurrentHeader() diff --git a/light/txpool_test.go b/light/txpool_test.go index 1181e3889e..1eec7bc427 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) type testTxRelay struct { @@ -96,7 +97,7 @@ func TestTxPool(t *testing.T) { panic(err) } - gspec.MustCommit(ldb) + gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults)) odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} relay := &testTxRelay{ send: make(chan int, 1), diff --git a/miner/miner_test.go b/miner/miner_test.go index 21db1ce4a9..489bc46a91 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -288,8 +288,9 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { } // Create chainConfig chainDB := rawdb.NewMemoryDatabase() + triedb := trie.NewDatabase(chainDB, nil) genesis := minerTestGenesisBlock(15, 11_500_000, common.HexToAddress("12345")) - chainConfig, _, err := core.SetupGenesisBlock(chainDB, trie.NewDatabase(chainDB), genesis) + chainConfig, _, err := core.SetupGenesisBlock(chainDB, triedb, genesis) if err != nil { t.Fatalf("can't create new chain config: %v", err) } @@ -300,7 +301,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { if err != nil { t.Fatalf("can't create new chain %v", err) } - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(chainDB), nil) + statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache(), nil) blockchain := &testBlockChain{chainConfig, statedb, 10000000, new(event.Feed)} pool := legacypool.New(testTxPoolConfig, blockchain) diff --git a/tests/block_test.go b/tests/block_test.go index 2405da1cc7..ad522ad653 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -18,6 +18,8 @@ package tests import ( "testing" + + "github.com/ethereum/go-ethereum/core/rawdb" ) func TestBlockchain(t *testing.T) { @@ -48,11 +50,17 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if err := bt.checkFailure(t, test.Run(false, nil)); err != nil { - t.Errorf("test without snapshotter failed: %v", err) + if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil { + t.Errorf("test in hash mode without snapshotter failed: %v", err) + } + if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil { + t.Errorf("test in hash mode with snapshotter failed: %v", err) + } + if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil { + t.Errorf("test in path mode without snapshotter failed: %v", err) } - if err := bt.checkFailure(t, test.Run(true, nil)); err != nil { - t.Errorf("test with snapshotter failed: %v", err) + if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil { + t.Errorf("test in path mode with snapshotter failed: %v", err) } }) // There is also a LegacyTests folder, containing blockchain tests generated diff --git a/tests/block_test_util.go b/tests/block_test_util.go index d3e525a387..712174884b 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -38,6 +38,9 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // A BlockTest checks handling of entire blocks. @@ -100,16 +103,30 @@ type btHeaderMarshaling struct { BaseFeePerGas *math.HexOrDecimal256 } -func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error { +func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error { config, ok := Forks[t.json.Network] if !ok { return UnsupportedForkError{t.json.Network} } - // import pre accounts & construct test genesis block & state root - db := rawdb.NewMemoryDatabase() + var ( + db = rawdb.NewMemoryDatabase() + tconf = &trie.Config{} + ) + if scheme == rawdb.PathScheme { + tconf.PathDB = pathdb.Defaults + } else { + tconf.HashDB = hashdb.Defaults + } + // Commit genesis state gspec := t.genesis(config) - gblock := gspec.MustCommit(db) + triedb := trie.NewDatabase(db, tconf) + gblock, err := gspec.Commit(db, triedb) + if err != nil { + return err + } + triedb.Close() // close the db to prevent memory leak + if gblock.Hash() != t.json.Genesis.Hash { return fmt.Errorf("genesis block hash doesn't match test: computed=%x, test=%x", gblock.Hash().Bytes()[:6], t.json.Genesis.Hash[:6]) } @@ -119,7 +136,7 @@ func (t *BlockTest) Run(snapshotter bool, tracer vm.EVMLogger) error { // Wrap the original engine within the beacon-engine engine := beacon.New(ethash.NewFaker()) - cache := &core.CacheConfig{TrieCleanLimit: 0} + cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme} if snapshotter { cache.SnapshotLimit = 1 cache.SnapshotWait = true diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index c62253a726..c29bb2ef12 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addresses []common.Address, txHashes []co } func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { - chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) - bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults)) + bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), trie.HashDefaults)) for i := 0; i < testChainLen; i++ { // The element in CHT is -> key := make([]byte, 8) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index aa81e5c9d9..ba490b761f 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -56,7 +56,7 @@ func (f *fuzzer) readInt() uint64 { } func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { - trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) size := f.readInt() // Fill it with some fluff diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 391bdf300b..3d65524095 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -136,10 +136,10 @@ func (f *fuzzer) fuzz() int { // This spongeDb is used to check the sequence of disk-db-writes var ( spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA)) + dbA = trie.NewDatabase(rawdb.NewDatabase(spongeA), nil) trieA = trie.NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} - dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB)) + dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB), nil) trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme()) }) diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index fe9bf3d0fd..687f5efb1c 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -143,7 +143,7 @@ func Fuzz(input []byte) int { func runRandTest(rt randTest) error { var ( - triedb = trie.NewDatabase(rawdb.NewMemoryDatabase()) + triedb = trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) tr = trie.NewEmpty(triedb) origin = types.EmptyRootHash values = make(map[string]string) // tracks content of the trie diff --git a/tests/state_test.go b/tests/state_test.go index 782f1b0b4c..094dafcafd 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" @@ -78,21 +80,52 @@ func TestState(t *testing.T) { subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) - t.Run(key+"/trie", func(t *testing.T) { + t.Run(key+"/hash/trie", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - _, _, err := test.Run(subtest, vmconfig, false) - return st.checkFailure(t, err) + var result error + test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + result = st.checkFailure(t, err) + }) + return result }) }) - t.Run(key+"/snap", func(t *testing.T) { + t.Run(key+"/hash/snap", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - snaps, statedb, err := test.Run(subtest, vmconfig, true) - if snaps != nil && statedb != nil { - if _, err := snaps.Journal(statedb.IntermediateRoot(false)); err != nil { - return err + var result error + test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + if snaps != nil && state != nil { + if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { + result = err + return + } } - } - return st.checkFailure(t, err) + result = st.checkFailure(t, err) + }) + return result + }) + }) + t.Run(key+"/path/trie", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + result = st.checkFailure(t, err) + }) + return result + }) + }) + t.Run(key+"/path/snap", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + var result error + test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + if snaps != nil && state != nil { + if _, err := snaps.Journal(state.IntermediateRoot(false)); err != nil { + result = err + return + } + } + result = st.checkFailure(t, err) + }) + return result }) }) } @@ -190,7 +223,8 @@ func runBenchmark(b *testing.B, t *StateTest) { vmconfig.ExtraEips = eips block := t.genesis(config).ToBlock() - _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false) + triedb, _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false, rawdb.HashScheme) + defer triedb.Close() var baseFee *big.Int if rules.IsLondon { diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 6ade3d200b..c1d48edc21 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -39,6 +39,8 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "golang.org/x/crypto/sha3" ) @@ -187,43 +189,50 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error { } // Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) { - snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter) - if checkedErr := t.checkError(subtest, err); checkedErr != nil { - return snaps, statedb, checkedErr +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) { + triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) + + // Invoke the callback at the end of function for further analysis. + defer func() { + postCheck(result, snaps, statedb) + + if triedb != nil { + triedb.Close() + } + }() + checkedErr := t.checkError(subtest, err) + if checkedErr != nil { + return checkedErr } // The error has been checked; if it was unexpected, it's already returned. if err != nil { // Here, an error exists but it was expected. // We do not check the post state or logs. - return snaps, statedb, nil + return nil } post := t.json.Post[subtest.Fork][subtest.Index] // N.B: We need to do this in a two-step process, because the first Commit takes care // of self-destructs, and we need to touch the coinbase _after_ it has potentially self-destructed. if root != common.Hash(post.Root) { - return snaps, statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) + return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { - return snaps, statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) - } - // Re-init the post-state instance for further operation - statedb, err = state.New(root, statedb.Database(), snaps) - if err != nil { - return nil, nil, err + return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - return snaps, statedb, nil + statedb, _ = state.New(root, statedb.Database(), snaps) + return nil } // RunNoVerify runs a specific subtest and returns the statedb and post-state root -func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, common.Hash, error) { +func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { - return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} + return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips + block := t.genesis(config).ToBlock() - snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) + triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme) var baseFee *big.Int if config.IsLondon(new(big.Int)) { @@ -237,7 +246,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post, baseFee) if err != nil { - return nil, nil, common.Hash{}, err + triedb.Close() + return nil, nil, nil, common.Hash{}, err } // Try to recover tx with current signer @@ -245,11 +255,13 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh var ttx types.Transaction err := ttx.UnmarshalBinary(post.TxBytes) if err != nil { - return nil, nil, common.Hash{}, err + triedb.Close() + return nil, nil, nil, common.Hash{}, err } if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { - return nil, nil, common.Hash{}, err + triedb.Close() + return nil, nil, nil, common.Hash{}, err } } @@ -268,6 +280,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context.Difficulty = big.NewInt(0) } evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) + // Execute the message. snapshot := statedb.Snapshot() gaspool := new(core.GasPool) @@ -288,15 +301,22 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) statedb.SetExpectedStateRoot(root) root, _, err = statedb.Commit(block.NumberU64(), nil) - return snaps, statedb, root, err + return triedb, snaps, statedb, root, err } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas] } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool) (*snapshot.Tree, *state.StateDB) { - sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) +func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) { + tconf := &trie.Config{Preimages: true} + if scheme == rawdb.HashScheme { + tconf.HashDB = hashdb.Defaults + } else { + tconf.PathDB = pathdb.Defaults + } + triedb := trie.NewDatabase(db, tconf) + sdb := state.NewDatabaseWithNodeDB(db, triedb) statedb, _ := state.New(types.EmptyRootHash, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) @@ -322,7 +342,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo snaps, _ = snapshot.New(snapconfig, db, sdb.TrieDB(), root, 128, false) } statedb, _ = state.New(root, sdb, snaps) - return snaps, statedb + return triedb, snaps, statedb } func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { diff --git a/trie/database.go b/trie/database.go index ae1f7871a4..6661c7b2ec 100644 --- a/trie/database.go +++ b/trie/database.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" @@ -30,14 +31,21 @@ import ( // Config defines all necessary options for database. type Config struct { NoTries bool - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Preimages bool // Flag whether the preimage of trie key is recorded - PathDB *pathdb.Config // Configs for experimental path-based scheme, not used yet. + Preimages bool // Flag whether the preimage of node key is recorded + HashDB *hashdb.Config // Configs for hash-based scheme + PathDB *pathdb.Config // Configs for experimental path-based scheme // Testing hooks OnCommit func(states *triestate.Set) // Hook invoked when commit is performed } +// HashDefaults represents a config for using hash-based scheme with +// default settings. +var HashDefaults = &Config{ + Preimages: false, + HashDB: hashdb.Defaults, +} + // backend defines the methods needed to access/update trie nodes in different // state scheme. type backend interface { @@ -92,22 +100,30 @@ func prepare(diskdb ethdb.Database, config *Config) *Database { } } -// NewDatabase initializes the trie database with default settings, namely +// NewDatabase initializes the trie database with default settings, note // the legacy hash-based scheme is used by default. -func NewDatabase(diskdb ethdb.Database) *Database { - return NewDatabaseWithConfig(diskdb, nil) -} - -// NewDatabaseWithConfig initializes the trie database with provided configs. -// The path-based scheme is not activated yet, always initialized with legacy -// hash-based scheme by default. -func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database { - var cleans int - if config != nil && config.Cache != 0 { - cleans = config.Cache * 1024 * 1024 +func NewDatabase(diskdb ethdb.Database, config *Config) *Database { + // Sanitize the config and use the default one if it's not specified. + if config == nil { + config = HashDefaults + } + var preimages *preimageStore + if config.Preimages { + preimages = newPreimageStore(diskdb) + } + db := &Database{ + config: config, + diskdb: diskdb, + preimages: preimages, + } + if config.HashDB != nil && config.PathDB != nil { + log.Crit("Both 'hash' and 'path' mode are configured") + } + if config.PathDB != nil { + db.backend = pathdb.New(diskdb, config.PathDB) + } else { + db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) } - db := prepare(diskdb, config) - db.backend = hashdb.New(diskdb, cleans, mptResolver{}) return db } @@ -245,3 +261,60 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { } return hdb.Node(hash) } + +// Recover rollbacks the database to a specified historical point. The state is +// supported as the rollback destination only if it's canonical state and the +// corresponding trie histories are existent. It's only supported by path-based +// database and will return an error for others. +func (db *Database) Recover(target common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Recover(target, &trieLoader{db: db}) +} + +// Recoverable returns the indicator if the specified state is enabled to be +// recovered. It's only supported by path-based database and will return an +// error for others. +func (db *Database) Recoverable(root common.Hash) (bool, error) { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return false, errors.New("not supported") + } + return pdb.Recoverable(root), nil +} + +// Reset wipes all available journal from the persistent database and discard +// all caches and diff layers. Using the given root to create a new disk layer. +// It's only supported by path-based database and will return an error for others. +func (db *Database) Reset(root common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Reset(root) +} + +// Journal commits an entire diff hierarchy to disk into a single journal entry. +// This is meant to be used during shutdown to persist the snapshot without +// flattening everything down (bad for reorgs). It's only supported by path-based +// database and will return an error for others. +func (db *Database) Journal(root common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Journal(root) +} + +// SetBufferSize sets the node buffer size to the provided value(in bytes). +// It's only supported by path-based database and will return an error for +// others. +func (db *Database) SetBufferSize(size int) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.SetBufferSize(size) +} diff --git a/trie/database_test.go b/trie/database_test.go index ed43a81e59..d675a2191b 100644 --- a/trie/database_test.go +++ b/trie/database_test.go @@ -27,7 +27,7 @@ import ( func newTestDatabase(diskdb ethdb.Database, scheme string) *Database { db := prepare(diskdb, nil) if scheme == rawdb.HashScheme { - db.backend = hashdb.New(diskdb, 0, mptResolver{}) + db.backend = hashdb.New(diskdb, &hashdb.Config{}, mptResolver{}) } else { db.backend = pathdb.New(diskdb, &pathdb.Config{}) // disable clean/dirty cache } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index e711ffb815..57d1f06a16 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -18,7 +18,6 @@ package trie import ( "bytes" - "encoding/binary" "fmt" "math/rand" "testing" @@ -27,13 +26,11 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/trie/trienode" ) func TestEmptyIterator(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) iter := trie.MustNodeIterator(nil) seen := make(map[string]struct{}) @@ -46,7 +43,7 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -89,7 +86,7 @@ func (k *kv) cmp(other *kv) int { } func TestIteratorLargeData(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -208,7 +205,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range testdata1 { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -249,7 +246,7 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase()) + dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) @@ -258,7 +255,7 @@ func TestDifferenceIterator(t *testing.T) { dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase()) + dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) @@ -291,7 +288,7 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase()) + dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) @@ -300,7 +297,7 @@ func TestUnionIterator(t *testing.T) { dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase()) + dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) @@ -344,7 +341,7 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range testdata1 { tr.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -537,96 +534,6 @@ func TestIteratorNodeBlob(t *testing.T) { testIteratorNodeBlob(t, rawdb.PathScheme) } -type loggingDb struct { - getCount uint64 - backend ethdb.KeyValueStore -} - -func (l *loggingDb) Has(key []byte) (bool, error) { - return l.backend.Has(key) -} - -func (l *loggingDb) Get(key []byte) ([]byte, error) { - l.getCount++ - return l.backend.Get(key) -} - -func (l *loggingDb) Put(key []byte, value []byte) error { - return l.backend.Put(key, value) -} - -func (l *loggingDb) Delete(key []byte) error { - return l.backend.Delete(key) -} - -func (l *loggingDb) NewBatch() ethdb.Batch { - return l.backend.NewBatch() -} - -func (l *loggingDb) NewBatchWithSize(size int) ethdb.Batch { - return l.backend.NewBatchWithSize(size) -} - -func (l *loggingDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - return l.backend.NewIterator(prefix, start) -} - -func (l *loggingDb) NewSnapshot() (ethdb.Snapshot, error) { - return l.backend.NewSnapshot() -} - -func (l *loggingDb) Stat(property string) (string, error) { - return l.backend.Stat(property) -} - -func (l *loggingDb) Compact(start []byte, limit []byte) error { - return l.backend.Compact(start, limit) -} - -func (l *loggingDb) Close() error { - return l.backend.Close() -} - -// makeLargeTestTrie create a sample test trie -func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { - // Create an empty trie - logDb := &loggingDb{0, memorydb.New()} - triedb := NewDatabase(rawdb.NewDatabase(logDb)) - trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb) - - // Fill it with some arbitrary data - for i := 0; i < 10000; i++ { - key := make([]byte, 32) - val := make([]byte, 32) - binary.BigEndian.PutUint64(key, uint64(i)) - binary.BigEndian.PutUint64(val, uint64(i)) - key = crypto.Keccak256(key) - val = crypto.Keccak256(val) - trie.MustUpdate(key, val) - } - root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - triedb.Commit(root, false) - - // Return the generated trie - trie, _ = NewStateTrie(TrieID(root), triedb) - return triedb, trie, logDb -} - -// Tests that the node iterator indeed walks over the entire database contents. -func TestNodeIteratorLargeTrie(t *testing.T) { - // Create some arbitrary test trie to iterate - db, trie, logDb := makeLargeTestTrie() - db.Cap(0) // flush everything - // Do a seek operation - trie.NodeIterator(common.FromHex("0x77667766776677766778855885885885")) - // master: 24 get operations - // this pr: 6 get operations - if have, want := logDb.getCount, uint64(6); have != want { - t.Fatalf("Too many lookups during seek, have %d want %d", have, want) - } -} - func testIteratorNodeBlob(t *testing.T, scheme string) { var ( db = rawdb.NewMemoryDatabase() @@ -700,7 +607,7 @@ func isTrieNode(scheme string, key, val []byte) (bool, []byte, common.Hash) { } hash = common.BytesToHash(key) } else { - ok, remain := rawdb.IsAccountTrieNode(key) + ok, remain := rawdb.ResolveAccountTrieNodeKey(key) if !ok { return false, nil, common.Hash{} } diff --git a/trie/proof_test.go b/trie/proof_test.go index e8ea116c88..fc2de62649 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -94,7 +94,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -145,7 +145,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -395,7 +395,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.MustUpdate(entry.k, entry.v) @@ -467,7 +467,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -502,7 +502,7 @@ func TestSingleSideRangeProof(t *testing.T) { // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. func TestReverseSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -609,7 +609,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -683,7 +683,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -1036,7 +1036,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -1055,7 +1055,7 @@ func randomTrie(n int) (*Trie, map[string]*kv) { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { @@ -1080,7 +1080,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { common.Hex2Bytes("02"), common.Hex2Bytes("03"), } - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i, key := range keys { trie.MustUpdate(key, vals[i]) } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 89bc9547d3..af67025006 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -32,14 +32,14 @@ import ( ) func newEmptySecure() *StateTrie { - trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase())) + trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase(), nil)) return trie } // makeTestStateTrie creates a large enough secure trie for testing. func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb) // Fill it with some arbitrary data diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index ea3eef7882..822dd31a93 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -254,7 +254,7 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) kvs := []struct { K string V string @@ -282,7 +282,7 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) kvs := []struct { K string V string @@ -351,7 +351,7 @@ func TestStacktrieNotModifyValues(t *testing.T) { func TestStacktrieSerialization(t *testing.T) { var ( st = NewStackTrie(nil) - nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) keyB = big.NewInt(1) keyDelta = big.NewInt(1) vals [][]byte diff --git a/trie/sync_test.go b/trie/sync_test.go index b6fe8d84a6..dd3506559d 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -109,8 +109,8 @@ type trieElement struct { // Tests that an empty trie is not scheduled for syncing. func TestEmptySync(t *testing.T) { - dbA := NewDatabase(rawdb.NewMemoryDatabase()) - dbB := NewDatabase(rawdb.NewMemoryDatabase()) + dbA := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + dbB := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) diff --git a/trie/tracer_test.go b/trie/tracer_test.go index 86daec6d20..acb8c2f6bf 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -61,7 +61,7 @@ func TestTrieTracer(t *testing.T) { // Tests if the trie diffs are tracked correctly. Tracer should capture // all non-leaf dirty nodes, no matter the node is embedded or not. func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) // Determine all new nodes are tracked @@ -104,7 +104,7 @@ func TestTrieTracerNoop(t *testing.T) { } func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range vals { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -128,7 +128,7 @@ func TestAccessList(t *testing.T) { func testAccessList(t *testing.T, vals []struct{ k, v string }) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) orig = trie.Copy() ) @@ -211,7 +211,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { // Tests origin values won't be tracked in Iterator or Prover func TestAccessListLeak(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) ) // Create trie from scratch @@ -262,7 +262,7 @@ func TestAccessListLeak(t *testing.T) { // in its parent due to the smaller size of the original tree node. func TestTinyTree(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) ) for _, val := range tiny { diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 1c63ff4544..4215964559 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -20,6 +20,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie/triestate" ) // Reader wraps the Node method of a backing trie store. @@ -83,3 +84,18 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) { } return blob, nil } + +// trieLoader implements triestate.TrieLoader for constructing tries. +type trieLoader struct { + db *Database +} + +// OpenTrie opens the main account trie. +func (l *trieLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { + return New(TrieID(root), l.db) +} + +// OpenStorageTrie opens the storage trie of an account. +func (l *trieLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { + return New(StorageTrieID(stateRoot, addrHash, root), l.db) +} diff --git a/trie/trie_test.go b/trie/trie_test.go index 3cb21c1956..35ccc77201 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -45,7 +45,7 @@ func init() { } func TestEmptyTrie(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) res := trie.Hash() exp := types.EmptyRootHash if res != exp { @@ -54,7 +54,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) key := make([]byte, 32) value := []byte("test") trie.MustUpdate(key, value) @@ -64,8 +64,13 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { + testMissingRoot(t, rawdb.HashScheme) + testMissingRoot(t, rawdb.PathScheme) +} + +func testMissingRoot(t *testing.T, scheme string) { root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") - trie, err := New(TrieID(root), NewDatabase(rawdb.NewMemoryDatabase())) + trie, err := New(TrieID(root), newTestDatabase(rawdb.NewMemoryDatabase(), scheme)) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -161,7 +166,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { } func TestInsert(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -173,7 +178,7 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") @@ -184,7 +189,7 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -209,7 +214,7 @@ func TestGet(t *testing.T) { } func TestDelete(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -236,7 +241,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -260,7 +265,7 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -321,7 +326,7 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) trie.MustUpdate([]byte("key1"), []byte{99, 99, 99, 99}) trie.MustUpdate([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() @@ -604,7 +609,7 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 func benchGet(b *testing.B) { - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { @@ -621,7 +626,7 @@ func benchGet(b *testing.B) { } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -651,7 +656,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) i := 0 for ; i < len(addresses)/2; i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -682,7 +687,7 @@ func BenchmarkCommitAfterHash(b *testing.B) { func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -696,7 +701,7 @@ func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -709,7 +714,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { checktr.MustUpdate(it.Key, it.Value) @@ -722,7 +727,7 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -788,11 +793,17 @@ func (s *spongeDb) Stat(property string) (string, error) { panic("implement func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } func (s *spongeDb) Put(key []byte, value []byte) error { - valbrief := value + var ( + keybrief = key + valbrief = value + ) + if len(keybrief) > 8 { + keybrief = keybrief[:8] + } if len(valbrief) > 8 { valbrief = valbrief[:8] } - s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, key[:8], len(value), valbrief)) + s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, keybrief, len(value), valbrief)) s.sponge.Write(key) s.sponge.Write(value) return nil @@ -830,7 +841,7 @@ func TestCommitSequence(t *testing.T) { addresses, accounts := makeAccounts(tc.count) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -861,7 +872,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { prng := rand.New(rand.NewSource(int64(i))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -893,7 +904,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { prng := rand.New(rand.NewSource(int64(count))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} @@ -952,7 +963,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do. func TestCommitSequenceSmallRoot(t *testing.T) { s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} @@ -1029,7 +1040,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1080,7 +1091,7 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1132,7 +1143,7 @@ func BenchmarkDerefRootFixedSize(b *testing.B) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(triedb) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) diff --git a/trie/triedb/hashdb/database.go b/trie/triedb/hashdb/database.go index d4a344452d..e9fd4722b6 100644 --- a/trie/triedb/hashdb/database.go +++ b/trie/triedb/hashdb/database.go @@ -65,6 +65,20 @@ type ChildResolver interface { ForEach(node []byte, onChild func(common.Hash)) } +// Config contains the settings for database. +type Config struct { + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes +} + +// Defaults is the default setting for database if it's not specified. +// Notably, clean cache is disabled explicitly, +var Defaults = &Config{ + // Explicitly set clean cache size to 0 to avoid creating fastcache, + // otherwise database must be closed when it's no longer needed to + // prevent memory leak. + CleanCacheSize: 0, +} + // Database is an intermediate write layer between the trie data structures and // the disk database. The aim is to accumulate trie writes in-memory and only // periodically flush a couple tries to disk, garbage collecting the remainder. @@ -122,12 +136,13 @@ func (n *cachedNode) forChildren(resolver ChildResolver, onChild func(hash commo } // New initializes the hash-based node database. -func New(diskdb ethdb.Database, size int, resolver ChildResolver) *Database { - // Initialize the clean cache if the specified cache allowance - // is non-zero. Note, the size is in bytes. +func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Database { + if config == nil { + config = Defaults + } var cleans *fastcache.Cache - if size > 0 { - cleans = fastcache.New(size) + if config.CleanCacheSize > 0 { + cleans = fastcache.New(config.CleanCacheSize) } return &Database{ diskdb: diskdb, @@ -638,7 +653,13 @@ func (db *Database) Size() common.StorageSize { } // Close closes the trie database and releases all held resources. -func (db *Database) Close() error { return nil } +func (db *Database) Close() error { + if db.cleans != nil { + db.cleans.Reset() + db.cleans = nil + } + return nil +} // Scheme returns the node scheme used in the database. func (db *Database) Scheme() string { diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index b1a14589a2..8f39ceba7b 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -33,8 +33,26 @@ import ( "github.com/ethereum/go-ethereum/trie/triestate" ) -// maxDiffLayers is the maximum diff layers allowed in the layer tree. -const maxDiffLayers = 128 +const ( + // maxDiffLayers is the maximum diff layers allowed in the layer tree. + maxDiffLayers = 128 + + // defaultCleanSize is the default memory allowance of clean cache. + defaultCleanSize = 16 * 1024 * 1024 + + // maxBufferSize is the maximum memory allowance of node buffer. + // Too large nodebuffer will cause the system to pause for a long + // time when write happens. Also, the largest batch that pebble can + // support is 4GB, node will panic if batch size exceeds this limit. + maxBufferSize = 256 * 1024 * 1024 + + // DefaultBufferSize is the default memory allowance of node buffer + // that aggregates the writes from above until it's flushed into the + // disk. It's meant to be used once the initial sync is finished. + // Do not increase the buffer size arbitrarily, otherwise the system + // pause time will increase when the database writes happen. + DefaultBufferSize = 64 * 1024 * 1024 +) // layer is the interface implemented by all state layers which includes some // public methods and some additional methods for internal usage. @@ -68,30 +86,33 @@ type layer interface { // Config contains the settings for database. type Config struct { - StateLimit uint64 // Number of recent blocks to maintain state history for - CleanSize int // Maximum memory allowance (in bytes) for caching clean nodes - DirtySize int // Maximum memory allowance (in bytes) for caching dirty nodes - ReadOnly bool // Flag whether the database is opened in read only mode. + StateHistory uint64 // Number of recent blocks to maintain state history for + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes + DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes + ReadOnly bool // Flag whether the database is opened in read only mode. } -var ( - // defaultCleanSize is the default memory allowance of clean cache. - defaultCleanSize = 16 * 1024 * 1024 - - // defaultBufferSize is the default memory allowance of node buffer - // that aggregates the writes from above until it's flushed into the - // disk. Do not increase the buffer size arbitrarily, otherwise the - // system pause time will increase when the database writes happen. - defaultBufferSize = 128 * 1024 * 1024 -) +// sanitize checks the provided user configurations and changes anything that's +// unreasonable or unworkable. +func (c *Config) sanitize() *Config { + conf := *c + if conf.DirtyCacheSize > maxBufferSize { + log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(maxBufferSize)) + conf.DirtyCacheSize = maxBufferSize + } + return &conf +} // Defaults contains default settings for Ethereum mainnet. var Defaults = &Config{ - StateLimit: params.FullImmutabilityThreshold, - CleanSize: defaultCleanSize, - DirtySize: defaultBufferSize, + StateHistory: params.FullImmutabilityThreshold, + CleanCacheSize: defaultCleanSize, + DirtyCacheSize: DefaultBufferSize, } +// ReadOnly is the config in order to open database in read only mode. +var ReadOnly = &Config{ReadOnly: true} + // Database is a multiple-layered structure for maintaining in-memory trie nodes. // It consists of one persistent base layer backed by a key-value store, on top // of which arbitrarily many in-memory diff layers are stacked. The memory diffs @@ -123,9 +144,11 @@ func New(diskdb ethdb.Database, config *Config) *Database { if config == nil { config = Defaults } + config = config.sanitize() + db := &Database{ readOnly: config.ReadOnly, - bufferSize: config.DirtySize, + bufferSize: config.DirtyCacheSize, config: config, diskdb: diskdb, } @@ -141,7 +164,7 @@ func New(diskdb ethdb.Database, config *Config) *Database { // is opened at the same time to prevent accidental mutation. if ancient, err := diskdb.AncientDatadir(); err == nil && ancient != "" && !db.readOnly { offset := uint64(0) //TODO(Nathan): just for passing compilation - freezer, err := rawdb.NewStateHistoryFreezer(ancient, false, offset) + freezer, err := rawdb.NewStateFreezer(ancient, false, offset) if err != nil { log.Crit("Failed to open state history freezer", "err", err) } @@ -345,7 +368,14 @@ func (db *Database) Close() error { db.lock.Lock() defer db.lock.Unlock() + // Set the database to read-only mode to prevent all + // following mutations. db.readOnly = true + + // Release the memory held by clean cache. + db.tree.bottom().resetCache() + + // Close the attached state history freezer. if db.freezer == nil { return nil } @@ -383,6 +413,10 @@ func (db *Database) SetBufferSize(size int) error { db.lock.Lock() defer db.lock.Unlock() + if size > maxBufferSize { + log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(maxBufferSize)) + size = maxBufferSize + } db.bufferSize = size return db.tree.bottom().setBufferSize(db.bufferSize) } diff --git a/trie/triedb/pathdb/database_test.go b/trie/triedb/pathdb/database_test.go index 83a299b020..e595b60280 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/trie/triedb/pathdb/database_test.go @@ -46,7 +46,8 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm h.Update(key.Bytes(), val) } } - return h.Commit(false) + root, nodes, _ := h.Commit(false) + return root, nodes } func generateAccount(storageRoot common.Hash) types.StateAccount { @@ -98,7 +99,7 @@ type tester struct { func newTester(t *testing.T) *tester { var ( disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false) - db = New(disk, &Config{CleanSize: 256 * 1024, DirtySize: 256 * 1024}) + db = New(disk, &Config{CleanCacheSize: 256 * 1024, DirtyCacheSize: 256 * 1024}) obj = &tester{ db: db, preimages: make(map[common.Hash]common.Address), diff --git a/trie/triedb/pathdb/difflayer_test.go b/trie/triedb/pathdb/difflayer_test.go index 77c4cd5722..9b5907c3c5 100644 --- a/trie/triedb/pathdb/difflayer_test.go +++ b/trie/triedb/pathdb/difflayer_test.go @@ -29,7 +29,7 @@ import ( func emptyLayer() *diskLayer { return &diskLayer{ db: New(rawdb.NewMemoryDatabase(), nil), - buffer: newNodeBuffer(defaultBufferSize, nil, 0), + buffer: newNodeBuffer(DefaultBufferSize, nil, 0), } } diff --git a/trie/triedb/pathdb/disklayer.go b/trie/triedb/pathdb/disklayer.go index b526b3b7dd..87718290f9 100644 --- a/trie/triedb/pathdb/disklayer.go +++ b/trie/triedb/pathdb/disklayer.go @@ -47,8 +47,8 @@ func newDiskLayer(root common.Hash, id uint64, db *Database, cleans *fastcache.C // Initialize a clean cache if the memory allowance is not zero // or reuse the provided cache if it is not nil (inherited from // the original disk layer). - if cleans == nil && db.config.CleanSize != 0 { - cleans = fastcache.New(db.config.CleanSize) + if cleans == nil && db.config.CleanCacheSize != 0 { + cleans = fastcache.New(db.config.CleanCacheSize) } return &diskLayer{ root: root, @@ -177,7 +177,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // corresponding states(journal), the stored state history will // be truncated in the next restart. if dl.db.freezer != nil { - err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateLimit) + err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateHistory) if err != nil { return nil, err } @@ -276,6 +276,20 @@ func (dl *diskLayer) size() common.StorageSize { return common.StorageSize(dl.buffer.size) } +// resetCache releases the memory held by clean cache to prevent memory leak. +func (dl *diskLayer) resetCache() { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // Stale disk layer loses the ownership of clean cache. + if dl.stale { + return + } + if dl.cleans != nil { + dl.cleans.Reset() + } +} + // hasher is used to compute the sha256 hash of the provided data. type hasher struct{ sha crypto.KeccakState } diff --git a/trie/triedb/pathdb/history_test.go b/trie/triedb/pathdb/history_test.go index 76b37d9f79..82801cf83a 100644 --- a/trie/triedb/pathdb/history_test.go +++ b/trie/triedb/pathdb/history_test.go @@ -226,7 +226,7 @@ func TestTruncateTailHistories(t *testing.T) { // openFreezer initializes the freezer instance for storing state histories. func openFreezer(datadir string, readOnly bool) (*rawdb.ResettableFreezer, error) { - return rawdb.NewStateHistoryFreezer(datadir, readOnly, 0) + return rawdb.NewStateFreezer(datadir, readOnly, 0) } func compareSet[k comparable](a, b map[k][]byte) bool { diff --git a/trie/triedb/pathdb/testutils.go b/trie/triedb/pathdb/testutils.go index 4406dbc521..d6fdacb421 100644 --- a/trie/triedb/pathdb/testutils.go +++ b/trie/triedb/pathdb/testutils.go @@ -80,7 +80,7 @@ func (h *testHasher) Delete(key []byte) error { // Commit computes the new hash of the states and returns the set with all // state changes. -func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { +func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { var ( nodes = make(map[common.Hash][]byte) set = trienode.NewNodeSet(h.owner) @@ -108,7 +108,7 @@ func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { if root == types.EmptyRootHash && h.root != types.EmptyRootHash { set.AddNode(nil, trienode.NewDeleted()) } - return root, set + return root, set, nil } // hash performs the hash computation upon the provided states. diff --git a/trie/triestate/state.go b/trie/triestate/state.go index cb3611baf9..4c47e9c397 100644 --- a/trie/triestate/state.go +++ b/trie/triestate/state.go @@ -43,7 +43,7 @@ type Trie interface { // Commit the trie and returns a set of dirty nodes generated along with // the new root hash. - Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) + Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) } // TrieLoader wraps functions to load tries. @@ -129,7 +129,10 @@ func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Addre return nil, fmt.Errorf("failed to revert state, err: %w", err) } } - root, result := tr.Commit(false) + root, result, err := tr.Commit(false) + if err != nil { + return nil, err + } if root != prevRoot { return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root) } @@ -181,7 +184,10 @@ func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error { return err } } - root, result := st.Commit(false) + root, result, err := st.Commit(false) + if err != nil { + return err + } if root != prev.Root { return errors.New("failed to reset storage trie") } @@ -232,7 +238,10 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error { return err } } - root, result := st.Commit(false) + root, result, err := st.Commit(false) + if err != nil { + return err + } if root != types.EmptyRootHash { return errors.New("failed to clear storage trie") } From 528d97b54132b2b623bdcc09d5980481daaef2ac Mon Sep 17 00:00:00 2001 From: joeycli Date: Sun, 24 Sep 2023 12:21:00 +0800 Subject: [PATCH 3/5] feat: active pbss on bsc fix: lint error fix: ut error fix: code review comments --- cmd/geth/pruneblock_test.go | 7 +- cmd/geth/snapshot.go | 2 + core/blockchain.go | 10 +- core/blockchain_snapshot_test.go | 6 +- core/blockchain_test.go | 746 +++---------------------- core/eip3529tests/eip3529_test_util.go | 5 +- core/genesis_test.go | 20 - core/rawdb/ancient_utils.go | 3 + core/state/database.go | 10 + core/state/statedb.go | 44 +- core/state/statedb_test.go | 4 - core/state_prefetcher_test.go | 13 +- eth/downloader/fetchers_concurrent.go | 40 +- eth/handler.go | 28 +- eth/protocols/trust/handler_test.go | 3 +- eth/sync.go | 2 + ethclient/ethclient_test.go | 5 +- trie/database.go | 47 +- trie/trie.go | 5 +- 19 files changed, 258 insertions(+), 742 deletions(-) diff --git a/cmd/geth/pruneblock_test.go b/cmd/geth/pruneblock_test.go index a20ba01fab..8361559212 100644 --- a/cmd/geth/pruneblock_test.go +++ b/cmd/geth/pruneblock_test.go @@ -41,6 +41,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -136,7 +137,11 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai t.Fatalf("failed to create database with ancient backend") } defer db.Close() - genesis := gspec.MustCommit(db) + + triedb := trie.NewDatabase(db, nil) + defer triedb.Close() + + genesis := gspec.MustCommit(db, triedb) // Initialize a fresh chain with only a genesis block blockchain, err := core.NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index ffae740a1f..0d9e583e79 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -478,6 +478,7 @@ func pruneAllState(ctx *cli.Context) error { } chaindb := utils.MakeChainDatabase(ctx, stack, false, false) + defer chaindb.Close() pruner, err := pruner.NewAllPruner(chaindb) if err != nil { log.Error("Failed to open snapshot tree", "err", err) @@ -495,6 +496,7 @@ func verifyState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, true, false) + defer chaindb.Close() headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") diff --git a/core/blockchain.go b/core/blockchain.go index 08d30872f8..8c49be8311 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -162,7 +162,11 @@ type CacheConfig struct { // triedbConfig derives the configures for trie database. func (c *CacheConfig) triedbConfig() *trie.Config { - config := &trie.Config{Preimages: c.Preimages} + config := &trie.Config{ + Cache: c.TrieCleanLimit, + Preimages: c.Preimages, + NoTries: c.NoTries, + } if c.StateScheme == rawdb.HashScheme { config.HashDB = &hashdb.Config{ CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, @@ -392,6 +396,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis var diskRoot common.Hash if bc.cacheConfig.SnapshotLimit > 0 { diskRoot = rawdb.ReadSnapshotRoot(bc.db) + } else if bc.triedb.Scheme() == rawdb.PathScheme { + _, diskRoot = rawdb.ReadAccountTrieNode(bc.db, nil) } if diskRoot != (common.Hash{}) { log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "snaproot", diskRoot) @@ -875,7 +881,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha log.Crit("Failed to rollback state", "err", err) // Shouldn't happen } } - log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) + log.Info("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash()) break } log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root()) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index f0000eaded..14d0309988 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -483,7 +483,8 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : G // Expected snapshot disk : C4 - for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + //for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + for _, scheme := range []string{rawdb.HashScheme} { test := &crashSnapshotTest{ snapshotTestBasic{ scheme: scheme, @@ -525,7 +526,8 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) { // Expected head fast block: C8 // Expected head block : C2 // Expected snapshot disk : C4 - for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + //for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + for _, scheme := range []string{rawdb.HashScheme} { test := &crashSnapshotTest{ snapshotTestBasic{ scheme: scheme, diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 56b67eedaf..56a5c7763e 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -55,22 +55,13 @@ var ( // chain. Depending on the full flag, if creates either a full block chain or a // header only chain. The database and genesis specification for block generation // are also returned in case more test blocks are needed later. -<<<<<<< HEAD func newCanonical(engine consensus.Engine, n int, full bool, pipeline bool) (ethdb.Database, *Genesis, *BlockChain, error) { -======= -func newCanonical(engine consensus.Engine, n int, full bool, scheme string) (ethdb.Database, *Genesis, *BlockChain, error) { ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) var ( genesis = &Genesis{ BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges, } ) -<<<<<<< HEAD -======= - // Initialize a fresh chain with only a genesis block - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Initialize a fresh chain with only a genesis block var ops []BlockChainOption @@ -99,7 +90,6 @@ func newGwei(n int64) *big.Int { } // Test fork of length N starting from block i -<<<<<<< HEAD func testInvalidStateRootBlockImport(t *testing.T, blockchain *BlockChain, i, n int, pipeline bool) { // Copy old chain up to #i into a new db db, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, true, pipeline) @@ -147,11 +137,6 @@ func testInvalidStateRootBlockImport(t *testing.T, blockchain *BlockChain, i, n func testFork(t *testing.T, blockchain *BlockChain, i, n int, full, pipeline bool, comparator func(td1, td2 *big.Int)) { // Copy old chain up to #i into a new db genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, pipeline) -======= -func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int), scheme string) { - // Copy old chain up to #i into a new db - genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatal("could not make new canonical in testFork", err) } @@ -270,7 +255,6 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error } return nil } -<<<<<<< HEAD func TestBlockImportVerification(t *testing.T) { length := 5 @@ -288,15 +272,6 @@ func TestBlockImportVerification(t *testing.T) { func TestLastBlock(t *testing.T) { genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, false) -======= -func TestLastBlock(t *testing.T) { - testLastBlock(t, rawdb.HashScheme) - testLastBlock(t, rawdb.PathScheme) -} - -func testLastBlock(t *testing.T, scheme string) { - genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -313,13 +288,9 @@ func testLastBlock(t *testing.T, scheme string) { // Test inserts the blocks/headers after the fork choice rule is changed. // The chain is reorged to whatever specified. -func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full bool, scheme string) { +func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full bool) { // Copy old chain up to #i into a new db -<<<<<<< HEAD genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, false) -======= - genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatal("could not make new canonical in testFork", err) } @@ -367,7 +338,6 @@ func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full b // Tests that given a starting canonical chain of a given size, it can be extended // with various length chains. func TestExtendCanonicalHeaders(t *testing.T) { -<<<<<<< HEAD testExtendCanonical(t, false, false) } @@ -381,21 +351,6 @@ func testExtendCanonical(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) -======= - testExtendCanonical(t, false, rawdb.HashScheme) - testExtendCanonical(t, false, rawdb.PathScheme) -} -func TestExtendCanonicalBlocks(t *testing.T) { - testExtendCanonical(t, true, rawdb.HashScheme) - testExtendCanonical(t, true, rawdb.PathScheme) -} - -func testExtendCanonical(t *testing.T, full bool, scheme string) { - length := 5 - - // Make first chain starting from genesis - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -408,51 +363,33 @@ func testExtendCanonical(t *testing.T, full bool, scheme string) { } } // Start fork from current height -<<<<<<< HEAD testFork(t, processor, length, 1, full, pipeline, better) testFork(t, processor, length, 2, full, pipeline, better) testFork(t, processor, length, 5, full, pipeline, better) testFork(t, processor, length, 10, full, pipeline, better) -======= - testFork(t, processor, length, 1, full, better, scheme) - testFork(t, processor, length, 2, full, better, scheme) - testFork(t, processor, length, 5, full, better, scheme) - testFork(t, processor, length, 10, full, better, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that given a starting canonical chain of a given size, it can be extended // with various length chains. -func TestExtendCanonicalHeadersAfterMerge(t *testing.T) { - testExtendCanonicalAfterMerge(t, false, rawdb.HashScheme) - testExtendCanonicalAfterMerge(t, false, rawdb.PathScheme) -} -func TestExtendCanonicalBlocksAfterMerge(t *testing.T) { - testExtendCanonicalAfterMerge(t, true, rawdb.HashScheme) - testExtendCanonicalAfterMerge(t, true, rawdb.PathScheme) -} +func TestExtendCanonicalHeadersAfterMerge(t *testing.T) { testExtendCanonicalAfterMerge(t, false) } +func TestExtendCanonicalBlocksAfterMerge(t *testing.T) { testExtendCanonicalAfterMerge(t, true) } -func testExtendCanonicalAfterMerge(t *testing.T, full bool, scheme string) { +func testExtendCanonicalAfterMerge(t *testing.T, full bool) { length := 5 // Make first chain starting from genesis -<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) -======= - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, length, 1, full, scheme) - testInsertAfterMerge(t, processor, length, 10, full, scheme) + testInsertAfterMerge(t, processor, length, 1, full) + testInsertAfterMerge(t, processor, length, 10, full) } // Tests that given a starting canonical chain of a given size, creating shorter // forks do not take canonical ownership. -<<<<<<< HEAD func TestShorterForkHeaders(t *testing.T) { testShorterFork(t, false, false) } func TestShorterForkBlocks(t *testing.T) { testShorterFork(t, true, false) @@ -464,22 +401,6 @@ func testShorterFork(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) -======= -func TestShorterForkHeaders(t *testing.T) { - testShorterFork(t, false, rawdb.HashScheme) - testShorterFork(t, false, rawdb.PathScheme) -} -func TestShorterForkBlocks(t *testing.T) { - testShorterFork(t, true, rawdb.HashScheme) - testShorterFork(t, true, rawdb.PathScheme) -} - -func testShorterFork(t *testing.T, full bool, scheme string) { - length := 10 - - // Make first chain starting from genesis - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -492,60 +413,40 @@ func testShorterFork(t *testing.T, full bool, scheme string) { } } // Sum of numbers must be less than `length` for this to be a shorter fork -<<<<<<< HEAD testFork(t, processor, 0, 3, full, pipeline, worse) testFork(t, processor, 0, 7, full, pipeline, worse) testFork(t, processor, 1, 1, full, pipeline, worse) testFork(t, processor, 1, 7, full, pipeline, worse) testFork(t, processor, 5, 3, full, pipeline, worse) testFork(t, processor, 5, 4, full, pipeline, worse) -======= - testFork(t, processor, 0, 3, full, worse, scheme) - testFork(t, processor, 0, 7, full, worse, scheme) - testFork(t, processor, 1, 1, full, worse, scheme) - testFork(t, processor, 1, 7, full, worse, scheme) - testFork(t, processor, 5, 3, full, worse, scheme) - testFork(t, processor, 5, 4, full, worse, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that given a starting canonical chain of a given size, creating shorter // forks do not take canonical ownership. -func TestShorterForkHeadersAfterMerge(t *testing.T) { - testShorterForkAfterMerge(t, false, rawdb.HashScheme) - testShorterForkAfterMerge(t, false, rawdb.PathScheme) -} -func TestShorterForkBlocksAfterMerge(t *testing.T) { - testShorterForkAfterMerge(t, true, rawdb.HashScheme) - testShorterForkAfterMerge(t, true, rawdb.PathScheme) -} +func TestShorterForkHeadersAfterMerge(t *testing.T) { testShorterForkAfterMerge(t, false) } +func TestShorterForkBlocksAfterMerge(t *testing.T) { testShorterForkAfterMerge(t, true) } -func testShorterForkAfterMerge(t *testing.T, full bool, scheme string) { +func testShorterForkAfterMerge(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis -<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) -======= - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 3, full, scheme) - testInsertAfterMerge(t, processor, 0, 7, full, scheme) - testInsertAfterMerge(t, processor, 1, 1, full, scheme) - testInsertAfterMerge(t, processor, 1, 7, full, scheme) - testInsertAfterMerge(t, processor, 5, 3, full, scheme) - testInsertAfterMerge(t, processor, 5, 4, full, scheme) + testInsertAfterMerge(t, processor, 0, 3, full) + testInsertAfterMerge(t, processor, 0, 7, full) + testInsertAfterMerge(t, processor, 1, 1, full) + testInsertAfterMerge(t, processor, 1, 7, full) + testInsertAfterMerge(t, processor, 5, 3, full) + testInsertAfterMerge(t, processor, 5, 4, full) } // Tests that given a starting canonical chain of a given size, creating longer // forks do take canonical ownership. func TestLongerForkHeaders(t *testing.T) { -<<<<<<< HEAD testLongerFork(t, false, false) } func TestLongerForkBlocks(t *testing.T) { @@ -558,70 +459,44 @@ func testLongerFork(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) -======= - testLongerFork(t, false, rawdb.HashScheme) - testLongerFork(t, false, rawdb.PathScheme) -} -func TestLongerForkBlocks(t *testing.T) { - testLongerFork(t, true, rawdb.HashScheme) - testLongerFork(t, true, rawdb.PathScheme) -} - -func testLongerFork(t *testing.T, full bool, scheme string) { - length := 10 - - // Make first chain starting from genesis - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 11, full, scheme) - testInsertAfterMerge(t, processor, 0, 15, full, scheme) - testInsertAfterMerge(t, processor, 1, 10, full, scheme) - testInsertAfterMerge(t, processor, 1, 12, full, scheme) - testInsertAfterMerge(t, processor, 5, 6, full, scheme) - testInsertAfterMerge(t, processor, 5, 8, full, scheme) + testInsertAfterMerge(t, processor, 0, 11, full) + testInsertAfterMerge(t, processor, 0, 15, full) + testInsertAfterMerge(t, processor, 1, 10, full) + testInsertAfterMerge(t, processor, 1, 12, full) + testInsertAfterMerge(t, processor, 5, 6, full) + testInsertAfterMerge(t, processor, 5, 8, full) } // Tests that given a starting canonical chain of a given size, creating longer // forks do take canonical ownership. -func TestLongerForkHeadersAfterMerge(t *testing.T) { - testLongerForkAfterMerge(t, false, rawdb.HashScheme) - testLongerForkAfterMerge(t, false, rawdb.PathScheme) -} -func TestLongerForkBlocksAfterMerge(t *testing.T) { - testLongerForkAfterMerge(t, true, rawdb.HashScheme) - testLongerForkAfterMerge(t, true, rawdb.PathScheme) -} +func TestLongerForkHeadersAfterMerge(t *testing.T) { testLongerForkAfterMerge(t, false) } +func TestLongerForkBlocksAfterMerge(t *testing.T) { testLongerForkAfterMerge(t, true) } -func testLongerForkAfterMerge(t *testing.T, full bool, scheme string) { +func testLongerForkAfterMerge(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis -<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) -======= - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 11, full, scheme) - testInsertAfterMerge(t, processor, 0, 15, full, scheme) - testInsertAfterMerge(t, processor, 1, 10, full, scheme) - testInsertAfterMerge(t, processor, 1, 12, full, scheme) - testInsertAfterMerge(t, processor, 5, 6, full, scheme) - testInsertAfterMerge(t, processor, 5, 8, full, scheme) + testInsertAfterMerge(t, processor, 0, 11, full) + testInsertAfterMerge(t, processor, 0, 15, full) + testInsertAfterMerge(t, processor, 1, 10, full) + testInsertAfterMerge(t, processor, 1, 12, full) + testInsertAfterMerge(t, processor, 5, 6, full) + testInsertAfterMerge(t, processor, 5, 8, full) } // Tests that given a starting canonical chain of a given size, creating equal // forks do take canonical ownership. -<<<<<<< HEAD func TestEqualForkHeaders(t *testing.T) { testEqualFork(t, false, false) } func TestEqualForkBlocks(t *testing.T) { testEqualFork(t, true, true) @@ -633,22 +508,6 @@ func testEqualFork(t *testing.T, full, pipeline bool) { // Make first chain starting from genesis _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, pipeline) -======= -func TestEqualForkHeaders(t *testing.T) { - testEqualFork(t, false, rawdb.HashScheme) - testEqualFork(t, false, rawdb.PathScheme) -} -func TestEqualForkBlocks(t *testing.T) { - testEqualFork(t, true, rawdb.HashScheme) - testEqualFork(t, true, rawdb.PathScheme) -} - -func testEqualFork(t *testing.T, full bool, scheme string) { - length := 10 - - // Make first chain starting from genesis - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -661,58 +520,38 @@ func testEqualFork(t *testing.T, full bool, scheme string) { } } // Sum of numbers must be equal to `length` for this to be an equal fork -<<<<<<< HEAD testFork(t, processor, 0, 10, full, pipeline, equal) testFork(t, processor, 1, 9, full, pipeline, equal) testFork(t, processor, 2, 8, full, pipeline, equal) testFork(t, processor, 5, 5, full, pipeline, equal) testFork(t, processor, 6, 4, full, pipeline, equal) testFork(t, processor, 9, 1, full, pipeline, equal) -======= - testFork(t, processor, 0, 10, full, equal, scheme) - testFork(t, processor, 1, 9, full, equal, scheme) - testFork(t, processor, 2, 8, full, equal, scheme) - testFork(t, processor, 5, 5, full, equal, scheme) - testFork(t, processor, 6, 4, full, equal, scheme) - testFork(t, processor, 9, 1, full, equal, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that given a starting canonical chain of a given size, creating equal // forks do take canonical ownership. -func TestEqualForkHeadersAfterMerge(t *testing.T) { - testEqualForkAfterMerge(t, false, rawdb.HashScheme) - testEqualForkAfterMerge(t, false, rawdb.PathScheme) -} -func TestEqualForkBlocksAfterMerge(t *testing.T) { - testEqualForkAfterMerge(t, true, rawdb.HashScheme) - testEqualForkAfterMerge(t, true, rawdb.PathScheme) -} +func TestEqualForkHeadersAfterMerge(t *testing.T) { testEqualForkAfterMerge(t, false) } +func TestEqualForkBlocksAfterMerge(t *testing.T) { testEqualForkAfterMerge(t, true) } -func testEqualForkAfterMerge(t *testing.T, full bool, scheme string) { +func testEqualForkAfterMerge(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis -<<<<<<< HEAD _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, false) -======= - _, _, processor, err := newCanonical(ethash.NewFaker(), length, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } defer processor.Stop() - testInsertAfterMerge(t, processor, 0, 10, full, scheme) - testInsertAfterMerge(t, processor, 1, 9, full, scheme) - testInsertAfterMerge(t, processor, 2, 8, full, scheme) - testInsertAfterMerge(t, processor, 5, 5, full, scheme) - testInsertAfterMerge(t, processor, 6, 4, full, scheme) - testInsertAfterMerge(t, processor, 9, 1, full, scheme) + testInsertAfterMerge(t, processor, 0, 10, full) + testInsertAfterMerge(t, processor, 1, 9, full) + testInsertAfterMerge(t, processor, 2, 8, full) + testInsertAfterMerge(t, processor, 5, 5, full) + testInsertAfterMerge(t, processor, 6, 4, full) + testInsertAfterMerge(t, processor, 9, 1, full) } // Tests that chains missing links do not get accepted by the processor. -<<<<<<< HEAD func TestBrokenHeaderChain(t *testing.T) { testBrokenChain(t, false, false) } func TestBrokenBlockChain(t *testing.T) { testBrokenChain(t, true, false) @@ -722,20 +561,6 @@ func TestBrokenBlockChain(t *testing.T) { func testBrokenChain(t *testing.T, full, pipeline bool) { // Make chain starting from genesis genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 10, full, pipeline) -======= -func TestBrokenHeaderChain(t *testing.T) { - testBrokenChain(t, false, rawdb.HashScheme) - testBrokenChain(t, false, rawdb.PathScheme) -} -func TestBrokenBlockChain(t *testing.T) { - testBrokenChain(t, true, rawdb.HashScheme) - testBrokenChain(t, true, rawdb.PathScheme) -} - -func testBrokenChain(t *testing.T, full bool, scheme string) { - // Make chain starting from genesis - genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 10, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -757,7 +582,6 @@ func testBrokenChain(t *testing.T, full bool, scheme string) { // Tests that reorganising a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. -<<<<<<< HEAD func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false, false) } func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true, false) @@ -766,24 +590,10 @@ func TestReorgLongBlocks(t *testing.T) { func testReorgLong(t *testing.T, full, pipeline bool) { testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280+params.GenesisDifficulty.Int64(), full, pipeline) -======= -func TestReorgLongHeaders(t *testing.T) { - testReorgLong(t, false, rawdb.HashScheme) - testReorgLong(t, false, rawdb.PathScheme) -} -func TestReorgLongBlocks(t *testing.T) { - testReorgLong(t, true, rawdb.HashScheme) - testReorgLong(t, true, rawdb.PathScheme) -} - -func testReorgLong(t *testing.T, full bool, scheme string) { - testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280+params.GenesisDifficulty.Int64(), full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } // Tests that reorganising a short difficult chain after a long easy one // overwrites the canonical numbers and links in the database. -<<<<<<< HEAD func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false, false) } func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true, false) @@ -791,18 +601,6 @@ func TestReorgShortBlocks(t *testing.T) { } func testReorgShort(t *testing.T, full, pipeline bool) { -======= -func TestReorgShortHeaders(t *testing.T) { - testReorgShort(t, false, rawdb.HashScheme) - testReorgShort(t, false, rawdb.PathScheme) -} -func TestReorgShortBlocks(t *testing.T) { - testReorgShort(t, true, rawdb.HashScheme) - testReorgShort(t, true, rawdb.PathScheme) -} - -func testReorgShort(t *testing.T, full bool, scheme string) { ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment // we need a fairly long chain of blocks with different difficulties for a short // one to become heavier than a long one. The 96 is an empirical value. @@ -814,21 +612,12 @@ func testReorgShort(t *testing.T, full bool, scheme string) { for i := 0; i < len(diff); i++ { diff[i] = -9 } -<<<<<<< HEAD testReorg(t, easy, diff, 12615120+params.GenesisDifficulty.Int64(), full, pipeline) } func testReorg(t *testing.T, first, second []int64, td int64, full, pipeline bool) { // Create a pristine chain and database genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) -======= - testReorg(t, easy, diff, 12615120+params.GenesisDifficulty.Int64(), full, scheme) -} - -func testReorg(t *testing.T, first, second []int64, td int64, full bool, scheme string) { - // Create a pristine chain and database - genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -896,7 +685,6 @@ func testReorg(t *testing.T, first, second []int64, td int64, full bool, scheme } // Tests that the insertion functions detect banned hashes. -<<<<<<< HEAD func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false, false) } func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true, true) @@ -906,20 +694,6 @@ func TestBadBlockHashes(t *testing.T) { func testBadHashes(t *testing.T, full, pipeline bool) { // Create a pristine chain and database genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) -======= -func TestBadHeaderHashes(t *testing.T) { - testBadHashes(t, false, rawdb.HashScheme) - testBadHashes(t, false, rawdb.PathScheme) -} -func TestBadBlockHashes(t *testing.T) { - testBadHashes(t, true, rawdb.HashScheme) - testBadHashes(t, true, rawdb.PathScheme) -} - -func testBadHashes(t *testing.T, full bool, scheme string) { - // Create a pristine chain and database - genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -948,7 +722,6 @@ func testBadHashes(t *testing.T, full bool, scheme string) { // Tests that bad hashes are detected on boot, and the chain rolled back to a // good state prior to the bad hash. -<<<<<<< HEAD func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false, false) } func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true, false) @@ -958,20 +731,6 @@ func TestReorgBadBlockHashes(t *testing.T) { func testReorgBadHashes(t *testing.T, full, pipeline bool) { // Create a pristine chain and database genDb, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) -======= -func TestReorgBadHeaderHashes(t *testing.T) { - testReorgBadHashes(t, false, rawdb.HashScheme) - testReorgBadHashes(t, false, rawdb.PathScheme) -} -func TestReorgBadBlockHashes(t *testing.T) { - testReorgBadHashes(t, true, rawdb.HashScheme) - testReorgBadHashes(t, true, rawdb.PathScheme) -} - -func testReorgBadHashes(t *testing.T, full bool, scheme string) { - // Create a pristine chain and database - genDb, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -1001,7 +760,7 @@ func testReorgBadHashes(t *testing.T, full bool, scheme string) { blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(blockchain.db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ncm, err := NewBlockChain(blockchain.db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -1021,7 +780,6 @@ func testReorgBadHashes(t *testing.T, full bool, scheme string) { } // Tests chain insertions in the face of one entity containing an invalid nonce. -<<<<<<< HEAD func TestHeadersInsertNonceError(t *testing.T) { testInsertNonceError(t, false, false) } func TestBlocksInsertNonceError(t *testing.T) { testInsertNonceError(t, true, false) @@ -1032,21 +790,6 @@ func testInsertNonceError(t *testing.T, full bool, pipeline bool) { doTest := func(i int) { // Create a pristine chain and database genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, pipeline) -======= -func TestHeadersInsertNonceError(t *testing.T) { - testInsertNonceError(t, false, rawdb.HashScheme) - testInsertNonceError(t, false, rawdb.PathScheme) -} -func TestBlocksInsertNonceError(t *testing.T) { - testInsertNonceError(t, true, rawdb.HashScheme) - testInsertNonceError(t, true, rawdb.PathScheme) -} - -func testInsertNonceError(t *testing.T, full bool, scheme string) { - doTest := func(i int) { - // Create a pristine chain and database - genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -1101,11 +844,6 @@ func testInsertNonceError(t *testing.T, full bool, scheme string) { // Tests that fast importing a block chain produces the same chain data as the // classical full block processing. func TestFastVsFullChains(t *testing.T) { - testFastVsFullChains(t, rawdb.HashScheme) - testFastVsFullChains(t, rawdb.PathScheme) -} - -func testFastVsFullChains(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1138,7 +876,7 @@ func testFastVsFullChains(t *testing.T, scheme string) { }) // Import the chain as an archive node for the comparison baseline archiveDb := rawdb.NewMemoryDatabase() - archive, _ := NewBlockChain(archiveDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + archive, _ := NewBlockChain(archiveDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer archive.Stop() if n, err := archive.InsertChain(blocks); err != nil { @@ -1146,7 +884,7 @@ func testFastVsFullChains(t *testing.T, scheme string) { } // Fast import the chain as a non-archive node to test fastDb := rawdb.NewMemoryDatabase() - fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -1165,8 +903,7 @@ func testFastVsFullChains(t *testing.T, scheme string) { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - - ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers); err != nil { @@ -1234,11 +971,6 @@ func testFastVsFullChains(t *testing.T, scheme string) { // Tests that various import methods move the chain head pointers to the correct // positions. func TestLightVsFastVsFullChainHeads(t *testing.T) { - testLightVsFastVsFullChainHeads(t, rawdb.HashScheme) - testLightVsFastVsFullChainHeads(t, rawdb.PathScheme) -} - -func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1284,7 +1016,6 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { archiveCaching := *defaultCacheConfig archiveCaching.TrieDirtyDisabled = true - archiveCaching.StateScheme = scheme archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := archive.InsertChain(blocks); err != nil { @@ -1299,7 +1030,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb := makeDb() defer fastDb.Close() - fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -1319,7 +1050,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Import the chain as a ancient-first node and ensure all pointers are updated ancientDb := makeDb() defer ancientDb.Close() - ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers); err != nil { @@ -1338,7 +1069,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Import the chain as a light node and ensure all pointers are updated lightDb := makeDb() defer lightDb.Close() - light, _ := NewBlockChain(lightDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + light, _ := NewBlockChain(lightDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := light.InsertHeaderChain(headers); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) } @@ -1351,11 +1082,6 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Tests that chain reorganisations handle transaction removals and reinsertions. func TestChainTxReorgs(t *testing.T) { - testChainTxReorgs(t, rawdb.HashScheme) - testChainTxReorgs(t, rawdb.PathScheme) -} - -func testChainTxReorgs(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") @@ -1411,7 +1137,7 @@ func testChainTxReorgs(t *testing.T, scheme string) { }) // Import the chain. This runs all block validation rules. db := rawdb.NewMemoryDatabase() - blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -1470,11 +1196,6 @@ func testChainTxReorgs(t *testing.T, scheme string) { } func TestLogReorgs(t *testing.T) { - testLogReorgs(t, rawdb.HashScheme) - testLogReorgs(t, rawdb.PathScheme) -} - -func testLogReorgs(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -1485,7 +1206,7 @@ func testLogReorgs(t *testing.T, scheme string) { signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() rmLogsCh := make(chan RemovedLogsEvent) @@ -1530,18 +1251,13 @@ var logCode = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd // This test checks that log events and RemovedLogsEvent are sent // when the chain reorganizes. func TestLogRebirth(t *testing.T) { - testLogRebirth(t, rawdb.HashScheme) - testLogRebirth(t, rawdb.PathScheme) -} - -func testLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() - blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1612,17 +1328,12 @@ func testLogRebirth(t *testing.T, scheme string) { // This test is a variation of TestLogRebirth. It verifies that log events are emitted // when a side chain containing log events overtakes the canonical chain. func TestSideLogRebirth(t *testing.T) { - testSideLogRebirth(t, rawdb.HashScheme) - testSideLogRebirth(t, rawdb.PathScheme) -} - -func testSideLogRebirth(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} signer = types.LatestSigner(gspec.Config) - blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() @@ -1707,11 +1418,6 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan Re } func TestReorgSideEvent(t *testing.T) { - testReorgSideEvent(t, rawdb.HashScheme) - testReorgSideEvent(t, rawdb.PathScheme) -} - -func testReorgSideEvent(t *testing.T, scheme string) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -1721,7 +1427,7 @@ func testReorgSideEvent(t *testing.T, scheme string) { } signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, gen *BlockGen) {}) @@ -1792,16 +1498,7 @@ done: // Tests if the canonical block can be fetched from the database during chain insertion. func TestCanonicalBlockRetrieval(t *testing.T) { -<<<<<<< HEAD _, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, false) -======= - testCanonicalBlockRetrieval(t, rawdb.HashScheme) - testCanonicalBlockRetrieval(t, rawdb.PathScheme) -} - -func testCanonicalBlockRetrieval(t *testing.T, scheme string) { - _, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, true, scheme) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -1845,12 +1542,8 @@ func testCanonicalBlockRetrieval(t *testing.T, scheme string) { } pend.Wait() } -func TestEIP155Transition(t *testing.T) { - testEIP155Transition(t, rawdb.HashScheme) - testEIP155Transition(t, rawdb.PathScheme) -} -func testEIP155Transition(t *testing.T, scheme string) { +func TestEIP155Transition(t *testing.T) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -1909,7 +1602,7 @@ func testEIP155Transition(t *testing.T, scheme string) { } }) - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if _, err := blockchain.InsertChain(blocks); err != nil { @@ -1959,12 +1652,8 @@ func testEIP155Transition(t *testing.T, scheme string) { t.Errorf("have %v, want %v", have, want) } } -func TestEIP161AccountRemoval(t *testing.T) { - testEIP161AccountRemoval(t, rawdb.HashScheme) - testEIP161AccountRemoval(t, rawdb.PathScheme) -} -func testEIP161AccountRemoval(t *testing.T, scheme string) { +func TestEIP161AccountRemoval(t *testing.T) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -2002,7 +1691,7 @@ func testEIP161AccountRemoval(t *testing.T, scheme string) { block.AddTx(tx) }) // account must exist pre eip 161 - blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if _, err := blockchain.InsertChain(types.Blocks{blocks[0]}); err != nil { @@ -2035,11 +1724,6 @@ func testEIP161AccountRemoval(t *testing.T, scheme string) { // // https://github.com/ethereum/go-ethereum/pull/15941 func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { - testBlockchainHeaderchainReorgConsistency(t, rawdb.HashScheme) - testBlockchainHeaderchainReorgConsistency(t, rawdb.PathScheme) -} - -func testBlockchainHeaderchainReorgConsistency(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -2060,7 +1744,7 @@ func testBlockchainHeaderchainReorgConsistency(t *testing.T, scheme string) { } // Import the canonical and fork chain side by side, verifying the current block // and current header consistency - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2119,15 +1803,9 @@ func TestTrieForkGC(t *testing.T) { } } // Dereference all the recent tries and ensure no past trie is left in -<<<<<<< HEAD for i := 0; i < TestTriesInMemory; i++ { chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) -======= - for i := 0; i < TriesInMemory; i++ { - chain.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) - chain.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } if nodes, _ := chain.TrieDB().Size(); nodes > 0 { t.Fatalf("stale tries still alive after garbase collection") @@ -2137,11 +1815,6 @@ func TestTrieForkGC(t *testing.T) { // Tests that doing large reorgs works even if the state associated with the // forking point is not available any more. func TestLargeReorgTrieGC(t *testing.T) { - testLargeReorgTrieGC(t, rawdb.HashScheme) - testLargeReorgTrieGC(t, rawdb.PathScheme) -} - -func testLargeReorgTrieGC(t *testing.T, scheme string) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() genesis := &Genesis{ @@ -2153,10 +1826,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) { competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - defer db.Close() - - chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2169,7 +1839,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) { t.Fatalf("failed to insert original chain: %v", err) } // Ensure that the state associated with the forking point is pruned away - if chain.HasState(shared[len(shared)-1].Root()) { + if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil { t.Fatalf("common-but-old ancestor still cache") } // Import the competitor chain without exceeding the canonical's TD and ensure @@ -2178,7 +1848,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) { t.Fatalf("failed to insert competitor chain: %v", err) } for i, block := range competitor[:len(competitor)-2] { - if chain.HasState(block.Root()) { + if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { t.Fatalf("competitor %d: low TD chain became processed", i) } } @@ -2187,36 +1857,14 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) { if _, err := chain.InsertChain(competitor[len(competitor)-2:]); err != nil { t.Fatalf("failed to finalize competitor chain: %v", err) } -<<<<<<< HEAD for i, block := range competitor[:len(competitor)-TestTriesInMemory] { if node, err := chain.stateCache.TrieDB().Node(block.Root()); node != nil { t.Fatalf("competitor %d: competing chain state missing, err: %v", i, err) -======= - // In path-based trie database implementation, it will keep 128 diff + 1 disk - // layers, totally 129 latest states available. In hash-based it's 128. - states := TriesInMemory - if scheme == rawdb.PathScheme { - states = states + 1 - } - for i, block := range competitor[:len(competitor)-states] { - if chain.HasState(block.Root()) { - t.Fatalf("competitor %d: unexpected competing chain state", i) - } - } - for i, block := range competitor[len(competitor)-states:] { - if !chain.HasState(block.Root()) { - t.Fatalf("competitor %d: competing chain state missing", i) ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) } } } func TestBlockchainRecovery(t *testing.T) { - testBlockchainRecovery(t, rawdb.HashScheme) - testBlockchainRecovery(t, rawdb.PathScheme) -} - -func testBlockchainRecovery(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -2233,7 +1881,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -2253,7 +1901,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) { rawdb.WriteHeadFastBlockHash(ancientDb, midBlock.Hash()) // Reopen broken blockchain again - ancient, _ = NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ = NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if num := ancient.CurrentBlock().Number.Uint64(); num != 0 { t.Errorf("head block mismatch: have #%v, want #%v", num, 0) @@ -2268,13 +1916,8 @@ func testBlockchainRecovery(t *testing.T, scheme string) { // This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain. func TestInsertReceiptChainRollback(t *testing.T) { - testInsertReceiptChainRollback(t, rawdb.HashScheme) - testInsertReceiptChainRollback(t, rawdb.PathScheme) -} - -func testInsertReceiptChainRollback(t *testing.T, scheme string) { // Generate forked chain. The returned BlockChain object is used to process the side chain blocks. - tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains(scheme) + tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains() if err != nil { t.Fatal(err) } @@ -2305,7 +1948,7 @@ func testInsertReceiptChainRollback(t *testing.T, scheme string) { } defer ancientDb.Close() - ancientChain, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + ancientChain, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancientChain.Stop() // Import the canonical header chain. @@ -2350,11 +1993,6 @@ func testInsertReceiptChainRollback(t *testing.T, scheme string) { // - https://github.com/ethereum/go-ethereum/issues/18977 // - https://github.com/ethereum/go-ethereum/pull/18988 func TestLowDiffLongChain(t *testing.T) { - testLowDiffLongChain(t, rawdb.HashScheme) - testLowDiffLongChain(t, rawdb.PathScheme) -} - -func testLowDiffLongChain(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -2369,14 +2007,11 @@ func testLowDiffLongChain(t *testing.T, scheme string) { }) // Import the canonical chain - diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - defer diskdb.Close() - - chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } - defer chain.Stop() + defer chain.stopWithoutSaving() if n, err := chain.InsertChain(blocks); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", n, err) @@ -2556,20 +2191,11 @@ func TestPrunedImportSideWithMerging(t *testing.T) { testSideImport(t, 1, -10, 1) } -func TestInsertKnownHeaders(t *testing.T) { - testInsertKnownChainData(t, "headers", rawdb.HashScheme) - testInsertKnownChainData(t, "headers", rawdb.PathScheme) -} -func TestInsertKnownReceiptChain(t *testing.T) { - testInsertKnownChainData(t, "receipts", rawdb.HashScheme) - testInsertKnownChainData(t, "receipts", rawdb.PathScheme) -} -func TestInsertKnownBlocks(t *testing.T) { - testInsertKnownChainData(t, "blocks", rawdb.HashScheme) - testInsertKnownChainData(t, "blocks", rawdb.PathScheme) -} +func TestInsertKnownHeaders(t *testing.T) { testInsertKnownChainData(t, "headers") } +func TestInsertKnownReceiptChain(t *testing.T) { testInsertKnownChainData(t, "receipts") } +func TestInsertKnownBlocks(t *testing.T) { testInsertKnownChainData(t, "blocks") } -func testInsertKnownChainData(t *testing.T, typ string, scheme string) { +func testInsertKnownChainData(t *testing.T, typ string) { engine := ethash.NewFaker() genesis := &Genesis{ Config: params.TestChainConfig, @@ -2592,7 +2218,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) { } defer chaindb.Close() - chain, err := NewBlockChain(chaindb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(chaindb, nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2865,7 +2491,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i } // getLongAndShortChains returns two chains: A is longer, B is heavier. -func getLongAndShortChains(scheme string) (*BlockChain, []*types.Block, []*types.Block, *Genesis, error) { +func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, *Genesis, error) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -2877,7 +2503,7 @@ func getLongAndShortChains(scheme string) (*BlockChain, []*types.Block, []*types genDb, longChain, _ := GenerateChainWithGenesis(genesis, engine, 80, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { return nil, nil, nil, nil, fmt.Errorf("failed to create tester chain: %v", err) } @@ -2923,12 +2549,7 @@ func getLongAndShortChains(scheme string) (*BlockChain, []*types.Block, []*types // 3. Then there should be no canon mapping for the block at height X // 4. The forked block should still be retrievable by hash func TestReorgToShorterRemovesCanonMapping(t *testing.T) { - testReorgToShorterRemovesCanonMapping(t, rawdb.HashScheme) - testReorgToShorterRemovesCanonMapping(t, rawdb.PathScheme) -} - -func testReorgToShorterRemovesCanonMapping(t *testing.T, scheme string) { - chain, canonblocks, sideblocks, _, err := getLongAndShortChains(scheme) + chain, canonblocks, sideblocks, _, err := getLongAndShortChains() if err != nil { t.Fatal(err) } @@ -2966,12 +2587,7 @@ func testReorgToShorterRemovesCanonMapping(t *testing.T, scheme string) { // as TestReorgToShorterRemovesCanonMapping, but applied on headerchain // imports -- that is, for fast sync func TestReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T) { - testReorgToShorterRemovesCanonMappingHeaderChain(t, rawdb.HashScheme) - testReorgToShorterRemovesCanonMappingHeaderChain(t, rawdb.PathScheme) -} - -func testReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T, scheme string) { - chain, canonblocks, sideblocks, _, err := getLongAndShortChains(scheme) + chain, canonblocks, sideblocks, _, err := getLongAndShortChains() if err != nil { t.Fatal(err) } @@ -3120,11 +2736,6 @@ func TestTransactionIndices(t *testing.T) { } func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { - testSkipStaleTxIndicesInSnapSync(t, rawdb.HashScheme) - testSkipStaleTxIndicesInSnapSync(t, rawdb.PathScheme) -} - -func testSkipStaleTxIndicesInSnapSync(t *testing.T, scheme string) { // Configure and generate a sample block chain var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -3183,7 +2794,7 @@ func testSkipStaleTxIndicesInSnapSync(t *testing.T, scheme string) { // Import all blocks into ancient db, only HEAD-32 indices are kept. l := uint64(32) - chain, err := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3317,11 +2928,6 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { // 2. Downloader starts to sync again // 3. The blocks fetched are all known and canonical blocks func TestSideImportPrunedBlocks(t *testing.T) { - testSideImportPrunedBlocks(t, rawdb.HashScheme) - testSideImportPrunedBlocks(t, rawdb.PathScheme) -} - -func testSideImportPrunedBlocks(t *testing.T, scheme string) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() genesis := &Genesis{ @@ -3331,7 +2937,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) { // Generate and import the canonical chain _, blocks, _ := GenerateChainWithGenesis(genesis, engine, 2*TriesInMemory, nil) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3340,30 +2946,15 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) { if n, err := chain.InsertChain(blocks); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } -<<<<<<< HEAD lastPrunedIndex := len(blocks) - TestTriesInMemory - 1 lastPrunedBlock := blocks[lastPrunedIndex-1] -======= - // In path-based trie database implementation, it will keep 128 diff + 1 disk - // layers, totally 129 latest states available. In hash-based it's 128. - states := TriesInMemory - if scheme == rawdb.PathScheme { - states = TriesInMemory + 1 - } - lastPrunedIndex := len(blocks) - states - 1 - lastPrunedBlock := blocks[lastPrunedIndex] ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Verify pruning of lastPrunedBlock if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64()) } -<<<<<<< HEAD firstNonPrunedBlock := blocks[len(blocks)-TestTriesInMemory] -======= - firstNonPrunedBlock := blocks[len(blocks)-states] ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) // Verify firstNonPrunedBlock is not pruned if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) { t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64()) @@ -3385,11 +2976,6 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) { // each transaction, so this works ok. The rework accumulated writes in memory // first, but the journal wiped the entire state object on create-revert. func TestDeleteCreateRevert(t *testing.T) { - testDeleteCreateRevert(t, rawdb.HashScheme) - testDeleteCreateRevert(t, rawdb.PathScheme) -} - -func testDeleteCreateRevert(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb") @@ -3441,7 +3027,7 @@ func testDeleteCreateRevert(t *testing.T, scheme string) { b.AddTx(tx) }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3460,11 +3046,6 @@ func testDeleteCreateRevert(t *testing.T, scheme string) { // Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct, // and then the new slots exist func TestDeleteRecreateSlots(t *testing.T) { - testDeleteRecreateSlots(t, rawdb.HashScheme) - testDeleteRecreateSlots(t, rawdb.PathScheme) -} - -func testDeleteRecreateSlots(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3554,7 +3135,7 @@ func testDeleteRecreateSlots(t *testing.T, scheme string) { b.AddTx(tx) }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) if err != nil { @@ -3588,11 +3169,6 @@ func testDeleteRecreateSlots(t *testing.T, scheme string) { // regular value-transfer // Expected outcome is that _all_ slots are cleared from A func TestDeleteRecreateAccount(t *testing.T) { - testDeleteRecreateAccount(t, rawdb.HashScheme) - testDeleteRecreateAccount(t, rawdb.PathScheme) -} - -func testDeleteRecreateAccount(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3636,7 +3212,7 @@ func testDeleteRecreateAccount(t *testing.T, scheme string) { b.AddTx(tx) }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) if err != nil { @@ -3666,11 +3242,6 @@ func testDeleteRecreateAccount(t *testing.T, scheme string) { // Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct, // and then the new slots exist func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { - testDeleteRecreateSlotsAcrossManyBlocks(t, rawdb.HashScheme) - testDeleteRecreateSlotsAcrossManyBlocks(t, rawdb.PathScheme) -} - -func testDeleteRecreateSlotsAcrossManyBlocks(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3811,7 +3382,7 @@ func testDeleteRecreateSlotsAcrossManyBlocks(t *testing.T, scheme string) { current = exp }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3869,14 +3440,7 @@ func testDeleteRecreateSlotsAcrossManyBlocks(t *testing.T, scheme string) { // to the destructset in case something is created "onto" an existing item. // We need to either roll back the snapDestructs, or not place it into snapDestructs // in the first place. -// - func TestInitThenFailCreateContract(t *testing.T) { - testInitThenFailCreateContract(t, rawdb.HashScheme) - testInitThenFailCreateContract(t, rawdb.PathScheme) -} - -func testInitThenFailCreateContract(t *testing.T, scheme string) { var ( engine = ethash.NewFaker() @@ -3949,7 +3513,7 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3986,7 +3550,6 @@ func testInitThenFailCreateContract(t *testing.T, scheme string) { // access list transaction, which specifies a single slot access, and then // checking that the gas usage of a hot SLOAD and a cold SLOAD are calculated // correctly. -<<<<<<< HEAD // TestEIP2718TransitionWithTestChainConfig tests EIP-2718 with TestChainConfig. func TestEIP2718TransitionWithTestChainConfig(t *testing.T) { @@ -4000,14 +3563,6 @@ func TestEIP2718TransitionWithParliaConfig(t *testing.T) { // testEIP2718TransitionWithConfig tests EIP02718 with given ChainConfig. func testEIP2718TransitionWithConfig(t *testing.T, config *params.ChainConfig) { -======= -func TestEIP2718Transition(t *testing.T) { - testEIP2718Transition(t, rawdb.HashScheme) - testEIP2718Transition(t, rawdb.PathScheme) -} - -func testEIP2718Transition(t *testing.T, scheme string) { ->>>>>>> 503f1f7ad (all: activate pbss as experimental feature (#26274)) var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") engine = ethash.NewFaker() @@ -4055,7 +3610,7 @@ func testEIP2718Transition(t *testing.T, scheme string) { }) // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -4085,11 +3640,6 @@ func testEIP2718Transition(t *testing.T, scheme string) { // gasFeeCap - gasTipCap < baseFee. // 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). func TestEIP1559Transition(t *testing.T) { - testEIP1559Transition(t, rawdb.HashScheme) - testEIP1559Transition(t, rawdb.PathScheme) -} - -func testEIP1559Transition(t *testing.T, scheme string) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") engine = ethash.NewFaker() @@ -4100,9 +3650,8 @@ func testEIP1559Transition(t *testing.T, scheme string) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) funds = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) - config = *params.AllEthashProtocolChanges gspec = &Genesis{ - Config: &config, + Config: params.AllEthashProtocolChanges, Alloc: GenesisAlloc{ addr1: {Balance: funds}, addr2: {Balance: funds}, @@ -4149,7 +3698,7 @@ func testEIP1559Transition(t *testing.T, scheme string) { b.AddTx(tx) }) - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -4231,11 +3780,6 @@ func testEIP1559Transition(t *testing.T, scheme string) { // Tests the scenario the chain is requested to another point with the missing state. // It expects the state is recovered and all relevant chain markers are set correctly. func TestSetCanonical(t *testing.T) { - testSetCanonical(t, rawdb.HashScheme) - testSetCanonical(t, rawdb.PathScheme) -} - -func testSetCanonical(t *testing.T, scheme string) { //log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) var ( @@ -4258,10 +3802,7 @@ func testSetCanonical(t *testing.T, scheme string) { } gen.AddTx(tx) }) - diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) - defer diskdb.Close() - - chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -4318,11 +3859,6 @@ func testSetCanonical(t *testing.T, scheme string) { // TestCanonicalHashMarker tests all the canonical hash markers are updated/deleted // correctly in case reorg is called. func TestCanonicalHashMarker(t *testing.T) { - testCanonicalHashMarker(t, rawdb.HashScheme) - testCanonicalHashMarker(t, rawdb.PathScheme) -} - -func testCanonicalHashMarker(t *testing.T, scheme string) { var cases = []struct { forkA int forkB int @@ -4370,7 +3906,7 @@ func testCanonicalHashMarker(t *testing.T, scheme string) { _, forkB, _ := GenerateChainWithGenesis(gspec, engine, c.forkB, func(i int, gen *BlockGen) {}) // Initialize test chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -4726,116 +4262,6 @@ func testCreateThenDelete(t *testing.T, config *params.ChainConfig) { } } -func TestDeleteThenCreate(t *testing.T) { - var ( - engine = ethash.NewFaker() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - factoryAddr = crypto.CreateAddress(address, 0) - funds = big.NewInt(1000000000000000) - ) - /* - contract Factory { - function deploy(bytes memory code) public { - address addr; - assembly { - addr := create2(0, add(code, 0x20), mload(code), 0) - if iszero(extcodesize(addr)) { - revert(0, 0) - } - } - } - } - */ - factoryBIN := common.Hex2Bytes("608060405234801561001057600080fd5b50610241806100206000396000f3fe608060405234801561001057600080fd5b506004361061002a5760003560e01c80627743601461002f575b600080fd5b610049600480360381019061004491906100d8565b61004b565b005b6000808251602084016000f59050803b61006457600080fd5b5050565b600061007b61007684610146565b610121565b905082815260208101848484011115610097576100966101eb565b5b6100a2848285610177565b509392505050565b600082601f8301126100bf576100be6101e6565b5b81356100cf848260208601610068565b91505092915050565b6000602082840312156100ee576100ed6101f5565b5b600082013567ffffffffffffffff81111561010c5761010b6101f0565b5b610118848285016100aa565b91505092915050565b600061012b61013c565b90506101378282610186565b919050565b6000604051905090565b600067ffffffffffffffff821115610161576101606101b7565b5b61016a826101fa565b9050602081019050919050565b82818337600083830152505050565b61018f826101fa565b810181811067ffffffffffffffff821117156101ae576101ad6101b7565b5b80604052505050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600080fd5b600080fd5b600080fd5b600080fd5b6000601f19601f830116905091905056fea2646970667358221220ea8b35ed310d03b6b3deef166941140b4d9e90ea2c92f6b41eb441daf49a59c364736f6c63430008070033") - - /* - contract C { - uint256 value; - constructor() { - value = 100; - } - function destruct() public payable { - selfdestruct(payable(msg.sender)); - } - receive() payable external {} - } - */ - contractABI := common.Hex2Bytes("6080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c63430008070033") - contractAddr := crypto.CreateAddress2(factoryAddr, [32]byte{}, crypto.Keccak256(contractABI)) - - gspec := &Genesis{ - Config: params.TestChainConfig, - Alloc: GenesisAlloc{ - address: {Balance: funds}, - }, - } - nonce := uint64(0) - signer := types.HomesteadSigner{} - _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 2, func(i int, b *BlockGen) { - fee := big.NewInt(1) - if b.header.BaseFee != nil { - fee = b.header.BaseFee - } - b.SetCoinbase(common.Address{1}) - - // Block 1 - if i == 0 { - tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: nonce, - GasPrice: new(big.Int).Set(fee), - Gas: 500000, - Data: factoryBIN, - }) - nonce++ - b.AddTx(tx) - - data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") - tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: nonce, - GasPrice: new(big.Int).Set(fee), - Gas: 500000, - To: &factoryAddr, - Data: data, - }) - b.AddTx(tx) - nonce++ - } else { - // Block 2 - tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: nonce, - GasPrice: new(big.Int).Set(fee), - Gas: 500000, - To: &contractAddr, - Data: common.Hex2Bytes("2b68b9c6"), // destruct - }) - nonce++ - b.AddTx(tx) - - data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") - tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: nonce, - GasPrice: new(big.Int).Set(fee), - Gas: 500000, - To: &factoryAddr, // re-creation - Data: data, - }) - b.AddTx(tx) - nonce++ - } - }) - // Import the canonical chain - chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - for _, block := range blocks { - if _, err := chain.InsertChain([]*types.Block{block}); err != nil { - t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) - } - } -} - // TestTransientStorageReset ensures the transient storage is wiped correctly // between transactions. func TestTransientStorageReset(t *testing.T) { diff --git a/core/eip3529tests/eip3529_test_util.go b/core/eip3529tests/eip3529_test_util.go index abae9fb895..c2d63f349d 100644 --- a/core/eip3529tests/eip3529_test_util.go +++ b/core/eip3529tests/eip3529_test_util.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) func newGwei(n int64) *big.Int { @@ -42,7 +43,7 @@ func TestGasUsage(t *testing.T, config *params.ChainConfig, engine consensus.Eng }, }, } - genesis = gspec.MustCommit(db) + genesis = gspec.MustCommit(db, trie.NewDatabase(db, nil)) ) blocks, _ := core.GenerateChain(gspec.Config, genesis, engine, db, 1, func(i int, b *core.BlockGen) { @@ -61,7 +62,7 @@ func TestGasUsage(t *testing.T, config *params.ChainConfig, engine consensus.Eng // Import the canonical chain diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) + gspec.MustCommit(diskdb, trie.NewDatabase(diskdb, nil)) chain, err := core.NewBlockChain(diskdb, nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { diff --git a/core/genesis_test.go b/core/genesis_test.go index deba7e265f..9902c5a306 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -33,15 +33,6 @@ import ( "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) -func TestInvalidCliqueConfig(t *testing.T) { - block := DefaultGoerliGenesisBlock() - block.ExtraData = []byte{} - db := rawdb.NewMemoryDatabase() - if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil { - t.Fatal("Expected error on invalid clique config") - } -} - func TestSetupGenesis(t *testing.T) { testSetupGenesis(t, rawdb.HashScheme) testSetupGenesis(t, rawdb.PathScheme) @@ -102,17 +93,6 @@ func testSetupGenesis(t *testing.T, scheme string) { wantHash: customghash, wantConfig: customg.Config, }, - { - name: "custom block in DB, genesis == goerli", - fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - tdb := trie.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb) - return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock()) - }, - wantErr: &GenesisMismatchError{Stored: customghash, New: params.GoerliGenesisHash}, - wantHash: params.GoerliGenesisHash, - wantConfig: params.GoerliChainConfig, - }, { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 7b4e6e7f12..61c816dace 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -88,6 +88,9 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { infos = append(infos, info) case stateFreezerName: + if ReadStateScheme(db) != PathScheme { + continue + } datadir, err := db.AncientDatadir() if err != nil { return nil, err diff --git a/core/state/database.go b/core/state/database.go index 51f0940cc5..cc5dc73c77 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -281,6 +281,12 @@ func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Addre } func (db *cachingDB) CacheAccount(root common.Hash, t Trie) { + // only the hash scheme trie db support account cache, because the path scheme trie db + // account trie bind the previous layer, touch the dirty data when next access. This is + // related to the implementation of the Reader interface of pathdb. + if db.TrieDB().Scheme() == rawdb.PathScheme { + return + } if db.accountTrieCache == nil { return } @@ -289,6 +295,10 @@ func (db *cachingDB) CacheAccount(root common.Hash, t Trie) { } func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie) { + // ditto `CacheAccount` + if db.TrieDB().Scheme() == rawdb.PathScheme { + return + } if db.storageTrieCache == nil { return } diff --git a/core/state/statedb.go b/core/state/statedb.go index 36bf442c25..a218922b95 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1608,6 +1608,22 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc if root != types.EmptyRootHash { s.db.CacheAccount(root, s.trie) } + + origin := s.originalRoot + if origin == (common.Hash{}) { + origin = types.EmptyRootHash + } + + if root != origin { + start := time.Now() + if err := s.db.TrieDB().Update(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil { + return err + } + s.originalRoot = root + if metrics.EnabledExpensive { + s.TrieDBCommits += time.Since(start) + } + } } for _, postFunc := range postCommitFuncs { @@ -1736,20 +1752,20 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc if root == (common.Hash{}) { root = types.EmptyRootHash } - origin := s.originalRoot - if origin == (common.Hash{}) { - origin = types.EmptyRootHash - } - if root != origin { - start := time.Now() - if err := s.db.TrieDB().Update(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil { - return common.Hash{}, nil, err - } - s.originalRoot = root - if metrics.EnabledExpensive { - s.TrieDBCommits += time.Since(start) - } - } + //origin := s.originalRoot + //if origin == (common.Hash{}) { + // origin = types.EmptyRootHash + //} + //if root != origin { + // start := time.Now() + // if err := s.db.TrieDB().Update(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil { + // return common.Hash{}, nil, err + // } + // s.originalRoot = root + // if metrics.EnabledExpensive { + // s.TrieDBCommits += time.Since(start) + // } + //} // Clear all internal flags at the end of commit operation. s.accounts = make(map[common.Hash][]byte) s.storages = make(map[common.Hash]map[common.Hash][]byte) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 50d485dae3..c61f63198a 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -19,7 +19,6 @@ package state import ( "bytes" "encoding/binary" - "errors" "fmt" "math" "math/big" @@ -718,9 +717,6 @@ func TestCommitCopy(t *testing.T) { if val := copied.GetCommittedState(addr, skey); val != (common.Hash{}) { t.Fatalf("unexpected storage slot: have %x", val) } - if !errors.Is(copied.Error(), trie.ErrCommitted) { - t.Fatalf("unexpected state error, %v", copied.Error()) - } } // TestDeleteCreateRevert tests a weird state transition corner case that we hit diff --git a/core/state_prefetcher_test.go b/core/state_prefetcher_test.go index 30d90a4898..b7224a0b36 100644 --- a/core/state_prefetcher_test.go +++ b/core/state_prefetcher_test.go @@ -1,16 +1,15 @@ package core import ( - "math/big" - "testing" - "time" - "bytes" "context" "errors" "fmt" + "math/big" "runtime/pprof" "strings" + "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -20,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" "github.com/google/pprof/profile" ) @@ -37,7 +37,8 @@ func TestPrefetchLeaking(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) + triedb = trie.NewDatabase(gendb, nil) + genesis = gspec.MustCommit(gendb, triedb) signer = types.LatestSigner(gspec.Config) ) blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, 1, func(i int, block *BlockGen) { @@ -51,7 +52,7 @@ func TestPrefetchLeaking(t *testing.T) { } }) archiveDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(archiveDb) + gspec.MustCommit(archiveDb, triedb) archive, _ := NewBlockChain(archiveDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer archive.Stop() diff --git a/eth/downloader/fetchers_concurrent.go b/eth/downloader/fetchers_concurrent.go index 649aa27615..2e485d028d 100644 --- a/eth/downloader/fetchers_concurrent.go +++ b/eth/downloader/fetchers_concurrent.go @@ -92,6 +92,10 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { }() ordering := make(map[*eth.Request]int) timeouts := prque.New[int64, *eth.Request](func(data *eth.Request, index int) { + if index < 0 { + delete(ordering, data) + return + } ordering[data] = index }) @@ -245,14 +249,16 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { req.Close() if index, live := ordering[req]; live { - timeouts.Remove(index) - if index == 0 { - if !timeout.Stop() { - <-timeout.C - } - if timeouts.Size() > 0 { - _, exp := timeouts.Peek() - timeout.Reset(time.Until(time.Unix(0, -exp))) + if index >= 0 && index < timeouts.Size() { + timeouts.Remove(index) + if index == 0 { + if !timeout.Stop() { + <-timeout.C + } + if timeouts.Size() > 0 { + _, exp := timeouts.Peek() + timeout.Reset(time.Until(time.Unix(0, -exp))) + } } } delete(ordering, req) @@ -333,14 +339,16 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { // reschedule the timeout timer. index, live := ordering[res.Req] if live { - timeouts.Remove(index) - if index == 0 { - if !timeout.Stop() { - <-timeout.C - } - if timeouts.Size() > 0 { - _, exp := timeouts.Peek() - timeout.Reset(time.Until(time.Unix(0, -exp))) + if index >= 0 && index < timeouts.Size() { + timeouts.Remove(index) + if index == 0 { + if !timeout.Stop() { + <-timeout.C + } + if timeouts.Size() > 0 { + _, exp := timeouts.Peek() + timeout.Reset(time.Until(time.Unix(0, -exp))) + } } } delete(ordering, res.Req) diff --git a/eth/handler.go b/eth/handler.go index 1ff9823e2d..d081c76266 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -164,6 +164,7 @@ type handler struct { // channels for fetcher, syncer, txsyncLoop quitSync chan struct{} + stopCh chan struct{} chainSync *chainSyncer wg sync.WaitGroup @@ -198,6 +199,7 @@ func newHandler(config *handlerConfig) (*handler, error) { quitSync: make(chan struct{}), handlerDoneCh: make(chan struct{}), handlerStartCh: make(chan struct{}), + stopCh: make(chan struct{}), } if config.Sync == downloader.FullSync { // The database seems empty as the current block is the genesis. Yet the snap @@ -365,6 +367,8 @@ func (h *handler) protoTracker() { <-h.handlerDoneCh } return + case <-h.stopCh: + return } } } @@ -729,6 +733,8 @@ func (h *handler) startMaliciousVoteMonitor() { h.maliciousVoteMonitor.ConflictDetect(event.Vote, pendingBlockNumber) case <-h.voteMonitorSub.Err(): return + case <-h.stopCh: + return } } } @@ -743,7 +749,7 @@ func (h *handler) Stop() { h.voteMonitorSub.Unsubscribe() } } - + close(h.stopCh) // Quit chainSync and txsync64. // After this is done, no new peers will be accepted. close(h.quitSync) @@ -908,10 +914,18 @@ func (h *handler) BroadcastVote(vote *types.VoteEnvelope) { func (h *handler) minedBroadcastLoop() { defer h.wg.Done() - for obj := range h.minedBlockSub.Chan() { - if ev, ok := obj.Data.(core.NewMinedBlockEvent); ok { - h.BroadcastBlock(ev.Block, true) // First propagate block to peers - h.BroadcastBlock(ev.Block, false) // Only then announce to the rest + for { + select { + case obj := <-h.minedBlockSub.Chan(): + if obj == nil { + continue + } + if ev, ok := obj.Data.(core.NewMinedBlockEvent); ok { + h.BroadcastBlock(ev.Block, true) // First propagate block to peers + h.BroadcastBlock(ev.Block, false) // Only then announce to the rest + } + case <-h.stopCh: + return } } } @@ -925,6 +939,8 @@ func (h *handler) txBroadcastLoop() { h.BroadcastTransactions(event.Txs) case <-h.txsSub.Err(): return + case <-h.stopCh: + return } } } @@ -938,6 +954,8 @@ func (h *handler) txReannounceLoop() { h.ReannounceTransactions(event.Txs) case <-h.reannoTxsSub.Err(): return + case <-h.stopCh: + return } } } diff --git a/eth/protocols/trust/handler_test.go b/eth/protocols/trust/handler_test.go index 55252c0d63..144f4e602a 100644 --- a/eth/protocols/trust/handler_test.go +++ b/eth/protocols/trust/handler_test.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) var ( @@ -53,7 +54,7 @@ func newTestBackendWithGenerator(blocks int) *testBackend { BaseFee: big.NewInt(0), } copy(genspec.ExtraData[32:], testAddr[:]) - genesis := genspec.MustCommit(db) + genesis := genspec.MustCommit(db, trie.NewDatabase(db, nil)) chain, _ := core.NewBlockChain(db, nil, genspec, nil, engine, vm.Config{}, nil, nil) generator := func(i int, block *core.BlockGen) { diff --git a/eth/sync.go b/eth/sync.go index 5fb8c2be2a..8e570ca5b4 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -132,6 +132,8 @@ func (cs *chainSyncer) loop() { <-cs.doneCh } return + case <-cs.handler.stopCh: + return } } } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 3ee8aa04bc..45e0e9f413 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie" ) // Verify that Client implements the ethereum interfaces. @@ -316,7 +317,7 @@ func generateTestChain() []*types.Block { signer := types.HomesteadSigner{} // Create a database pre-initialize with a genesis block db := rawdb.NewMemoryDatabase() - genesis.MustCommit(db) + genesis.MustCommit(db, trie.NewDatabase(db, nil)) chain, _ := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil, core.EnablePersistDiff(860000)) generate := func(i int, block *core.BlockGen) { block.OffsetTime(5) @@ -352,7 +353,7 @@ func generateTestChain() []*types.Block { } } } - gblock := genesis.MustCommit(db) + gblock := genesis.MustCommit(db, trie.NewDatabase(db, nil)) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(genesis.Config, gblock, engine, db, testBlockNum, generate) blocks = append([]*types.Block{gblock}, blocks...) diff --git a/trie/database.go b/trie/database.go index 6661c7b2ec..7bad532dde 100644 --- a/trie/database.go +++ b/trie/database.go @@ -18,8 +18,10 @@ package trie import ( "errors" + "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" @@ -31,7 +33,8 @@ import ( // Config defines all necessary options for database. type Config struct { NoTries bool - Preimages bool // Flag whether the preimage of node key is recorded + Preimages bool // Flag whether the preimage of node key is recorded + Cache int HashDB *hashdb.Config // Configs for hash-based scheme PathDB *pathdb.Config // Configs for experimental path-based scheme @@ -104,8 +107,22 @@ func prepare(diskdb ethdb.Database, config *Config) *Database { // the legacy hash-based scheme is used by default. func NewDatabase(diskdb ethdb.Database, config *Config) *Database { // Sanitize the config and use the default one if it's not specified. + dbScheme := rawdb.ReadStateScheme(diskdb) if config == nil { - config = HashDefaults + if dbScheme == rawdb.PathScheme { + config = &Config{ + PathDB: pathdb.Defaults, + } + } else { + config = HashDefaults + } + } + if config.PathDB == nil && config.HashDB == nil { + if dbScheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } else { + config.HashDB = hashdb.Defaults + } } var preimages *preimageStore if config.Preimages { @@ -116,12 +133,30 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database { diskdb: diskdb, preimages: preimages, } - if config.HashDB != nil && config.PathDB != nil { - log.Crit("Both 'hash' and 'path' mode are configured") - } - if config.PathDB != nil { + /* + * 1. First, initialize db according to the user config + * 2. Second, initialize the db according to the scheme already used by db + * 3. Last, use the default scheme, namely hash scheme + */ + if config.HashDB != nil { + if rawdb.ReadStateScheme(diskdb) == rawdb.PathScheme { + log.Warn("incompatible state scheme", "old", rawdb.PathScheme, "new", rawdb.HashScheme) + } + db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) + } else if config.PathDB != nil { + if rawdb.ReadStateScheme(diskdb) == rawdb.HashScheme { + log.Warn("incompatible state scheme", "old", rawdb.HashScheme, "new", rawdb.PathScheme) + } + db.backend = pathdb.New(diskdb, config.PathDB) + } else if strings.Compare(dbScheme, rawdb.PathScheme) == 0 { + if config.PathDB == nil { + config.PathDB = pathdb.Defaults + } db.backend = pathdb.New(diskdb, config.PathDB) } else { + if config.HashDB == nil { + config.HashDB = hashdb.Defaults + } db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) } return db diff --git a/trie/trie.go b/trie/trie.go index 47bdb39548..d19cb31063 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -610,7 +610,10 @@ func (t *Trie) Hash() common.Hash { func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { defer t.tracer.reset() defer func() { - t.committed = true + // StateDB will cache the trie and reuse it to read and write, + // the committed flag is true will prevent the cache trie access + // the trie node. + t.committed = false }() // Trie is empty and can be classified into two types of situations: // (a) The trie was empty and no update happens => return nil From b8bad314edf4ba00dec29982337bca844960a028 Mon Sep 17 00:00:00 2001 From: Fynn Date: Mon, 25 Sep 2023 11:16:36 +0800 Subject: [PATCH 4/5] cmd/geth: add hash2path & trie get tools --- cmd/geth/dbcmd.go | 267 +++++++++++++++++++++++++++++++++++++++++ core/rawdb/database.go | 22 ++++ trie/hbss2pbss.go | 243 +++++++++++++++++++++++++++++++++++++ trie/node.go | 5 + 4 files changed, 537 insertions(+) create mode 100644 trie/hbss2pbss.go diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index b8fa365049..1962fd60f8 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -19,6 +19,7 @@ package main import ( "bytes" "fmt" + "math" "os" "os/signal" "path/filepath" @@ -72,6 +73,10 @@ Remove blockchain and state databases`, // no legacy stored receipts for bsc // dbMigrateFreezerCmd, dbCheckStateContentCmd, + dbHbss2PbssCmd, + dbPruneHashTrieCmd, + dbTrieGetCmd, + dbTrieDeleteCmd, }, } dbInspectCmd = &cli.Command{ @@ -94,6 +99,54 @@ Remove blockchain and state databases`, For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates a data corruption.`, } + dbHbss2PbssCmd = &cli.Command{ + Action: hbss2pbss, + Name: "hbss-to-pbss", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + }, + Usage: "Convert Hash-Base to Path-Base trie node.", + Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`, + } + dbTrieGetCmd = &cli.Command{ + Action: dbTrieGet, + Name: "trie-get", + Usage: "Show the value of a trie node path key", + ArgsUsage: "[trie owner] ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.StateSchemeFlag, + }, + Description: "This command looks up the specified trie node key from the database.", + } + dbTrieDeleteCmd = &cli.Command{ + Action: dbTrieDelete, + Name: "trie-delete", + Usage: "delete the specify trie node", + ArgsUsage: "[trie owner] | ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.StateSchemeFlag, + }, + Description: "This command delete the specify trie node from the database.", + } + dbPruneHashTrieCmd = &cli.Command{ + Action: pruneHashTrie, + Name: "prune-hash-trie", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + }, + Usage: "[Caution]Prune all the hash trie node in diskdb", + Description: `This command iterates the entrie kv in leveldb and delete all the hash trie node.`, + } dbStatCmd = &cli.Command{ Action: dbStats, Name: "stats", @@ -433,6 +486,133 @@ func dbGet(ctx *cli.Context) error { return nil } +// dbTrieGet shows the value of a given database key +func dbTrieGet(ctx *cli.Context) error { + if ctx.NArg() < 1 || ctx.NArg() > 2 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false, false) + defer db.Close() + + scheme := ctx.String(utils.StateSchemeFlag.Name) + if scheme == "" { + scheme = rawdb.HashScheme + } + + if scheme == rawdb.PathScheme { + var ( + pathKey []byte + owner []byte + err error + ) + if ctx.NArg() == 1 { + pathKey, err = hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + nodeVal, hash := rawdb.ReadAccountTrieNode(db, pathKey) + log.Info("TrieGet result ", "PathKey", common.Bytes2Hex(pathKey), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal)) + } else if ctx.NArg() == 2 { + owner, err = hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + pathKey, err = hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + + nodeVal, hash := rawdb.ReadStorageTrieNode(db, common.BytesToHash(owner), pathKey) + log.Info("TrieGet result ", "PathKey: ", common.Bytes2Hex(pathKey), "Owner: ", common.BytesToHash(owner), "Hash: ", hash, "node: ", trie.NodeString(hash.Bytes(), nodeVal)) + } + } else if scheme == rawdb.HashScheme { + if ctx.NArg() == 1 { + hashKey, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + val, err := db.Get(hashKey) + if err != nil { + log.Error("db get failed, ", "error: ", err) + return err + } + log.Info("TrieGet result ", "HashKey: ", common.BytesToHash(hashKey), "node: ", trie.NodeString(hashKey, val)) + } else { + log.Error("args too much") + } + } + + return nil +} + +// dbTrieDelete delete the trienode of a given database key +func dbTrieDelete(ctx *cli.Context) error { + if ctx.NArg() < 1 || ctx.NArg() > 2 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false, false) + defer db.Close() + + scheme := ctx.String(utils.StateSchemeFlag.Name) + if scheme == "" { + scheme = rawdb.HashScheme + } + + if scheme == rawdb.PathScheme { + var ( + pathKey []byte + owner []byte + err error + ) + if ctx.NArg() == 1 { + pathKey, err = hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + rawdb.DeleteAccountTrieNode(db, pathKey) + } else if ctx.NArg() == 2 { + owner, err = hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + pathKey, err = hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + rawdb.DeleteStorageTrieNode(db, common.BytesToHash(owner), pathKey) + } + } else if scheme == rawdb.HashScheme { + if ctx.NArg() == 1 { + hashKey, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + log.Info("Could not decode the value", "error", err) + return err + } + err = db.Delete(hashKey) + if err != nil { + log.Error("db delete failed", "err", err) + return err + } + } else { + log.Error("args too much") + } + } + return nil +} + // dbDelete deletes a key from the database func dbDelete(ctx *cli.Context) error { if ctx.NArg() != 1 { @@ -743,3 +923,90 @@ func showMetaData(ctx *cli.Context) error { table.Render() return nil } + +func hbss2pbss(ctx *cli.Context) error { + if ctx.NArg() > 1 { + return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) + } + + var jobnum uint64 + var err error + if ctx.NArg() == 1 { + jobnum, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) + if err != nil { + return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + } + } else { + // by default + jobnum = 1000 + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false, false) + db.Sync() + defer db.Close() + + config := trie.HashDefaults + triedb := trie.NewDatabase(db, config) + triedb.Cap(0) + log.Info("hbss2pbss triedb", "scheme", triedb.Scheme()) + defer triedb.Close() + + headerHash := rawdb.ReadHeadHeaderHash(db) + blockNumber := rawdb.ReadHeaderNumber(db, headerHash) + if blockNumber == nil { + log.Error("read header number failed.") + return fmt.Errorf("read header number failed") + } + + log.Info("hbss2pbss converting", "HeaderHash: ", headerHash.String(), ", blockNumber: ", *blockNumber) + + var headerBlockHash common.Hash + var trieRootHash common.Hash + + if *blockNumber != math.MaxUint64 { + headerBlockHash = rawdb.ReadCanonicalHash(db, *blockNumber) + if headerBlockHash == (common.Hash{}) { + return fmt.Errorf("ReadHeadBlockHash empty hash") + } + blockHeader := rawdb.ReadHeader(db, headerBlockHash, *blockNumber) + trieRootHash = blockHeader.Root + fmt.Println("Canonical Hash: ", headerBlockHash.String(), ", TrieRootHash: ", trieRootHash.String()) + } + if (trieRootHash == common.Hash{}) { + log.Error("Empty root hash") + return fmt.Errorf("Empty root hash.") + } + + id := trie.StateTrieID(trieRootHash) + theTrie, err := trie.New(id, triedb) + if err != nil { + log.Error("fail to new trie tree", "err", err, "rootHash", err, trieRootHash.String()) + return err + } + + h2p, err := trie.NewHbss2Pbss(theTrie, triedb, trieRootHash, *blockNumber, jobnum) + if err != nil { + log.Error("fail to new hash2pbss", "err", err, "rootHash", err, trieRootHash.String()) + return err + } + h2p.Run() + + return nil +} + +func pruneHashTrie(ctx *cli.Context) error { + if ctx.NArg() != 0 { + return fmt.Errorf("required none argument") + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false, false) + defer db.Close() + + return rawdb.PruneHashTrieNodeInDataBase(db) +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 7188efa6f3..12384a66fa 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -581,6 +581,28 @@ func AncientInspect(db ethdb.Database) error { return nil } +func PruneHashTrieNodeInDataBase(db ethdb.Database) error { + it := db.NewIterator([]byte{}, []byte{}) + defer it.Release() + + total_num := 0 + for it.Next() { + var key = it.Key() + switch { + case IsLegacyTrieNode(key, it.Value()): + db.Delete(key) + total_num++ + if total_num%100000 == 0 { + log.Info("Pruning ", "Complete progress: ", total_num, "hash-base trie nodes") + } + default: + continue + } + } + log.Info("Pruning ", "Complete progress", total_num, "hash-base trie nodes") + return nil +} + // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go new file mode 100644 index 0000000000..40845bdc1e --- /dev/null +++ b/trie/hbss2pbss.go @@ -0,0 +1,243 @@ +package trie + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie/trienode" +) + +type Hbss2Pbss struct { + trie *Trie // traverse trie + db *Database + blocknum uint64 + root node // root of triedb + stateRootHash common.Hash + concurrentQueue chan struct{} + totalNum uint64 + wg sync.WaitGroup +} + +const ( + DEFAULT_TRIEDBCACHE_SIZE = 1024 * 1024 * 1024 +) + +// NewHbss2Pbss return a hash2Path obj +func NewHbss2Pbss(tr *Trie, db *Database, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Hbss2Pbss, error) { + if tr == nil { + return nil, errors.New("trie is nil") + } + + if tr.root == nil { + return nil, errors.New("trie root is nil") + } + + ins := &Hbss2Pbss{ + trie: tr, + blocknum: blocknum, + db: db, + stateRootHash: stateRootHash, + root: tr.root, + concurrentQueue: make(chan struct{}, jobnum), + wg: sync.WaitGroup{}, + } + + return ins, nil +} + +func (t *Trie) resloveWithoutTrack(n node, prefix []byte) (node, error) { + if n, ok := n.(hashNode); ok { + blob, err := t.reader.node(prefix, common.BytesToHash(n)) + if err != nil { + return nil, err + } + return mustDecodeNode(n, blob), nil + } + return n, nil +} + +func (h2p *Hbss2Pbss) writeNode(pathKey []byte, n *trienode.Node, owner common.Hash) { + if owner == (common.Hash{}) { + rawdb.WriteAccountTrieNode(h2p.db.diskdb, pathKey, n.Blob) + log.Debug("WriteNodes account node, ", "path: ", common.Bytes2Hex(pathKey), "Hash: ", n.Hash, "BlobHash: ", crypto.Keccak256Hash(n.Blob)) + } else { + rawdb.WriteStorageTrieNode(h2p.db.diskdb, owner, pathKey, n.Blob) + log.Debug("WriteNodes storage node, ", "path: ", common.Bytes2Hex(pathKey), "owner: ", owner.String(), "Hash: ", n.Hash, "BlobHash: ", crypto.Keccak256Hash(n.Blob)) + } +} + +// Run statistics, external call +func (h2p *Hbss2Pbss) Run() { + log.Debug("Find Account Trie Tree, rootHash: ", h2p.trie.Hash().String(), "BlockNum: ", h2p.blocknum) + + h2p.ConcurrentTraversal(h2p.trie, h2p.root, []byte{}) + h2p.wg.Wait() + + log.Info("Total complete: %v, go routines Num: %v, h2p concurrentQueue: %v\n", h2p.totalNum, runtime.NumGoroutine(), len(h2p.concurrentQueue)) + + rawdb.WritePersistentStateID(h2p.db.diskdb, h2p.blocknum) + rawdb.WriteStateID(h2p.db.diskdb, h2p.stateRootHash, h2p.blocknum) +} + +func (h2p *Hbss2Pbss) SubConcurrentTraversal(theTrie *Trie, theNode node, path []byte) { + h2p.concurrentQueue <- struct{}{} + h2p.ConcurrentTraversal(theTrie, theNode, path) + <-h2p.concurrentQueue + h2p.wg.Done() +} + +func (h2p *Hbss2Pbss) ConcurrentTraversal(theTrie *Trie, theNode node, path []byte) { + total_num := uint64(0) + // nil node + if theNode == nil { + return + } + + switch current := (theNode).(type) { + case *shortNode: + collapsed := current.copy() + collapsed.Key = hexToCompact(current.Key) + var hash, _ = current.cache() + h2p.writeNode(path, trienode.New(common.BytesToHash(hash), nodeToBytes(collapsed)), theTrie.owner) + + h2p.ConcurrentTraversal(theTrie, current.Val, append(path, current.Key...)) + + case *fullNode: + // copy from trie/Committer (*committer).commit + collapsed := current.copy() + var hash, _ = collapsed.cache() + collapsed.Children = h2p.commitChildren(path, current) + + nodebytes := nodeToBytes(collapsed) + if common.BytesToHash(hash) != common.BytesToHash(crypto.Keccak256(nodebytes)) { + log.Error("Hash is inconsistent, hash: ", common.BytesToHash(hash), "node hash: ", common.BytesToHash(crypto.Keccak256(nodebytes)), "node: ", collapsed.fstring("")) + panic("hash inconsistent.") + } + + h2p.writeNode(path, trienode.New(common.BytesToHash(hash), nodeToBytes(collapsed)), theTrie.owner) + + for idx, child := range current.Children { + if child == nil { + continue + } + childPath := append(path, byte(idx)) + if len(h2p.concurrentQueue)*2 < cap(h2p.concurrentQueue) { + h2p.wg.Add(1) + dst := make([]byte, len(childPath)) + copy(dst, childPath) + go h2p.SubConcurrentTraversal(theTrie, child, dst) + } else { + h2p.ConcurrentTraversal(theTrie, child, childPath) + } + } + case hashNode: + n, err := theTrie.resloveWithoutTrack(current, path) + if err != nil { + log.Error("Resolve HashNode", "error", err, "TrieRoot", theTrie.Hash(), "Path", path) + return + } + h2p.ConcurrentTraversal(theTrie, n, path) + total_num = atomic.AddUint64(&h2p.totalNum, 1) + if total_num%100000 == 0 { + log.Info("Converting ", "Complete progress", total_num, "go routines Num", runtime.NumGoroutine(), "h2p concurrentQueue", len(h2p.concurrentQueue)) + } + return + case valueNode: + if !hasTerm(path) { + log.Info("ValueNode miss path term", "path", common.Bytes2Hex(path)) + break + } + var account types.StateAccount + if err := rlp.Decode(bytes.NewReader(current), &account); err != nil { + // log.Info("Rlp decode account failed.", "err", err) + break + } + if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + // log.Info("Not a storage trie.", "account", common.BytesToHash(path).String()) + break + } + + ownerAddress := common.BytesToHash(hexToCompact(path)) + tr, err := New(StorageTrieID(h2p.stateRootHash, ownerAddress, account.Root), h2p.db) + if err != nil { + log.Error("New Storage trie error", "err", err, "root", account.Root.String(), "owner", ownerAddress.String()) + break + } + log.Debug("Find Contract Trie Tree", "rootHash: ", tr.Hash().String(), "") + h2p.wg.Add(1) + go h2p.SubConcurrentTraversal(tr, tr.root, []byte{}) + default: + panic(errors.New("Invalid node type to traverse.")) + } +} + +// copy from trie/Commiter (*committer).commit +func (h2p *Hbss2Pbss) commitChildren(path []byte, n *fullNode) [17]node { + var children [17]node + for i := 0; i < 16; i++ { + child := n.Children[i] + if child == nil { + continue + } + // If it's the hashed child, save the hash value directly. + // Note: it's impossible that the child in range [0, 15] + // is a valueNode. + if hn, ok := child.(hashNode); ok { + children[i] = hn + continue + } + + children[i] = h2p.commit(append(path, byte(i)), child) + } + // For the 17th child, it's possible the type is valuenode. + if n.Children[16] != nil { + children[16] = n.Children[16] + } + return children +} + +// commit collapses a node down into a hash node and returns it. +func (h2p *Hbss2Pbss) commit(path []byte, n node) node { + // if this path is clean, use available cached data + hash, dirty := n.cache() + if hash != nil && !dirty { + return hash + } + // Commit children, then parent, and remove the dirty flag. + switch cn := n.(type) { + case *shortNode: + // Commit child + collapsed := cn.copy() + + // If the child is fullNode, recursively commit, + // otherwise it can only be hashNode or valueNode. + if _, ok := cn.Val.(*fullNode); ok { + collapsed.Val = h2p.commit(append(path, cn.Key...), cn.Val) + } + // The key needs to be copied, since we're adding it to the + // modified nodeset. + collapsed.Key = hexToCompact(cn.Key) + return collapsed + case *fullNode: + hashedKids := h2p.commitChildren(path, cn) + collapsed := cn.copy() + collapsed.Children = hashedKids + + return collapsed + case hashNode: + return cn + default: + // nil, valuenode shouldn't be committed + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } +} diff --git a/trie/node.go b/trie/node.go index 15bbf62f1c..d78ed5c569 100644 --- a/trie/node.go +++ b/trie/node.go @@ -112,6 +112,11 @@ func (n rawNode) EncodeRLP(w io.Writer) error { return err } +func NodeString(hash, buf []byte) string { + node := mustDecodeNode(hash, buf) + return node.fstring("NodeString: ") +} + // mustDecodeNode is a wrapper of decodeNode and panic if any error is encountered. func mustDecodeNode(hash, buf []byte) node { n, err := decodeNode(hash, buf) From 4259f4c1f878c22970fb4a725cbb0ab2c251fe58 Mon Sep 17 00:00:00 2001 From: Fynn Date: Wed, 27 Sep 2023 15:01:49 +0800 Subject: [PATCH 5/5] fix: state history hasn't write --- cmd/geth/dbcmd.go | 119 +++++++++++++++++----------------- cmd/utils/flags.go | 7 +- core/rawdb/accessors_state.go | 30 +++++++-- core/rawdb/ancient_utils.go | 23 +++++++ core/rawdb/database.go | 4 +- core/rawdb/freezer_table.go | 30 +++++++++ trie/hbss2pbss.go | 2 +- 7 files changed, 147 insertions(+), 68 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 1962fd60f8..eb6185fc2f 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -74,7 +74,6 @@ Remove blockchain and state databases`, // dbMigrateFreezerCmd, dbCheckStateContentCmd, dbHbss2PbssCmd, - dbPruneHashTrieCmd, dbTrieGetCmd, dbTrieDeleteCmd, }, @@ -106,6 +105,8 @@ a data corruption.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.SyncModeFlag, + utils.ForceFlag, + utils.AncientFlag, }, Usage: "Convert Hash-Base to Path-Base trie node.", Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`, @@ -136,17 +137,6 @@ a data corruption.`, }, Description: "This command delete the specify trie node from the database.", } - dbPruneHashTrieCmd = &cli.Command{ - Action: pruneHashTrie, - Name: "prune-hash-trie", - ArgsUsage: "", - Flags: []cli.Flag{ - utils.DataDirFlag, - utils.SyncModeFlag, - }, - Usage: "[Caution]Prune all the hash trie node in diskdb", - Description: `This command iterates the entrie kv in leveldb and delete all the hash trie node.`, - } dbStatCmd = &cli.Command{ Action: dbStats, Name: "stats", @@ -941,6 +931,8 @@ func hbss2pbss(ctx *cli.Context) error { jobnum = 1000 } + force := ctx.Bool(utils.ForceFlag.Name) + stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -948,65 +940,74 @@ func hbss2pbss(ctx *cli.Context) error { db.Sync() defer db.Close() - config := trie.HashDefaults - triedb := trie.NewDatabase(db, config) - triedb.Cap(0) - log.Info("hbss2pbss triedb", "scheme", triedb.Scheme()) - defer triedb.Close() + // convert hbss trie node to pbss trie node + lastStateID := rawdb.ReadPersistentStateID(db) + if lastStateID == 0 || force { + config := trie.HashDefaults + triedb := trie.NewDatabase(db, config) + triedb.Cap(0) + log.Info("hbss2pbss triedb", "scheme", triedb.Scheme()) + defer triedb.Close() + + headerHash := rawdb.ReadHeadHeaderHash(db) + blockNumber := rawdb.ReadHeaderNumber(db, headerHash) + if blockNumber == nil { + log.Error("read header number failed.") + return fmt.Errorf("read header number failed") + } - headerHash := rawdb.ReadHeadHeaderHash(db) - blockNumber := rawdb.ReadHeaderNumber(db, headerHash) - if blockNumber == nil { - log.Error("read header number failed.") - return fmt.Errorf("read header number failed") - } + log.Info("hbss2pbss converting", "HeaderHash: ", headerHash.String(), ", blockNumber: ", *blockNumber) - log.Info("hbss2pbss converting", "HeaderHash: ", headerHash.String(), ", blockNumber: ", *blockNumber) + var headerBlockHash common.Hash + var trieRootHash common.Hash - var headerBlockHash common.Hash - var trieRootHash common.Hash + if *blockNumber != math.MaxUint64 { + headerBlockHash = rawdb.ReadCanonicalHash(db, *blockNumber) + if headerBlockHash == (common.Hash{}) { + return fmt.Errorf("ReadHeadBlockHash empty hash") + } + blockHeader := rawdb.ReadHeader(db, headerBlockHash, *blockNumber) + trieRootHash = blockHeader.Root + fmt.Println("Canonical Hash: ", headerBlockHash.String(), ", TrieRootHash: ", trieRootHash.String()) + } + if (trieRootHash == common.Hash{}) { + log.Error("Empty root hash") + return fmt.Errorf("Empty root hash.") + } - if *blockNumber != math.MaxUint64 { - headerBlockHash = rawdb.ReadCanonicalHash(db, *blockNumber) - if headerBlockHash == (common.Hash{}) { - return fmt.Errorf("ReadHeadBlockHash empty hash") + id := trie.StateTrieID(trieRootHash) + theTrie, err := trie.New(id, triedb) + if err != nil { + log.Error("fail to new trie tree", "err", err, "rootHash", err, trieRootHash.String()) + return err } - blockHeader := rawdb.ReadHeader(db, headerBlockHash, *blockNumber) - trieRootHash = blockHeader.Root - fmt.Println("Canonical Hash: ", headerBlockHash.String(), ", TrieRootHash: ", trieRootHash.String()) - } - if (trieRootHash == common.Hash{}) { - log.Error("Empty root hash") - return fmt.Errorf("Empty root hash.") + + h2p, err := trie.NewHbss2Pbss(theTrie, triedb, trieRootHash, *blockNumber, jobnum) + if err != nil { + log.Error("fail to new hash2pbss", "err", err, "rootHash", err, trieRootHash.String()) + return err + } + h2p.Run() + } else { + log.Info("Convert hbss to pbss success. Nothing to do.") } - id := trie.StateTrieID(trieRootHash) - theTrie, err := trie.New(id, triedb) + // repair state ancient offset + lastStateID = rawdb.ReadPersistentStateID(db) + if lastStateID == 0 { + log.Error("Convert hbss to pbss trie node error. The last state id is still 0") + } + ancient := stack.ResolveAncient("chaindata", ctx.String(utils.AncientFlag.Name)) + err = rawdb.ResetStateFreezerTableOffset(ancient, lastStateID) if err != nil { - log.Error("fail to new trie tree", "err", err, "rootHash", err, trieRootHash.String()) + log.Error("Reset state freezer table offset failed", "error", err) return err } - - h2p, err := trie.NewHbss2Pbss(theTrie, triedb, trieRootHash, *blockNumber, jobnum) + // prune hbss trie node + err = rawdb.PruneHashTrieNodeInDataBase(db) if err != nil { - log.Error("fail to new hash2pbss", "err", err, "rootHash", err, trieRootHash.String()) + log.Error("Prune Hash trie node in database failed", "error", err) return err } - h2p.Run() - return nil } - -func pruneHashTrie(ctx *cli.Context) error { - if ctx.NArg() != 0 { - return fmt.Errorf("required none argument") - } - - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - db := utils.MakeChainDatabase(ctx, stack, false, false) - defer db.Close() - - return rawdb.PruneHashTrieNodeInDataBase(db) -} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8bff3d7fa9..2f23414ee8 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -207,7 +207,12 @@ var ( Usage: "Exits after block synchronisation completes", Category: flags.EthCategory, } - + // hbss2pbss command options + ForceFlag = &cli.BoolFlag{ + Name: "force", + Usage: "Force convert hbss trie node to pbss trie node. Ingore any metadata", + Value: false, + } // Dump command options. IterativeOutputFlag = &cli.BoolFlag{ Name: "iterative", diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 9ce58e7d27..da06596583 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -256,11 +256,31 @@ func ReadStateHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []by // history starts from one(zero for initial state). func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIndex []byte, storageIndex []byte, accounts []byte, storages []byte) { db.ModifyAncients(func(op ethdb.AncientWriteOp) error { - op.AppendRaw(stateHistoryMeta, id-1, meta) - op.AppendRaw(stateHistoryAccountIndex, id-1, accountIndex) - op.AppendRaw(stateHistoryStorageIndex, id-1, storageIndex) - op.AppendRaw(stateHistoryAccountData, id-1, accounts) - op.AppendRaw(stateHistoryStorageData, id-1, storages) + err := op.AppendRaw(stateHistoryMeta, id-1, meta) + if err != nil { + log.Error("WriteStateHistory failed", "err", err) + return err + } + err = op.AppendRaw(stateHistoryAccountIndex, id-1, accountIndex) + if err != nil { + log.Error("WriteStateHistory failed", "err", err) + return err + } + err = op.AppendRaw(stateHistoryStorageIndex, id-1, storageIndex) + if err != nil { + log.Error("WriteStateHistory failed", "err", err) + return err + } + err = op.AppendRaw(stateHistoryAccountData, id-1, accounts) + if err != nil { + log.Error("WriteStateHistory failed", "err", err) + return err + } + err = op.AppendRaw(stateHistoryStorageData, id-1, storages) + if err != nil { + log.Error("WriteStateHistory failed", "err", err) + return err + } return nil }) } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 61c816dace..392ac79631 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -18,9 +18,12 @@ package rawdb import ( "fmt" + "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" ) type tableSize struct { @@ -144,3 +147,23 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s table.dumpIndexStdout(start, end) return nil } + +func ResetStateFreezerTableOffset(ancient string, virtualTail uint64) error { + path, tables := filepath.Join(ancient, stateFreezerName), stateFreezerNoSnappy + + for name, disableSnappy := range tables { + log.Info("Handle table", "name", name, "disableSnappy", disableSnappy) + table, err := newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, false) + if err != nil { + log.Error("New table failed", "error", err) + return err + } + // Reset the metadata of the freezer table + err = table.ResetItemsOffset(virtualTail) + if err != nil { + log.Error("Reset items offset of the table", "name", name, "error", err) + return err + } + } + return nil +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 12384a66fa..40a034923d 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -593,13 +593,13 @@ func PruneHashTrieNodeInDataBase(db ethdb.Database) error { db.Delete(key) total_num++ if total_num%100000 == 0 { - log.Info("Pruning ", "Complete progress: ", total_num, "hash-base trie nodes") + log.Info("Pruning hash-base state trie nodes", "Complete progress: ", total_num) } default: continue } } - log.Info("Pruning ", "Complete progress", total_num, "hash-base trie nodes") + log.Info("Pruning hash-base state trie nodes", "Complete progress", total_num) return nil } diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 369fd4b694..76293a2da2 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -949,3 +949,33 @@ func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { } fmt.Fprintf(w, "|--------------------------|\n") } + +func (t *freezerTable) ResetItemsOffset(virtualTail uint64) error { + stat, err := t.index.Stat() + if err != nil { + return err + } + + if stat.Size() == 0 { + return fmt.Errorf("Stat size is zero when ResetVirtualTail.") + } + + var firstIndex indexEntry + + buffer := make([]byte, indexEntrySize) + + t.index.ReadAt(buffer, 0) + firstIndex.unmarshalBinary(buffer) + + firstIndex.offset = uint32(virtualTail) + t.index.WriteAt(firstIndex.append(nil), 0) + + var firstIndex2 indexEntry + buffer2 := make([]byte, indexEntrySize) + t.index.ReadAt(buffer2, 0) + firstIndex2.unmarshalBinary(buffer2) + + log.Info("Reset Index", "filenum", t.index.Name(), "offset", firstIndex2.offset) + + return nil +} diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go index 40845bdc1e..1faaf34783 100644 --- a/trie/hbss2pbss.go +++ b/trie/hbss2pbss.go @@ -83,7 +83,7 @@ func (h2p *Hbss2Pbss) Run() { h2p.ConcurrentTraversal(h2p.trie, h2p.root, []byte{}) h2p.wg.Wait() - log.Info("Total complete: %v, go routines Num: %v, h2p concurrentQueue: %v\n", h2p.totalNum, runtime.NumGoroutine(), len(h2p.concurrentQueue)) + log.Info("Total", "complete", h2p.totalNum, "go routines Num", runtime.NumGoroutine, "h2p concurrentQueue", len(h2p.concurrentQueue)) rawdb.WritePersistentStateID(h2p.db.diskdb, h2p.blocknum) rawdb.WriteStateID(h2p.db.diskdb, h2p.stateRootHash, h2p.blocknum)