feat: support separate trie database (bnb-chain#2021)

* feat: support separate database for state data

cmd/geth/chaincmd.go

@@ -237,6 +237,15 @@ func initGenesis(ctx *cli.Context) error {
 		}
 		defer chaindb.Close()
 
+		// if the trie data dir has been set, new trie db with a new state database
+		if ctx.IsSet(utils.SeparateDBFlag.Name) {
+			statediskdb, dbErr := stack.OpenDatabaseWithFreezer(name+"/state", 0, 0, "", "", false, false, false, false)
+			if dbErr != nil {
+				utils.Fatalf("Failed to open separate trie database: %v", dbErr)
+			}
+			chaindb.SetStateStore(statediskdb)
+		}
+
 		triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
 		defer triedb.Close()
 
@@ -600,7 +609,6 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 	}
 
 	db := utils.MakeChainDatabase(ctx, stack, true, false)
-	defer db.Close()
 	scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db)
 	if err != nil {
 		return nil, nil, common.Hash{}, err
@@ -609,7 +617,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 		fmt.Println("You are using geth dump in path mode, please use `geth dump-roothash` command to get all available blocks.")
 	}
 
-	var header *types.Header
+	header := &types.Header{}
 	if ctx.NArg() == 1 {
 		arg := ctx.Args().First()
 		if hashish(arg) {

cmd/geth/config.go

@@ -203,6 +203,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		v := ctx.Uint64(utils.OverrideFeynman.Name)
 		cfg.Eth.OverrideFeynman = &v
 	}
+	if ctx.IsSet(utils.SeparateDBFlag.Name) && !stack.IsSeparatedDB() {
+		utils.Fatalf("Failed to locate separate database subdirectory when separatedb parameter has been set")
+	}
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 
 	// Create gauge with geth system and build information

cmd/geth/dbcmd.go

@@ -381,7 +381,6 @@ func inspectTrie(ctx *cli.Context) error {
 
 	db := utils.MakeChainDatabase(ctx, stack, true, false)
 	defer db.Close()
-
 	var headerBlockHash common.Hash
 	if ctx.NArg() >= 1 {
 		if ctx.Args().Get(0) == "latest" {
@@ -562,6 +561,11 @@ func dbStats(ctx *cli.Context) error {
 	defer db.Close()
 
 	showLeveldbStats(db)
+	if db.StateStore() != nil {
+		fmt.Println("show stats of state store")
+		showLeveldbStats(db.StateStore())
+	}
+
 	return nil
 }
 
@@ -575,13 +579,31 @@ func dbCompact(ctx *cli.Context) error {
 	log.Info("Stats before compaction")
 	showLeveldbStats(db)
 
+	statediskdb := db.StateStore()
+	if statediskdb != nil {
+		fmt.Println("show stats of state store")
+		showLeveldbStats(statediskdb)
+	}
+
 	log.Info("Triggering compaction")
 	if err := db.Compact(nil, nil); err != nil {
-		log.Info("Compact err", "error", err)
+		log.Error("Compact err", "error", err)
 		return err
 	}
+
+	if statediskdb != nil {
+		if err := statediskdb.Compact(nil, nil); err != nil {
+			log.Error("Compact err", "error", err)
+			return err
+		}
+	}
+
 	log.Info("Stats after compaction")
 	showLeveldbStats(db)
+	if statediskdb != nil {
+		fmt.Println("show stats of state store after compaction")
+		showLeveldbStats(statediskdb)
+	}
 	return nil
 }
 
@@ -602,8 +624,17 @@ func dbGet(ctx *cli.Context) error {
 		return err
 	}
 
+	statediskdb := db.StateStore()
 	data, err := db.Get(key)
 	if err != nil {
+		// if separate trie db exist, try to get it from separate db
+		if statediskdb != nil {
+			statedata, dberr := statediskdb.Get(key)
+			if dberr == nil {
+				fmt.Printf("key %#x: %#x\n", key, statedata)
+				return nil
+			}
+		}
 		log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err)
 		return err
 	}
@@ -619,8 +650,14 @@ func dbTrieGet(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	db := utils.MakeChainDatabase(ctx, stack, false, false)
-	defer db.Close()
+	var db ethdb.Database
+	chaindb := utils.MakeChainDatabase(ctx, stack, true, false)
+	if chaindb.StateStore() != nil {
+		db = chaindb.StateStore()
+	} else {
+		db = chaindb
+	}
+	defer chaindb.Close()
 
 	scheme := ctx.String(utils.StateSchemeFlag.Name)
 	if scheme == "" {
@@ -685,8 +722,14 @@ func dbTrieDelete(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	db := utils.MakeChainDatabase(ctx, stack, false, false)
-	defer db.Close()
+	var db ethdb.Database
+	chaindb := utils.MakeChainDatabase(ctx, stack, true, false)
+	if chaindb.StateStore() != nil {
+		db = chaindb.StateStore()
+	} else {
+		db = chaindb
+	}
+	defer chaindb.Close()
 
 	scheme := ctx.String(utils.StateSchemeFlag.Name)
 	if scheme == "" {
@@ -1076,10 +1119,16 @@ func hbss2pbss(ctx *cli.Context) error {
 
 	db := utils.MakeChainDatabase(ctx, stack, false, false)
 	db.Sync()
+	stateDiskDb := db.StateStore()
 	defer db.Close()
 
 	// convert hbss trie node to pbss trie node
-	lastStateID := rawdb.ReadPersistentStateID(db)
+	var lastStateID uint64
+	if stateDiskDb != nil {
+		lastStateID = rawdb.ReadPersistentStateID(stateDiskDb)
+	} else {
+		lastStateID = rawdb.ReadPersistentStateID(db)
+	}
 	if lastStateID == 0 || force {
 		config := trie.HashDefaults
 		triedb := trie.NewDatabase(db, config)
@@ -1131,18 +1180,34 @@ func hbss2pbss(ctx *cli.Context) error {
 	}
 
 	// repair state ancient offset
-	lastStateID = rawdb.ReadPersistentStateID(db)
+	if stateDiskDb != nil {
+		lastStateID = rawdb.ReadPersistentStateID(stateDiskDb)
+	} else {
+		lastStateID = rawdb.ReadPersistentStateID(db)
+	}
+
 	if lastStateID == 0 {
 		log.Error("Convert hbss to pbss trie node error. The last state id is still 0")
 	}
-	ancient := stack.ResolveAncient("chaindata", ctx.String(utils.AncientFlag.Name))
+
+	var ancient string
+	if db.StateStore() != nil {
+		dirName := filepath.Join(stack.ResolvePath("chaindata"), "state")
+		ancient = filepath.Join(dirName, "ancient")
+	} else {
+		ancient = stack.ResolveAncient("chaindata", ctx.String(utils.AncientFlag.Name))
+	}
 	err = rawdb.ResetStateFreezerTableOffset(ancient, lastStateID)
 	if err != nil {
 		log.Error("Reset state freezer table offset failed", "error", err)
 		return err
 	}
 	// prune hbss trie node
-	err = rawdb.PruneHashTrieNodeInDataBase(db)
+	if stateDiskDb != nil {
+		err = rawdb.PruneHashTrieNodeInDataBase(stateDiskDb)
+	} else {
+		err = rawdb.PruneHashTrieNodeInDataBase(db)
+	}
 	if err != nil {
 		log.Error("Prune Hash trie node in database failed", "error", err)
 		return err

cmd/geth/snapshot.go

@@ -436,13 +436,15 @@ func pruneState(ctx *cli.Context) error {
 	chaindb := utils.MakeChainDatabase(ctx, stack, false, false)
 	defer chaindb.Close()
 
-	if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
-		log.Crit("Offline pruning is not required for path scheme")
-	}
 	prunerconfig := pruner.Config{
 		Datadir:   stack.ResolvePath(""),
 		BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name),
 	}
+
+	if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
+		log.Crit("Offline pruning is not required for path scheme")
+	}
+
 	pruner, err := pruner.NewPruner(chaindb, prunerconfig, ctx.Uint64(utils.TriesInMemoryFlag.Name))
 	if err != nil {
 		log.Error("Failed to open snapshot tree", "err", err)

cmd/utils/flags.go

@@ -93,6 +93,12 @@ var (
 		Value:    flags.DirectoryString(node.DefaultDataDir()),
 		Category: flags.EthCategory,
 	}
+	SeparateDBFlag = &cli.BoolFlag{
+		Name: "separatedb",
+		Usage: "Enable a separated trie database, it will be created within a subdirectory called state, " +
+			"Users can copy this state directory to another directory or disk, and then create a symbolic link to the state directory under the chaindata",
+		Category: flags.EthCategory,
+	}
 	DirectBroadcastFlag = &cli.BoolFlag{
 		Name:     "directbroadcast",
 		Usage:    "Enable directly broadcast mined block to all peers",
@@ -1112,6 +1118,7 @@ var (
 		DBEngineFlag,
 		StateSchemeFlag,
 		HttpHeaderFlag,
+		SeparateDBFlag,
 	}
 )
 
@@ -2314,13 +2321,29 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFree
 		chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly)
 	default:
 		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly, disableFreeze, false, false)
+		// set the separate state database
+		if stack.IsSeparatedDB() && err == nil {
+			stateDiskDb := MakeStateDataBase(ctx, stack, readonly, false)
+			chainDb.SetStateStore(stateDiskDb)
+		}
 	}
 	if err != nil {
 		Fatalf("Could not open database: %v", err)
 	}
 	return chainDb
 }
 
+// MakeStateDataBase open a separate state database using the flags passed to the client and will hard crash if it fails.
+func MakeStateDataBase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database {
+	cache := ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100
+	handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) / 2
+	statediskdb, err := stack.OpenDatabaseWithFreezer("chaindata/state", cache, handles, "", "", readonly, disableFreeze, false, false)
+	if err != nil {
+		Fatalf("Failed to open separate trie database: %v", err)
+	}
+	return statediskdb
+}
+
 // tryMakeReadOnlyDatabase try to open the chain database in read-only mode,
 // or fallback to write mode if the database is not initialized.
 //

core/rawdb/accessors_trie.go

@@ -288,13 +288,13 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
 // if the state is not present in database.
 func ReadStateScheme(db ethdb.Reader) string {
 	// Check if state in path-based scheme is present
-	blob, _ := ReadAccountTrieNode(db, nil)
+	blob, _ := ReadAccountTrieNode(db.StateStoreReader(), nil)
 	if len(blob) != 0 {
 		return PathScheme
 	}
 	// The root node might be deleted during the initial snap sync, check
 	// the persistent state id then.
-	if id := ReadPersistentStateID(db); id != 0 {
+	if id := ReadPersistentStateID(db.StateStoreReader()); id != 0 {
 		return PathScheme
 	}
 	// In a hash-based scheme, the genesis state is consistently stored
@@ -304,7 +304,7 @@ func ReadStateScheme(db ethdb.Reader) string {
 	if header == nil {
 		return "" // empty datadir
 	}
-	blob = ReadLegacyTrieNode(db, header.Root)
+	blob = ReadLegacyTrieNode(db.StateStoreReader(), header.Root)
 	if len(blob) == 0 {
 		return "" // no state in disk
 	}

core/rawdb/ancient_utils.go

@@ -91,7 +91,7 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
 			infos = append(infos, info)
 
 		case StateFreezerName:
-			if ReadStateScheme(db) != PathScheme {
+			if ReadStateScheme(db) != PathScheme || db.StateStore() != nil {
 				continue
 			}
 			datadir, err := db.AncientDatadir()