diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d001587dd0..da15df096f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -84,7 +84,7 @@ jobs: go-version: 1.15 - name: test & coverage report creation run: | - cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic + CGO_ENABLED=0 cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic if: env.GIT_DIFF - uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6495a36d46..0a4a2f6367 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: - uses: golangci/golangci-lint-action@v2.5.2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.31 + version: v1.38 args: --timeout 10m github-token: ${{ secrets.github_token }} if: env.GIT_DIFF diff --git a/.golangci.yml b/.golangci.yml index 6840a3307e..f6a41a95c6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,7 +26,7 @@ linters: # - maligned - nakedret - prealloc - - scopelint + - exportloopref - staticcheck - structcheck - stylecheck diff --git a/abci/example/kvstore/kvstore.go b/abci/example/kvstore/kvstore.go index 80f11f6e6f..2189421bdf 100644 --- a/abci/example/kvstore/kvstore.go +++ b/abci/example/kvstore/kvstore.go @@ -6,10 +6,10 @@ import ( "encoding/json" "fmt" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/abci/example/code" "github.com/lazyledger/lazyledger-core/abci/types" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + memdb "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/version" ) @@ -71,7 +71,7 @@ type Application struct { } func NewApplication() *Application { - state := loadState(dbm.NewMemDB()) + state := loadState(memdb.NewDB()) return &Application{state: state} } diff --git a/abci/example/kvstore/persistent_kvstore.go b/abci/example/kvstore/persistent_kvstore.go index 0a464aaddf..c6dfcb596d 100644 --- a/abci/example/kvstore/persistent_kvstore.go +++ b/abci/example/kvstore/persistent_kvstore.go @@ -7,11 +7,10 @@ import ( "strconv" "strings" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/abci/example/code" "github.com/lazyledger/lazyledger-core/abci/types" cryptoenc "github.com/lazyledger/lazyledger-core/crypto/encoding" + dbb "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" "github.com/lazyledger/lazyledger-core/libs/log" pc "github.com/lazyledger/lazyledger-core/proto/tendermint/crypto" ) @@ -37,7 +36,7 @@ type PersistentKVStoreApplication struct { func NewPersistentKVStoreApplication(dbDir string) *PersistentKVStoreApplication { name := "kvstore" - db, err := dbm.NewGoLevelDB(name, dbDir) + db, err := dbb.NewDB(name, dbDir) if err != nil { panic(err) } diff --git a/blockchain/v0/reactor_test.go b/blockchain/v0/reactor_test.go index 3599381e98..ffa842cc33 100644 --- a/blockchain/v0/reactor_test.go +++ b/blockchain/v0/reactor_test.go @@ -11,10 +11,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/mempool/mock" "github.com/lazyledger/lazyledger-core/p2p" @@ -69,8 +68,8 @@ func newBlockchainReactor( panic(fmt.Errorf("error start app: %w", err)) } - blockDB := dbm.NewMemDB() - stateDB := dbm.NewMemDB() + blockDB := memdb.NewDB() + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) blockStore := store.NewBlockStore(blockDB) @@ -83,7 +82,7 @@ func newBlockchainReactor( // NOTE we have to create and commit the blocks first because // pool.height is determined from the store. fastSync := true - db := dbm.NewMemDB() + db := memdb.NewDB() stateStore = sm.NewStore(db) blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mock.Mempool{}, sm.EmptyEvidencePool{}) diff --git a/blockchain/v2/reactor_test.go b/blockchain/v2/reactor_test.go index f01d9f8d1b..187b38b9d4 100644 --- a/blockchain/v2/reactor_test.go +++ b/blockchain/v2/reactor_test.go @@ -12,12 +12,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" abci "github.com/lazyledger/lazyledger-core/abci/types" "github.com/lazyledger/lazyledger-core/behaviour" bc "github.com/lazyledger/lazyledger-core/blockchain" cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/libs/service" "github.com/lazyledger/lazyledger-core/mempool/mock" @@ -65,14 +65,17 @@ type mockBlockStore struct { blocks map[int64]*types.Block } +//nolint:unused func (ml *mockBlockStore) Height() int64 { return int64(len(ml.blocks)) } +//nolint:unused func (ml *mockBlockStore) LoadBlock(height int64) *types.Block { return ml.blocks[height] } +//nolint:unused func (ml *mockBlockStore) SaveBlock(block *types.Block, part *types.PartSet, commit *types.Commit) { ml.blocks[block.Height] = block } @@ -166,7 +169,7 @@ func newTestReactor(p testReactorParams) *BlockchainReactor { if err != nil { panic(fmt.Errorf("error start app: %w", err)) } - db := dbm.NewMemDB() + db := memdb.NewDB() stateStore := sm.NewStore(db) appl = sm.NewBlockExecutor(stateStore, p.logger, proxyApp.Consensus(), mock.Mempool{}, sm.EmptyEvidencePool{}) if err = stateStore.Save(state); err != nil { @@ -511,15 +514,15 @@ func newReactorStore( panic(fmt.Errorf("error start app: %w", err)) } - stateDB := dbm.NewMemDB() - blockStore := store.NewBlockStore(dbm.NewMemDB()) + stateDB := memdb.NewDB() + blockStore := store.NewBlockStore(memdb.NewDB()) stateStore := sm.NewStore(stateDB) state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc) if err != nil { panic(fmt.Errorf("error constructing state from genesis file: %w", err)) } - db := dbm.NewMemDB() + db := memdb.NewDB() stateStore = sm.NewStore(db) blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mock.Mempool{}, sm.EmptyEvidencePool{}) diff --git a/cmd/tendermint/commands/light.go b/cmd/tendermint/commands/light.go index dab520661b..f389f25214 100644 --- a/cmd/tendermint/commands/light.go +++ b/cmd/tendermint/commands/light.go @@ -14,9 +14,9 @@ import ( "github.com/spf13/cobra" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/crypto/merkle" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" "github.com/lazyledger/lazyledger-core/libs/log" tmmath "github.com/lazyledger/lazyledger-core/libs/math" tmos "github.com/lazyledger/lazyledger-core/libs/os" @@ -122,7 +122,7 @@ func runProxy(cmd *cobra.Command, args []string) error { witnessesAddrs = strings.Split(witnessAddrsJoined, ",") } - db, err := dbm.NewGoLevelDB("light-client-db", dir) + db, err := badgerdb.NewDB("light-client-db", dir) if err != nil { return fmt.Errorf("can't create a db: %w", err) } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index db03590b65..4e42ff3767 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -85,10 +85,6 @@ func AddNodeFlags(cmd *cobra.Command) { "the possible interval between empty blocks") // db flags - cmd.Flags().String( - "db-backend", - config.DBBackend, - "database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb") cmd.Flags().String( "db-dir", config.DBPath, diff --git a/config/config.go b/config/config.go index d4f2c84121..ab1af804ad 100644 --- a/config/config.go +++ b/config/config.go @@ -165,27 +165,6 @@ type BaseConfig struct { //nolint: maligned // and verifying their commits FastSyncMode bool `mapstructure:"fast-sync"` - // Database backend: goleveldb | cleveldb | boltdb | rocksdb - // * goleveldb (github.com/syndtr/goleveldb - most popular implementation) - // - pure go - // - stable - // * cleveldb (uses levigo wrapper) - // - fast - // - requires gcc - // - use cleveldb build tag (go build -tags cleveldb) - // * boltdb (uses etcd's fork of bolt - github.com/etcd-io/bbolt) - // - EXPERIMENTAL - // - may be faster is some use-cases (random reads - indexer) - // - use boltdb build tag (go build -tags boltdb) - // * rocksdb (uses github.com/tecbot/gorocksdb) - // - EXPERIMENTAL - // - requires gcc - // - use rocksdb build tag (go build -tags rocksdb) - // * badgerdb (uses github.com/dgraph-io/badger) - // - EXPERIMENTAL - // - use badgerdb build tag (go build -tags badgerdb) - DBBackend string `mapstructure:"db-backend"` - // Database directory DBPath string `mapstructure:"db-dir"` @@ -233,7 +212,6 @@ func DefaultBaseConfig() BaseConfig { LogFormat: LogFormatPlain, FastSyncMode: true, FilterPeers: false, - DBBackend: "goleveldb", DBPath: "data", } } @@ -244,7 +222,6 @@ func TestBaseConfig() BaseConfig { cfg.chainID = "tendermint_test" cfg.ProxyApp = "kvstore" cfg.FastSyncMode = false - cfg.DBBackend = "memdb" return cfg } diff --git a/config/toml.go b/config/toml.go index e4e94950e9..c39cdcaffb 100644 --- a/config/toml.go +++ b/config/toml.go @@ -93,27 +93,6 @@ moniker = "{{ .BaseConfig.Moniker }}" # and verifying their commits fast-sync = {{ .BaseConfig.FastSyncMode }} -# Database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb -# * goleveldb (github.com/syndtr/goleveldb - most popular implementation) -# - pure go -# - stable -# * cleveldb (uses levigo wrapper) -# - fast -# - requires gcc -# - use cleveldb build tag (go build -tags cleveldb) -# * boltdb (uses etcd's fork of bolt - github.com/etcd-io/bbolt) -# - EXPERIMENTAL -# - may be faster is some use-cases (random reads - indexer) -# - use boltdb build tag (go build -tags boltdb) -# * rocksdb (uses github.com/tecbot/gorocksdb) -# - EXPERIMENTAL -# - requires gcc -# - use rocksdb build tag (go build -tags rocksdb) -# * badgerdb (uses github.com/dgraph-io/badger) -# - EXPERIMENTAL -# - use badgerdb build tag (go build -tags badgerdb) -db-backend = "{{ .BaseConfig.DBBackend }}" - # Database directory db-dir = "{{ js .BaseConfig.DBPath }}" diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index a623f81f97..336275b0d3 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -12,11 +12,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - abcicli "github.com/lazyledger/lazyledger-core/abci/client" abci "github.com/lazyledger/lazyledger-core/abci/types" "github.com/lazyledger/lazyledger-core/evidence" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/libs/service" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" @@ -45,7 +44,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { for i := 0; i < nValidators; i++ { logger := consensusLogger().With("test", "byzantine", "validator", i) - stateDB := dbm.NewMemDB() // each state needs its own db + stateDB := memdb.NewDB() // each state needs its own db stateStore := sm.NewStore(stateDB) state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc) thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) @@ -55,7 +54,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { vals := types.TM2PB.ValidatorUpdates(state.Validators) app.InitChain(abci.RequestInitChain{Validators: vals}) - blockDB := dbm.NewMemDB() + blockDB := memdb.NewDB() blockStore := store.NewBlockStore(blockDB) // one for mempool, one for consensus @@ -71,7 +70,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { } // Make a full instance of the evidence pool - evidenceDB := dbm.NewMemDB() + evidenceDB := memdb.NewDB() evpool, err := evidence.NewPool(evidenceDB, stateStore, blockStore) require.NoError(t, err) evpool.SetLogger(logger.With("module", "evidence")) diff --git a/consensus/common_test.go b/consensus/common_test.go index c448d65249..b70dcf7fd3 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -17,8 +17,6 @@ import ( "path" - dbm "github.com/tendermint/tm-db" - abcicli "github.com/lazyledger/lazyledger-core/abci/client" "github.com/lazyledger/lazyledger-core/abci/example/counter" "github.com/lazyledger/lazyledger-core/abci/example/kvstore" @@ -26,6 +24,8 @@ import ( cfg "github.com/lazyledger/lazyledger-core/config" cstypes "github.com/lazyledger/lazyledger-core/consensus/types" tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmos "github.com/lazyledger/lazyledger-core/libs/os" tmpubsub "github.com/lazyledger/lazyledger-core/libs/pubsub" @@ -52,7 +52,7 @@ type cleanupFunc func() var ( config *cfg.Config // NOTE: must be reset for each _test.go file consensusReplayConfig *cfg.Config - ensureTimeout = time.Millisecond * 200 + ensureTimeout = 600 * time.Millisecond ) func ensureDir(dir string, mode os.FileMode) { @@ -363,7 +363,7 @@ func newStateWithConfig( pv types.PrivValidator, app abci.Application, ) *State { - blockDB := dbm.NewMemDB() + blockDB := memdb.NewDB() return newStateWithConfigAndBlockStore(thisConfig, state, pv, app, blockDB) } @@ -685,7 +685,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou logger := consensusLogger() configRootDirs := make([]string, 0, nValidators) for i := 0; i < nValidators; i++ { - stateDB := dbm.NewMemDB() // each state needs its own db + stateDB := memdb.NewDB() // each state needs its own db stateStore := sm.NewStore(stateDB) state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc) thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) @@ -723,7 +723,7 @@ func randConsensusNetWithPeers( var peer0Config *cfg.Config configRootDirs := make([]string, 0, nPeers) for i := 0; i < nPeers; i++ { - stateDB := dbm.NewMemDB() // each state needs its own db + stateDB := memdb.NewDB() // each state needs its own db stateStore := sm.NewStore(stateDB) state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc) thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 88e41ac29a..511cc8fdcb 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -10,10 +10,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/abci/example/code" abci "github.com/lazyledger/lazyledger-core/abci/types" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" mempl "github.com/lazyledger/lazyledger-core/mempool" sm "github.com/lazyledger/lazyledger-core/state" "github.com/lazyledger/lazyledger-core/types" @@ -114,7 +113,7 @@ func deliverTxsRange(cs *State, start, end int) { func TestMempoolTxConcurrentWithCommit(t *testing.T) { state, privVals := randGenesisState(1, false, 10) - blockDB := dbm.NewMemDB() + blockDB := memdb.NewDB() stateStore := sm.NewStore(blockDB) cs := newStateWithConfigAndBlockStore(config, state, privVals[0], NewCounterApplication(), blockDB) err := stateStore.Save(state) @@ -139,7 +138,7 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) { func TestMempoolRmBadTx(t *testing.T) { state, privVals := randGenesisState(1, false, 10) app := NewCounterApplication() - blockDB := dbm.NewMemDB() + blockDB := memdb.NewDB() stateStore := sm.NewStore(blockDB) cs := newStateWithConfigAndBlockStore(config, state, privVals[0], app, blockDB) err := stateStore.Save(state) diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index b3c601aaf7..f2bbb06961 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -15,8 +15,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - abcicli "github.com/lazyledger/lazyledger-core/abci/client" "github.com/lazyledger/lazyledger-core/abci/example/kvstore" abci "github.com/lazyledger/lazyledger-core/abci/types" @@ -26,6 +24,7 @@ import ( "github.com/lazyledger/lazyledger-core/crypto/tmhash" "github.com/lazyledger/lazyledger-core/libs/bits" "github.com/lazyledger/lazyledger-core/libs/bytes" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" mempl "github.com/lazyledger/lazyledger-core/mempool" @@ -135,7 +134,7 @@ func TestReactorWithEvidence(t *testing.T) { css := make([]*State, nValidators) logger := consensusLogger() for i := 0; i < nValidators; i++ { - stateDB := dbm.NewMemDB() // each state needs its own db + stateDB := memdb.NewDB() // each state needs its own db stateStore := sm.NewStore(stateDB) state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc) thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) @@ -149,7 +148,7 @@ func TestReactorWithEvidence(t *testing.T) { // duplicate code from: // css[i] = newStateWithConfig(thisConfig, state, privVals[i], app) - blockDB := dbm.NewMemDB() + blockDB := memdb.NewDB() blockStore := store.NewBlockStore(blockDB) // one for mempool, one for consensus diff --git a/consensus/replay_file.go b/consensus/replay_file.go index df1cd16a57..dd3aadca82 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -10,9 +10,8 @@ import ( "strconv" "strings" - dbm "github.com/tendermint/tm-db" - cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" "github.com/lazyledger/lazyledger-core/libs/log" tmos "github.com/lazyledger/lazyledger-core/libs/os" "github.com/lazyledger/lazyledger-core/proxy" @@ -284,16 +283,15 @@ func (pb *playback) replayConsoleLoop() int { // convenience for replay mode func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *State { - dbType := dbm.BackendType(config.DBBackend) // Get BlockStore - blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir()) + blockStoreDB, err := badgerdb.NewDB("blockstore", config.DBDir()) if err != nil { tmos.Exit(err.Error()) } blockStore := store.NewBlockStore(blockStoreDB) // Get State - stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) + stateDB, err := badgerdb.NewDB("state", config.DBDir()) if err != nil { tmos.Exit(err.Error()) } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index a22b2ccc24..6a45456491 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -16,13 +16,14 @@ import ( "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" "github.com/lazyledger/lazyledger-core/abci/example/kvstore" abci "github.com/lazyledger/lazyledger-core/abci/types" cfg "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto" cryptoenc "github.com/lazyledger/lazyledger-core/crypto/encoding" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" mempl "github.com/lazyledger/lazyledger-core/mempool" @@ -157,7 +158,7 @@ LOOP: // create consensus state from a clean slate logger := log.NewNopLogger() - blockDB := dbm.NewMemDB() + blockDB := memdb.NewDB() stateDB := blockDB stateStore := sm.NewStore(stateDB) state, err := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile()) @@ -664,7 +665,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin if testValidatorsChange { testConfig := ResetConfig(fmt.Sprintf("%s_%v_m", t.Name(), mode)) defer func() { _ = os.RemoveAll(testConfig.RootDir) }() - stateDB = dbm.NewMemDB() + stateDB = memdb.NewDB() genesisState = sim.GenesisState config = sim.Config @@ -716,7 +717,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin // run nBlocks against a new client to build up the app state. // use a throwaway tendermint state proxyApp := proxy.NewAppConns(clientCreator2) - stateDB1 := dbm.NewMemDB() + stateDB1 := memdb.NewDB() stateStore := sm.NewStore(stateDB1) err := stateStore.Save(genesisState) require.NoError(t, err) @@ -1160,7 +1161,7 @@ func stateAndStore( config *cfg.Config, pubKey crypto.PubKey, appVersion uint64) (dbm.DB, sm.State, *mockBlockStore) { - stateDB := dbm.NewMemDB() + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile()) state.Version.Consensus.App = appVersion diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 7aafe21e84..6ecbb2401d 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -9,10 +9,9 @@ import ( "testing" "time" - db "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/abci/example/kvstore" cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" "github.com/lazyledger/lazyledger-core/privval" @@ -48,7 +47,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) { if err != nil { return fmt.Errorf("failed to read genesis file: %w", err) } - blockStoreDB := db.NewMemDB() + blockStoreDB := memdb.NewDB() stateDB := blockStoreDB stateStore := sm.NewStore(stateDB) state, err := sm.MakeGenesisState(genDoc) diff --git a/crypto/hash.go b/crypto/hash.go index dd1b4c1ddf..e1d22523f2 100644 --- a/crypto/hash.go +++ b/crypto/hash.go @@ -6,6 +6,6 @@ import ( func Sha256(bytes []byte) []byte { hasher := sha256.New() - hasher.Write(bytes) //nolint:errcheck // ignore error + hasher.Write(bytes) return hasher.Sum(nil) } diff --git a/crypto/merkle/proof_value.go b/crypto/merkle/proof_value.go index 411d1f31e0..b68332024a 100644 --- a/crypto/merkle/proof_value.go +++ b/crypto/merkle/proof_value.go @@ -80,7 +80,7 @@ func (op ValueOp) Run(args [][]byte) ([][]byte, error) { } value := args[0] hasher := tmhash.New() - hasher.Write(value) //nolint: errcheck // does not error + hasher.Write(value) vhash := hasher.Sum(nil) bz := new(bytes.Buffer) diff --git a/evidence/pool.go b/evidence/pool.go index c03708e78b..ceec9e95fe 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -11,9 +11,9 @@ import ( "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" - dbm "github.com/tendermint/tm-db" clist "github.com/lazyledger/lazyledger-core/libs/clist" + dbm "github.com/lazyledger/lazyledger-core/libs/db" "github.com/lazyledger/lazyledger-core/libs/log" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" sm "github.com/lazyledger/lazyledger-core/state" diff --git a/evidence/pool_test.go b/evidence/pool_test.go index 0dfdb4ebbd..2f8af65eb6 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -9,10 +9,10 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/evidence" "github.com/lazyledger/lazyledger-core/evidence/mocks" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" tmversion "github.com/lazyledger/lazyledger-core/proto/tendermint/version" @@ -40,7 +40,7 @@ func TestEvidencePoolBasic(t *testing.T) { var ( height = int64(1) stateStore = &smmocks.Store{} - evidenceDB = dbm.NewMemDB() + evidenceDB = memdb.NewDB() blockStore = &mocks.BlockStore{} ) @@ -100,7 +100,7 @@ func TestAddExpiredEvidence(t *testing.T) { val = types.NewMockPV() height = int64(30) stateStore = initializeValidatorState(val, height) - evidenceDB = dbm.NewMemDB() + evidenceDB = memdb.NewDB() blockStore = &mocks.BlockStore{} expiredEvidenceTime = time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) expiredHeight = int64(2) @@ -275,7 +275,7 @@ func TestCheckEvidenceWithLightClientAttack(t *testing.T) { blockStore.On("LoadBlockMeta", height).Return(&types.BlockMeta{Header: *trustedHeader}) blockStore.On("LoadBlockCommit", height).Return(trustedCommit) - pool, err := evidence.NewPool(dbm.NewMemDB(), stateStore, blockStore) + pool, err := evidence.NewPool(memdb.NewDB(), stateStore, blockStore) require.NoError(t, err) pool.SetLogger(log.TestingLogger()) @@ -298,11 +298,11 @@ func TestRecoverPendingEvidence(t *testing.T) { height := int64(10) val := types.NewMockPV() valAddress := val.PrivKey.PubKey().Address() - evidenceDB := dbm.NewMemDB() + evidenceDB := memdb.NewDB() stateStore := initializeValidatorState(val, height) state, err := stateStore.Load() require.NoError(t, err) - blockStore := initializeBlockStore(dbm.NewMemDB(), state, valAddress) + blockStore := initializeBlockStore(memdb.NewDB(), state, valAddress) // create previous pool and populate it pool, err := evidence.NewPool(evidenceDB, stateStore, blockStore) require.NoError(t, err) @@ -343,7 +343,7 @@ func TestRecoverPendingEvidence(t *testing.T) { } func initializeStateFromValidatorSet(valSet *types.ValidatorSet, height int64) sm.Store { - stateDB := dbm.NewMemDB() + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) state := sm.State{ ChainID: evidenceChainID, @@ -426,10 +426,10 @@ func makeCommit(height int64, valAddr []byte) *types.Commit { func defaultTestPool(height int64) (*evidence.Pool, types.MockPV) { val := types.NewMockPV() valAddress := val.PrivKey.PubKey().Address() - evidenceDB := dbm.NewMemDB() + evidenceDB := memdb.NewDB() stateStore := initializeValidatorState(val, height) state, _ := stateStore.Load() - blockStore := initializeBlockStore(dbm.NewMemDB(), state, valAddress) + blockStore := initializeBlockStore(memdb.NewDB(), state, valAddress) pool, err := evidence.NewPool(evidenceDB, stateStore, blockStore) if err != nil { panic("test evidence pool could not be created") diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go index f8952fc89f..f533621534 100644 --- a/evidence/reactor_test.go +++ b/evidence/reactor_test.go @@ -12,13 +12,12 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - cfg "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/tmhash" "github.com/lazyledger/lazyledger-core/evidence" "github.com/lazyledger/lazyledger-core/evidence/mocks" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/p2p" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" @@ -209,7 +208,7 @@ func makeAndConnectReactorsAndPools(config *cfg.Config, stateStores []sm.Store) evidenceTime := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC) for i := 0; i < N; i++ { - evidenceDB := dbm.NewMemDB() + evidenceDB := memdb.NewDB() blockStore := &mocks.BlockStore{} blockStore.On("LoadBlockMeta", mock.AnythingOfType("int64")).Return( &types.BlockMeta{Header: types.Header{Time: evidenceTime}}, diff --git a/evidence/verify_test.go b/evidence/verify_test.go index f1f5362ff9..6f28fceb85 100644 --- a/evidence/verify_test.go +++ b/evidence/verify_test.go @@ -7,12 +7,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/tmhash" "github.com/lazyledger/lazyledger-core/evidence" "github.com/lazyledger/lazyledger-core/evidence/mocks" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" tmversion "github.com/lazyledger/lazyledger-core/proto/tendermint/version" @@ -103,7 +102,7 @@ func TestVerifyLightClientAttack_Lunatic(t *testing.T) { blockStore.On("LoadBlockCommit", int64(4)).Return(commit) blockStore.On("LoadBlockCommit", int64(10)).Return(trustedCommit) - pool, err := evidence.NewPool(dbm.NewMemDB(), stateStore, blockStore) + pool, err := evidence.NewPool(memdb.NewDB(), stateStore, blockStore) require.NoError(t, err) pool.SetLogger(log.TestingLogger()) @@ -201,7 +200,7 @@ func TestVerifyLightClientAttack_Equivocation(t *testing.T) { blockStore.On("LoadBlockMeta", int64(10)).Return(&types.BlockMeta{Header: *trustedHeader}) blockStore.On("LoadBlockCommit", int64(10)).Return(trustedCommit) - pool, err := evidence.NewPool(dbm.NewMemDB(), stateStore, blockStore) + pool, err := evidence.NewPool(memdb.NewDB(), stateStore, blockStore) require.NoError(t, err) pool.SetLogger(log.TestingLogger()) @@ -276,7 +275,7 @@ func TestVerifyLightClientAttack_Amnesia(t *testing.T) { blockStore.On("LoadBlockMeta", int64(10)).Return(&types.BlockMeta{Header: *trustedHeader}) blockStore.On("LoadBlockCommit", int64(10)).Return(trustedCommit) - pool, err := evidence.NewPool(dbm.NewMemDB(), stateStore, blockStore) + pool, err := evidence.NewPool(memdb.NewDB(), stateStore, blockStore) require.NoError(t, err) pool.SetLogger(log.TestingLogger()) @@ -369,7 +368,7 @@ func TestVerifyDuplicateVoteEvidence(t *testing.T) { blockStore := &mocks.BlockStore{} blockStore.On("LoadBlockMeta", int64(10)).Return(&types.BlockMeta{Header: types.Header{Time: defaultEvidenceTime}}) - pool, err := evidence.NewPool(dbm.NewMemDB(), stateStore, blockStore) + pool, err := evidence.NewPool(memdb.NewDB(), stateStore, blockStore) require.NoError(t, err) evList := types.EvidenceList{goodEv} diff --git a/go.mod b/go.mod index 7a20c45827..1863dc9628 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,13 @@ require ( github.com/Workiva/go-datastructures v1.0.52 github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 + github.com/dgraph-io/badger/v3 v3.2011.1 github.com/fortytw2/leaktest v1.3.0 github.com/go-kit/kit v0.10.0 github.com/go-logfmt/logfmt v0.5.0 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.4.3 + github.com/google/btree v1.0.0 github.com/gorilla/websocket v1.4.2 github.com/gtank/merlin v0.1.1 github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff @@ -37,7 +39,7 @@ require ( github.com/spf13/cobra v1.1.1 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 - github.com/tendermint/tm-db v0.6.4 + github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 google.golang.org/genproto v0.0.0-20201119123407-9b1e624d6bc4 // indirect diff --git a/go.sum b/go.sum index e1a876d17d..47a57c4fe3 100644 --- a/go.sum +++ b/go.sum @@ -143,6 +143,7 @@ github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -167,11 +168,11 @@ github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6ps github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= -github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v3 v3.2011.1 h1:Hmyof0WMEF/QtutX5SQHzIMnJQxb/IrSzhjckV2SD6g= +github.com/dgraph-io/badger/v3 v3.2011.1/go.mod h1:0rLLrQpKVQAL0or/lBLMQznhr6dWWX7h5AKnmnqx268= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d h1:eQYOG6A4td1tht0NdJB9Ls6DsXRGb2Ft6X9REU/MbbE= +github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d/go.mod h1:tv2ec8nA7vRpSYX7/MbP52ihrUMXIHit54CQMq8npXQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -193,12 +194,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -219,6 +214,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-delve/delve v1.5.0/go.mod h1:c6b3a1Gry6x8a4LGCe/CWzrocrfaHvkUxCj3k4bvSUQ= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -278,12 +274,15 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.0 h1:/PtAHvnBY4Kqnx/xCQ3OIV9uYcSFGScBsWI3Oogeh6w= +github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -564,8 +563,6 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -894,6 +891,7 @@ github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl5 github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -937,6 +935,7 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mmcloughlin/avo v0.0.0-20201105074841-5d2f697d268f/go.mod h1:6aKT4zZIrpGqB3RpFU14ByCSSyKY6LfJz4J/JJChHfI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -1049,6 +1048,7 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= @@ -1176,12 +1176,14 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -1210,15 +1212,12 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= -github.com/tendermint/tm-db v0.6.4 h1:3N2jlnYQkXNQclQwd/eKV/NzlqPlfK21cpRRIx80XXQ= -github.com/tendermint/tm-db v0.6.4/go.mod h1:dptYhIpJ2M5kUuenLr+Yyf3zQOv1SgBZcl8/BmWlMBw= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/twitchyliquid64/golang-asm v0.15.0/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -1269,8 +1268,6 @@ gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWj gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40/go.mod h1:rOnSnoRyxMI3fe/7KIbVcsHRGxe30OONv8dEgo+vCfA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1283,6 +1280,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1311,6 +1309,8 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1536,6 +1536,7 @@ golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1545,6 +1546,7 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/libs/db/badgerdb/db.go b/libs/db/badgerdb/db.go new file mode 100644 index 0000000000..94a727dc46 --- /dev/null +++ b/libs/db/badgerdb/db.go @@ -0,0 +1,270 @@ +package badgerdb + +import ( + "bytes" + "os" + "path/filepath" + + "github.com/dgraph-io/badger/v3" + + "github.com/lazyledger/lazyledger-core/libs/db" +) + +// NewDB creates a Badger key-value store backed to the +// directory dir supplied. If dir does not exist, it will be created. +func NewDB(dbName, dir string) (*BadgerDB, error) { + // Since Badger doesn't support database names, we join both to obtain + // the final directory to use for the database. + path := filepath.Join(dir, dbName) + + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } + opts := badger.DefaultOptions(path) + opts.SyncWrites = false // note that we have Sync methods + opts.Logger = nil // badger is too chatty by default + return NewDBWithOptions(opts) +} + +// NewDBWithOptions creates a BadgerDB key value store +// gives the flexibility of initializing a database with the +// respective options. +func NewDBWithOptions(opts badger.Options) (*BadgerDB, error) { + db, err := badger.Open(opts) + if err != nil { + return nil, err + } + return &BadgerDB{db: db}, nil +} + +type BadgerDB struct { + db *badger.DB +} + +var _ db.DB = (*BadgerDB)(nil) + +func (b *BadgerDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, db.ErrKeyEmpty + } + var val []byte + err := b.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(key) + if err == badger.ErrKeyNotFound { + return nil + } else if err != nil { + return err + } + val, err = item.ValueCopy(nil) + if err == nil && val == nil { + val = []byte{} + } + return err + }) + return val, err +} + +func (b *BadgerDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, db.ErrKeyEmpty + } + var found bool + err := b.db.View(func(txn *badger.Txn) error { + _, err := txn.Get(key) + if err != nil && err != badger.ErrKeyNotFound { + return err + } + found = (err != badger.ErrKeyNotFound) + return nil + }) + return found, err +} + +func (b *BadgerDB) Set(key, value []byte) error { + if len(key) == 0 { + return db.ErrKeyEmpty + } + if value == nil { + return db.ErrValueNil + } + return b.db.Update(func(txn *badger.Txn) error { + return txn.Set(key, value) + }) +} + +func withSync(db *badger.DB, err error) error { + if err != nil { + return err + } + return db.Sync() +} + +func (b *BadgerDB) SetSync(key, value []byte) error { + return withSync(b.db, b.Set(key, value)) +} + +func (b *BadgerDB) Delete(key []byte) error { + if len(key) == 0 { + return db.ErrKeyEmpty + } + return b.db.Update(func(txn *badger.Txn) error { + return txn.Delete(key) + }) +} + +func (b *BadgerDB) DeleteSync(key []byte) error { + return withSync(b.db, b.Delete(key)) +} + +func (b *BadgerDB) Close() error { + return b.db.Close() +} + +func (b *BadgerDB) Print() error { + return nil +} + +func (b *BadgerDB) iteratorOpts(start, end []byte, opts badger.IteratorOptions) (*badgerDBIterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, db.ErrKeyEmpty + } + txn := b.db.NewTransaction(false) + iter := txn.NewIterator(opts) + iter.Rewind() + iter.Seek(start) + if opts.Reverse && iter.Valid() && bytes.Equal(iter.Item().Key(), start) { + // If we're going in reverse, our starting point was "end", + // which is exclusive. + iter.Next() + } + return &badgerDBIterator{ + reverse: opts.Reverse, + start: start, + end: end, + + txn: txn, + iter: iter, + }, nil +} + +func (b *BadgerDB) Iterator(start, end []byte) (db.Iterator, error) { + opts := badger.DefaultIteratorOptions + return b.iteratorOpts(start, end, opts) +} + +func (b *BadgerDB) ReverseIterator(start, end []byte) (db.Iterator, error) { + opts := badger.DefaultIteratorOptions + opts.Reverse = true + return b.iteratorOpts(end, start, opts) +} + +// todo: see about exposing badgerdb stats +func (b *BadgerDB) Stats() map[string]string { + return nil +} + +func (b *BadgerDB) NewBatch() db.Batch { + wb := &badgerDBBatch{ + db: b.db, + wb: b.db.NewWriteBatch(), + } + return wb +} + +var _ db.Batch = (*badgerDBBatch)(nil) + +type badgerDBBatch struct { + db *badger.DB + wb *badger.WriteBatch +} + +func (b *badgerDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return db.ErrKeyEmpty + } + if value == nil { + return db.ErrValueNil + } + return b.wb.Set(key, value) +} + +func (b *badgerDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return db.ErrKeyEmpty + } + return b.wb.Delete(key) +} + +func (b *badgerDBBatch) Write() error { + return b.wb.Flush() +} + +func (b *badgerDBBatch) WriteSync() error { + return withSync(b.db, b.Write()) +} + +func (b *badgerDBBatch) Close() error { + b.wb.Cancel() + + return nil +} + +type badgerDBIterator struct { + reverse bool + start, end []byte + + txn *badger.Txn + iter *badger.Iterator + + lastErr error +} + +func (i *badgerDBIterator) Close() error { + i.iter.Close() + i.txn.Discard() + return nil +} + +func (i *badgerDBIterator) Domain() (start, end []byte) { return i.start, i.end } +func (i *badgerDBIterator) Error() error { return i.lastErr } + +func (i *badgerDBIterator) Next() { + if !i.Valid() { + panic("iterator is invalid") + } + i.iter.Next() +} + +func (i *badgerDBIterator) Valid() bool { + if !i.iter.Valid() { + return false + } + if len(i.end) > 0 { + key := i.iter.Item().Key() + if c := bytes.Compare(key, i.end); (!i.reverse && c >= 0) || (i.reverse && c < 0) { + // We're at the end key, or past the end. + return false + } + } + return true +} + +func (i *badgerDBIterator) Key() []byte { + if !i.Valid() { + panic("iterator is invalid") + } + // Note that we don't use KeyCopy, so this is only valid until the next + // call to Next. + return i.iter.Item().KeyCopy(nil) +} + +func (i *badgerDBIterator) Value() []byte { + if !i.Valid() { + panic("iterator is invalid") + } + val, err := i.iter.Item().ValueCopy(nil) + if err != nil { + i.lastErr = err + } + return val +} diff --git a/libs/db/badgerdb/db_test.go b/libs/db/badgerdb/db_test.go new file mode 100644 index 0000000000..d68ead87a4 --- /dev/null +++ b/libs/db/badgerdb/db_test.go @@ -0,0 +1,369 @@ +package badgerdb + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + tmdb "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/internal/dbtest" +) + +func TestBackendGetSetDelete(t *testing.T) { + // Default + dirname, err := ioutil.TempDir("", fmt.Sprintf("test_backend_%s_", "badger")) + require.Nil(t, err) + db, err := NewDB("badger", dirname) + require.NoError(t, err) + defer dbtest.CleanupDBDir(dirname, "testdb") + + // A nonexistent key should return nil. + value, err := db.Get([]byte("a")) + require.NoError(t, err) + require.Nil(t, value) + + ok, err := db.Has([]byte("a")) + require.NoError(t, err) + require.False(t, ok) + + // Set and get a value. + err = db.Set([]byte("a"), []byte{0x01}) + require.NoError(t, err) + + ok, err = db.Has([]byte("a")) + require.NoError(t, err) + require.True(t, ok) + + value, err = db.Get([]byte("a")) + require.NoError(t, err) + require.Equal(t, []byte{0x01}, value) + + err = db.SetSync([]byte("b"), []byte{0x02}) + require.NoError(t, err) + + value, err = db.Get([]byte("b")) + require.NoError(t, err) + require.Equal(t, []byte{0x02}, value) + + // Deleting a non-existent value is fine. + err = db.Delete([]byte("x")) + require.NoError(t, err) + + err = db.DeleteSync([]byte("x")) + require.NoError(t, err) + + // Delete a value. + err = db.Delete([]byte("a")) + require.NoError(t, err) + + value, err = db.Get([]byte("a")) + require.NoError(t, err) + require.Nil(t, value) + + err = db.DeleteSync([]byte("b")) + require.NoError(t, err) + + value, err = db.Get([]byte("b")) + require.NoError(t, err) + require.Nil(t, value) + + // Setting, getting, and deleting an empty key should error. + _, err = db.Get([]byte{}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + _, err = db.Get(nil) + require.Equal(t, tmdb.ErrKeyEmpty, err) + + _, err = db.Has([]byte{}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + _, err = db.Has(nil) + require.Equal(t, tmdb.ErrKeyEmpty, err) + + err = db.Set([]byte{}, []byte{0x01}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = db.Set(nil, []byte{0x01}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = db.SetSync([]byte{}, []byte{0x01}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = db.SetSync(nil, []byte{0x01}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + + err = db.Delete([]byte{}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = db.Delete(nil) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = db.DeleteSync([]byte{}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = db.DeleteSync(nil) + require.Equal(t, tmdb.ErrKeyEmpty, err) + + // Setting a nil value should error, but an empty value is fine. + err = db.Set([]byte("x"), nil) + require.Equal(t, tmdb.ErrValueNil, err) + err = db.SetSync([]byte("x"), nil) + require.Equal(t, tmdb.ErrValueNil, err) + + err = db.Set([]byte("x"), []byte{}) + require.NoError(t, err) + err = db.SetSync([]byte("x"), []byte{}) + require.NoError(t, err) + value, err = db.Get([]byte("x")) + require.NoError(t, err) + require.Equal(t, []byte{}, value) +} + +func TestDBIterator(t *testing.T) { + name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) + dir := os.TempDir() + db, err := NewDB(name, dir) + require.NoError(t, err) + defer dbtest.CleanupDBDir(dir, name) + + for i := 0; i < 10; i++ { + if i != 6 { // but skip 6. + err := db.Set(dbtest.Int642Bytes(int64(i)), []byte{}) + require.NoError(t, err) + } + } + + // Blank iterator keys should error + _, err = db.Iterator([]byte{}, nil) + require.Equal(t, tmdb.ErrKeyEmpty, err) + _, err = db.Iterator(nil, []byte{}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + _, err = db.ReverseIterator([]byte{}, nil) + require.Equal(t, tmdb.ErrKeyEmpty, err) + _, err = db.ReverseIterator(nil, []byte{}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + + itr, err := db.Iterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator") + + ritr, err := db.ReverseIterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator") + + itr, err = db.Iterator(nil, dbtest.Int642Bytes(0)) + require.NoError(t, err) + verifyIterator(t, itr, []int64(nil), "forward iterator to 0") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(10), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64(nil), "reverse iterator from 10 (ex)") + + itr, err = db.Iterator(dbtest.Int642Bytes(0), nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 0") + + itr, err = db.Iterator(dbtest.Int642Bytes(1), nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 1") + + ritr, err = db.ReverseIterator(nil, dbtest.Int642Bytes(10)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 10 (ex)") + + ritr, err = db.ReverseIterator(nil, dbtest.Int642Bytes(9)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 9 (ex)") + + ritr, err = db.ReverseIterator(nil, dbtest.Int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 8 (ex)") + + itr, err = db.Iterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 6") + + itr, err = db.Iterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 7") + + itr, err = db.Iterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5, 7}, "forward iterator from 5 to 8") + + itr, err = db.Iterator(dbtest.Int642Bytes(6), dbtest.Int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, itr, []int64(nil), "forward iterator from 6 to 7") + + itr, err = db.Iterator(dbtest.Int642Bytes(6), dbtest.Int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{7}, "forward iterator from 6 to 8") + + itr, err = db.Iterator(dbtest.Int642Bytes(7), dbtest.Int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{7}, "forward iterator from 7 to 8") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(5)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{4}, "reverse iterator from 5 (ex) to 4") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{5, 4}, "reverse iterator from 6 (ex) to 4") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{5, 4}, "reverse iterator from 7 (ex) to 4") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{5}, "reverse iterator from 6 (ex) to 5") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(5), dbtest.Int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{5}, "reverse iterator from 7 (ex) to 5") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(6), dbtest.Int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64(nil), "reverse iterator from 7 (ex) to 6") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(10), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64(nil), "reverse iterator to 10") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(6), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7}, "reverse iterator to 6") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(5), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7, 5}, "reverse iterator to 5") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(8), dbtest.Int642Bytes(9)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{8}, "reverse iterator from 9 (ex) to 8") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(2), dbtest.Int642Bytes(4)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{3, 2}, "reverse iterator from 4 (ex) to 2") + + ritr, err = db.ReverseIterator(dbtest.Int642Bytes(4), dbtest.Int642Bytes(2)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64(nil), "reverse iterator from 2 (ex) to 4") + + // Ensure that the iterators don't panic with an empty database. + dir2, err := ioutil.TempDir("", "tm-db-test") + require.NoError(t, err) + db2, err := NewDB("badger", dir2) + require.NoError(t, err) + defer dbtest.CleanupDBDir(dir2, name) + + itr, err = db2.Iterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, itr, nil, "forward iterator with empty db") + + ritr, err = db2.ReverseIterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, ritr, nil, "reverse iterator with empty db") + +} + +func verifyIterator(t *testing.T, itr tmdb.Iterator, expected []int64, msg string) { + var list []int64 + for itr.Valid() { + key := itr.Key() + list = append(list, dbtest.Bytes2Int64(key)) + itr.Next() + } + assert.Equal(t, expected, list, msg) +} + +func TestDBBatch(t *testing.T) { + name := fmt.Sprintf("test_%x", dbtest.RandStr(12)) + dir := os.TempDir() + db, err := NewDB(name, dir) + require.NoError(t, err) + defer dbtest.CleanupDBDir(dir, name) + + // create a new batch, and some items - they should not be visible until we write + batch := db.NewBatch() + require.NoError(t, batch.Set([]byte("a"), []byte{1})) + require.NoError(t, batch.Set([]byte("b"), []byte{2})) + require.NoError(t, batch.Set([]byte("c"), []byte{3})) + assertKeyValues(t, db, map[string][]byte{}) + + err = batch.Write() + require.NoError(t, err) + assertKeyValues(t, db, map[string][]byte{"a": {1}, "b": {2}, "c": {3}}) + + // trying to modify or rewrite a written batch should error, but closing it should work + require.Error(t, batch.Set([]byte("a"), []byte{9})) + require.Error(t, batch.Delete([]byte("a"))) + require.Error(t, batch.Write()) + require.Error(t, batch.WriteSync()) + require.NoError(t, batch.Close()) + + // batches should write changes in order + batch = db.NewBatch() + require.NoError(t, batch.Delete([]byte("a"))) + require.NoError(t, batch.Set([]byte("a"), []byte{1})) + require.NoError(t, batch.Set([]byte("b"), []byte{1})) + require.NoError(t, batch.Set([]byte("b"), []byte{2})) + require.NoError(t, batch.Set([]byte("c"), []byte{3})) + require.NoError(t, batch.Delete([]byte("c"))) + require.NoError(t, batch.Write()) + require.NoError(t, batch.Close()) + assertKeyValues(t, db, map[string][]byte{"a": {1}, "b": {2}}) + + // empty and nil keys, as well as nil values, should be disallowed + batch = db.NewBatch() + err = batch.Set([]byte{}, []byte{0x01}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = batch.Set(nil, []byte{0x01}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = batch.Set([]byte("a"), nil) + require.Equal(t, tmdb.ErrValueNil, err) + + err = batch.Delete([]byte{}) + require.Equal(t, tmdb.ErrKeyEmpty, err) + err = batch.Delete(nil) + require.Equal(t, tmdb.ErrKeyEmpty, err) + + err = batch.Close() + require.NoError(t, err) + + // it should be possible to write an empty batch + batch = db.NewBatch() + err = batch.Write() + require.NoError(t, err) + assertKeyValues(t, db, map[string][]byte{"a": {1}, "b": {2}}) + + // it should be possible to close an empty batch, and to re-close a closed batch + batch = db.NewBatch() + batch.Close() + batch.Close() + + // all other operations on a closed batch should error + require.Error(t, batch.Set([]byte("a"), []byte{9})) + require.Error(t, batch.Delete([]byte("a"))) + require.Error(t, batch.Write()) + require.Error(t, batch.WriteSync()) +} + +func assertKeyValues(t *testing.T, db tmdb.DB, expect map[string][]byte) { + iter, err := db.Iterator(nil, nil) + require.NoError(t, err) + defer iter.Close() + + actual := make(map[string][]byte) + for ; iter.Valid(); iter.Next() { + require.NoError(t, iter.Error()) + actual[string(iter.Key())] = iter.Value() + } + + assert.Equal(t, expect, actual) +} diff --git a/libs/db/internal/dbtest/dbtest.go b/libs/db/internal/dbtest/dbtest.go new file mode 100644 index 0000000000..d21fa0b0fe --- /dev/null +++ b/libs/db/internal/dbtest/dbtest.go @@ -0,0 +1,207 @@ +package dbtest + +import ( + "bytes" + "encoding/binary" + "math/rand" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + lldb "github.com/lazyledger/lazyledger-core/libs/db" +) + +//---------------------------------------- +// Helper functions. + +func Valid(t *testing.T, itr lldb.Iterator, expected bool) { + valid := itr.Valid() + require.Equal(t, expected, valid) +} + +func Next(t *testing.T, itr lldb.Iterator, expected bool) { + itr.Next() + // assert.NoError(t, err) TODO: look at fixing this + valid := itr.Valid() + require.Equal(t, expected, valid) +} + +func NextPanics(t *testing.T, itr lldb.Iterator) { + assert.Panics(t, func() { itr.Next() }, "checkNextPanics expected an error but didn't") +} + +func Domain(t *testing.T, itr lldb.Iterator, start, end []byte) { + ds, de := itr.Domain() + assert.Equal(t, start, ds, "checkDomain domain start incorrect") + assert.Equal(t, end, de, "checkDomain domain end incorrect") +} + +func Item(t *testing.T, itr lldb.Iterator, key []byte, value []byte) { + v := itr.Value() + + k := itr.Key() + + assert.Exactly(t, key, k) + assert.Exactly(t, value, v) +} + +func Invalid(t *testing.T, itr lldb.Iterator) { + Valid(t, itr, false) + KeyPanics(t, itr) + ValuePanics(t, itr) + NextPanics(t, itr) +} + +func KeyPanics(t *testing.T, itr lldb.Iterator) { + assert.Panics(t, func() { itr.Key() }, "checkKeyPanics expected panic but didn't") +} + +func Value(t *testing.T, db lldb.DB, key []byte, valueWanted []byte) { + valueGot, err := db.Get(key) + assert.NoError(t, err) + assert.Equal(t, valueWanted, valueGot) +} + +func ValuePanics(t *testing.T, itr lldb.Iterator) { + assert.Panics(t, func() { itr.Value() }) +} + +func CleanupDBDir(dir, name string) { + err := os.RemoveAll(filepath.Join(dir, name) + ".db") + if err != nil { + panic(err) + } +} + +const strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters + +// RandStr constructs a random alphanumeric string of given length. +func RandStr(length int) string { + chars := []byte{} +MAIN_LOOP: + for { + val := rand.Int63() // nolint:gosec // G404: Use of weak random number generator + for i := 0; i < 10; i++ { + v := int(val & 0x3f) // rightmost 6 bits + if v >= 62 { // only 62 characters in strChars + val >>= 6 + continue + } else { + chars = append(chars, strChars[v]) + if len(chars) == length { + break MAIN_LOOP + } + val >>= 6 + } + } + } + + return string(chars) +} + +func Int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func Bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} + +func BenchmarkRangeScans(b *testing.B, db lldb.DB, dbSize int64) { + b.StopTimer() + + rangeSize := int64(10000) + if dbSize < rangeSize { + b.Errorf("db size %v cannot be less than range size %v", dbSize, rangeSize) + } + + for i := int64(0); i < dbSize; i++ { + bytes := Int642Bytes(i) + err := db.Set(bytes, bytes) + if err != nil { + // require.NoError() is very expensive (according to profiler), so check manually + b.Fatal(b, err) + } + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + + start := rand.Int63n(dbSize - rangeSize) // nolint: gosec + end := start + rangeSize + iter, err := db.Iterator(Int642Bytes(start), Int642Bytes(end)) + require.NoError(b, err) + count := 0 + for ; iter.Valid(); iter.Next() { + count++ + } + iter.Close() + require.EqualValues(b, rangeSize, count) + } +} + +func BenchmarkRandomReadsWrites(b *testing.B, db lldb.DB) { + b.StopTimer() + + // create dummy data + const numItems = int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + + // fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := rand.Int63n(numItems) // nolint: gosec + internal[idx]++ + val := internal[idx] + idxBytes := Int642Bytes(idx) + valBytes := Int642Bytes(val) + // fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + err := db.Set(idxBytes, valBytes) + if err != nil { + // require.NoError() is very expensive (according to profiler), so check manually + b.Fatal(b, err) + } + } + + // Read something + { + idx := rand.Int63n(numItems) // nolint: gosec + valExp := internal[idx] + idxBytes := Int642Bytes(idx) + valBytes, err := db.Get(idxBytes) + if err != nil { + // require.NoError() is very expensive (according to profiler), so check manually + b.Fatal(b, err) + } + // fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if valExp == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %v for %v, got %X", nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", idx, valBytes) + break + } + valGot := Bytes2Int64(valBytes) + if valExp != valGot { + b.Errorf("Expected %v for %v, got %v", valExp, idx, valGot) + break + } + } + } + + } +} diff --git a/libs/db/memdb/batch.go b/libs/db/memdb/batch.go new file mode 100644 index 0000000000..999f3958bb --- /dev/null +++ b/libs/db/memdb/batch.go @@ -0,0 +1,98 @@ +package memdb + +import ( + "fmt" + + lldb "github.com/lazyledger/lazyledger-core/libs/db" +) + +// memDBBatch operations +type opType int + +const ( + opTypeSet opType = iota + 1 + opTypeDelete +) + +type operation struct { + opType + key []byte + value []byte +} + +// memDBBatch handles in-memory batching. +type memDBBatch struct { + db *MemDB + ops []operation +} + +var _ lldb.Batch = (*memDBBatch)(nil) + +// newMemDBBatch creates a new memDBBatch +func newMemDBBatch(db *MemDB) *memDBBatch { + return &memDBBatch{ + db: db, + ops: []operation{}, + } +} + +// Set implements Batch. +func (b *memDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return lldb.ErrKeyEmpty + } + if value == nil { + return lldb.ErrValueNil + } + if b.ops == nil { + return lldb.ErrBatchClosed + } + b.ops = append(b.ops, operation{opTypeSet, key, value}) + return nil +} + +// Delete implements Batch. +func (b *memDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return lldb.ErrKeyEmpty + } + if b.ops == nil { + return lldb.ErrBatchClosed + } + b.ops = append(b.ops, operation{opTypeDelete, key, nil}) + return nil +} + +// Write implements Batch. +func (b *memDBBatch) Write() error { + if b.ops == nil { + return lldb.ErrBatchClosed + } + b.db.mtx.Lock() + defer b.db.mtx.Unlock() + + for _, op := range b.ops { + switch op.opType { + case opTypeSet: + b.db.set(op.key, op.value) + case opTypeDelete: + b.db.delete(op.key) + default: + return fmt.Errorf("unknown operation type %v (%v)", op.opType, op) + } + } + + // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. + return b.Close() +} + +// WriteSync implements Batch. +func (b *memDBBatch) WriteSync() error { + return b.Write() +} + +// Close implements Batch. +func (b *memDBBatch) Close() error { + b.ops = nil + return nil +} diff --git a/libs/db/memdb/db.go b/libs/db/memdb/db.go new file mode 100644 index 0000000000..e13dafe796 --- /dev/null +++ b/libs/db/memdb/db.go @@ -0,0 +1,188 @@ +package memdb + +import ( + "bytes" + "fmt" + "sync" + + "github.com/google/btree" + + lldb "github.com/lazyledger/lazyledger-core/libs/db" +) + +const ( + // The approximate number of items and children per B-tree node. Tuned with benchmarks. + bTreeDegree = 32 +) + +// item is a btree.Item with byte slices as keys and values +type item struct { + key []byte + value []byte +} + +// Less implements btree.Item. +func (i *item) Less(other btree.Item) bool { + // this considers nil == []byte{}, but that's ok since we handle nil endpoints + // in iterators specially anyway + return bytes.Compare(i.key, other.(*item).key) == -1 +} + +// newKey creates a new key item. +func newKey(key []byte) *item { + return &item{key: key} +} + +// newPair creates a new pair item. +func newPair(key, value []byte) *item { + return &item{key: key, value: value} +} + +// MemDB is an in-memory database backend using a B-tree for storage. +// +// For performance reasons, all given and returned keys and values are pointers to the in-memory +// database, so modifying them will cause the stored values to be modified as well. All DB methods +// already specify that keys and values should be considered read-only, but this is especially +// important with MemDB. +type MemDB struct { + mtx sync.RWMutex + btree *btree.BTree +} + +var _ lldb.DB = (*MemDB)(nil) + +// NewDB creates a new in-memory database. +func NewDB() *MemDB { + database := &MemDB{ + btree: btree.New(bTreeDegree), + } + return database +} + +// Get implements DB. +func (db *MemDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, lldb.ErrKeyEmpty + } + db.mtx.RLock() + defer db.mtx.RUnlock() + + i := db.btree.Get(newKey(key)) + if i != nil { + return i.(*item).value, nil + } + return nil, nil +} + +// Has implements DB. +func (db *MemDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, lldb.ErrKeyEmpty + } + db.mtx.RLock() + defer db.mtx.RUnlock() + + return db.btree.Has(newKey(key)), nil +} + +// Set implements DB. +func (db *MemDB) Set(key []byte, value []byte) error { + if len(key) == 0 { + return lldb.ErrKeyEmpty + } + if value == nil { + return lldb.ErrValueNil + } + db.mtx.Lock() + defer db.mtx.Unlock() + + db.set(key, value) + return nil +} + +// set sets a value without locking the mutex. +func (db *MemDB) set(key []byte, value []byte) { + db.btree.ReplaceOrInsert(newPair(key, value)) +} + +// SetSync implements DB. +func (db *MemDB) SetSync(key []byte, value []byte) error { + return db.Set(key, value) +} + +// Delete implements DB. +func (db *MemDB) Delete(key []byte) error { + if len(key) == 0 { + return lldb.ErrKeyEmpty + } + db.mtx.Lock() + defer db.mtx.Unlock() + + db.delete(key) + return nil +} + +// delete deletes a key without locking the mutex. +func (db *MemDB) delete(key []byte) { + db.btree.Delete(newKey(key)) +} + +// DeleteSync implements DB. +func (db *MemDB) DeleteSync(key []byte) error { + return db.Delete(key) +} + +// Close implements DB. +func (db *MemDB) Close() error { + // Close is a noop since for an in-memory database, we don't have a destination to flush + // contents to nor do we want any data loss on invoking Close(). + // See the discussion in https://github.com/tendermint/tendermint/libs/pull/56 + return nil +} + +// Print implements DB. +func (db *MemDB) Print() error { + db.mtx.RLock() + defer db.mtx.RUnlock() + + db.btree.Ascend(func(i btree.Item) bool { + item := i.(*item) + fmt.Printf("[%X]:\t[%X]\n", item.key, item.value) + return true + }) + return nil +} + +// Stats implements DB. +func (db *MemDB) Stats() map[string]string { + db.mtx.RLock() + defer db.mtx.RUnlock() + + stats := make(map[string]string) + stats["database.type"] = "memDB" + stats["database.size"] = fmt.Sprintf("%d", db.btree.Len()) + return stats +} + +// NewBatch implements DB. +func (db *MemDB) NewBatch() lldb.Batch { + return newMemDBBatch(db) +} + +// Iterator implements DB. +// Takes out a read-lock on the database until the iterator is closed. +func (db *MemDB) Iterator(start, end []byte) (lldb.Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, lldb.ErrKeyEmpty + } + return newMemDBIterator(db, start, end, false), nil +} + +// ReverseIterator implements DB. +// Takes out a read-lock on the database until the iterator is closed. +func (db *MemDB) ReverseIterator(start, end []byte) (lldb.Iterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, lldb.ErrKeyEmpty + } + return newMemDBIterator(db, start, end, true), nil +} diff --git a/libs/db/memdb/db_test.go b/libs/db/memdb/db_test.go new file mode 100644 index 0000000000..241b1b4dfa --- /dev/null +++ b/libs/db/memdb/db_test.go @@ -0,0 +1,28 @@ +package memdb + +import ( + "testing" + + "github.com/lazyledger/lazyledger-core/libs/db/internal/dbtest" +) + +func BenchmarkMemDBRangeScans1M(b *testing.B) { + db := NewDB() + defer db.Close() + + dbtest.BenchmarkRangeScans(b, db, int64(1e6)) +} + +func BenchmarkMemDBRangeScans10M(b *testing.B) { + db := NewDB() + defer db.Close() + + dbtest.BenchmarkRangeScans(b, db, int64(10e6)) +} + +func BenchmarkMemDBRandomReadsWrites(b *testing.B) { + db := NewDB() + defer db.Close() + + dbtest.BenchmarkRandomReadsWrites(b, db) +} diff --git a/libs/db/memdb/iterator.go b/libs/db/memdb/iterator.go new file mode 100644 index 0000000000..edacb57d96 --- /dev/null +++ b/libs/db/memdb/iterator.go @@ -0,0 +1,149 @@ +package memdb + +import ( + "bytes" + "context" + + "github.com/google/btree" + + lldb "github.com/lazyledger/lazyledger-core/libs/db" +) + +const ( + // Size of the channel buffer between traversal goroutine and iterator. Using an unbuffered + // channel causes two context switches per item sent, while buffering allows more work per + // context switch. Tuned with benchmarks. + chBufferSize = 64 +) + +// memDBIterator is a memDB iterator. +type memDBIterator struct { + ch <-chan *item + cancel context.CancelFunc + item *item + start []byte + end []byte +} + +var _ lldb.Iterator = (*memDBIterator)(nil) + +// newMemDBIterator creates a new memDBIterator. +func newMemDBIterator(db *MemDB, start []byte, end []byte, reverse bool) *memDBIterator { + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan *item, chBufferSize) + iter := &memDBIterator{ + ch: ch, + cancel: cancel, + start: start, + end: end, + } + + db.mtx.RLock() + go func() { + defer db.mtx.RUnlock() + // Because we use [start, end) for reverse ranges, while btree uses (start, end], we need + // the following variables to handle some reverse iteration conditions ourselves. + var ( + skipEqual []byte + abortLessThan []byte + ) + visitor := func(i btree.Item) bool { + item := i.(*item) + if skipEqual != nil && bytes.Equal(item.key, skipEqual) { + skipEqual = nil + return true + } + if abortLessThan != nil && bytes.Compare(item.key, abortLessThan) == -1 { + return false + } + select { + case <-ctx.Done(): + return false + case ch <- item: + return true + } + } + switch { + case start == nil && end == nil && !reverse: + db.btree.Ascend(visitor) + case start == nil && end == nil && reverse: + db.btree.Descend(visitor) + case end == nil && !reverse: + // must handle this specially, since nil is considered less than anything else + db.btree.AscendGreaterOrEqual(newKey(start), visitor) + case !reverse: + db.btree.AscendRange(newKey(start), newKey(end), visitor) + case end == nil: + // abort after start, since we use [start, end) while btree uses (start, end] + abortLessThan = start + db.btree.Descend(visitor) + default: + // skip end and abort after start, since we use [start, end) while btree uses (start, end] + skipEqual = end + abortLessThan = start + db.btree.DescendLessOrEqual(newKey(end), visitor) + } + close(ch) + }() + + // prime the iterator with the first value, if any + if item, ok := <-ch; ok { + iter.item = item + } + + return iter +} + +// Close implements Iterator. +func (i *memDBIterator) Close() error { + i.cancel() + for range i.ch { // drain channel + } + i.item = nil + return nil +} + +// Domain implements Iterator. +func (i *memDBIterator) Domain() ([]byte, []byte) { + return i.start, i.end +} + +// Valid implements Iterator. +func (i *memDBIterator) Valid() bool { + return i.item != nil +} + +// Next implements Iterator. +func (i *memDBIterator) Next() { + i.assertIsValid() + item, ok := <-i.ch + switch { + case ok: + i.item = item + default: + i.item = nil + } +} + +// Error implements Iterator. +func (i *memDBIterator) Error() error { + return nil // famous last words +} + +// Key implements Iterator. +func (i *memDBIterator) Key() []byte { + i.assertIsValid() + return i.item.key +} + +// Value implements Iterator. +func (i *memDBIterator) Value() []byte { + i.assertIsValid() + return i.item.value +} + +func (i *memDBIterator) assertIsValid() { + if !i.Valid() { + panic("iterator is invalid") + } +} diff --git a/libs/db/prefixdb_iterator.go b/libs/db/prefixdb_iterator.go new file mode 100644 index 0000000000..6db1f881bd --- /dev/null +++ b/libs/db/prefixdb_iterator.go @@ -0,0 +1,19 @@ +package db + +// IteratePrefix is a convenience function for iterating over a key domain +// restricted by prefix. +func IteratePrefix(db DB, prefix []byte) (Iterator, error) { + var start, end []byte + if len(prefix) == 0 { + start = nil + end = nil + } else { + start = cp(prefix) + end = increment(prefix) + } + itr, err := db.Iterator(start, end) + if err != nil { + return nil, err + } + return itr, nil +} diff --git a/libs/db/types.go b/libs/db/types.go new file mode 100644 index 0000000000..43d0f4e06f --- /dev/null +++ b/libs/db/types.go @@ -0,0 +1,147 @@ +package db + +import "errors" + +var ( + // ErrBatchClosed is returned when a closed or written batch is used. + ErrBatchClosed = errors.New("batch has been written or closed") + + // ErrKeyEmpty is returned when attempting to use an empty or nil key. + ErrKeyEmpty = errors.New("key cannot be empty") + + // ErrValueNil is returned when attempting to set a nil value. + ErrValueNil = errors.New("value cannot be nil") +) + +// DB is the main interface for all database backends. DBs are concurrency-safe. Callers must call +// Close on the database when done. +// +// Keys cannot be nil or empty, while values cannot be nil. Keys and values should be considered +// read-only, both when returned and when given, and must be copied before they are modified. +type DB interface { + // Get fetches the value of the given key, or nil if it does not exist. + // CONTRACT: key, value readonly []byte + Get([]byte) ([]byte, error) + + // Has checks if a key exists. + // CONTRACT: key, value readonly []byte + Has(key []byte) (bool, error) + + // Set sets the value for the given key, replacing it if it already exists. + // CONTRACT: key, value readonly []byte + Set([]byte, []byte) error + + // SetSync sets the value for the given key, and flushes it to storage before returning. + SetSync([]byte, []byte) error + + // Delete deletes the key, or does nothing if the key does not exist. + // CONTRACT: key readonly []byte + Delete([]byte) error + + // DeleteSync deletes the key, and flushes the delete to storage before returning. + DeleteSync([]byte) error + + // Iterator returns an iterator over a domain of keys, in ascending order. The caller must call + // Close when done. End is exclusive, and start must be less than end. A nil start iterates + // from the first key, and a nil end iterates to the last key (inclusive). Empty keys are not + // valid. + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + Iterator(start, end []byte) (Iterator, error) + + // ReverseIterator returns an iterator over a domain of keys, in descending order. The caller + // must call Close when done. End is exclusive, and start must be less than end. A nil end + // iterates from the last key (inclusive), and a nil start iterates to the first key (inclusive). + // Empty keys are not valid. + // CONTRACT: No writes may happen within a domain while an iterator exists over it. + // CONTRACT: start, end readonly []byte + ReverseIterator(start, end []byte) (Iterator, error) + + // Close closes the database connection. + Close() error + + // NewBatch creates a batch for atomic updates. The caller must call Batch.Close. + NewBatch() Batch + + // Print is used for debugging. + Print() error + + // Stats returns a map of property values for all keys and the size of the cache. + Stats() map[string]string +} + +// Batch represents a group of writes. They may or may not be written atomically depending on the +// backend. Callers must call Close on the batch when done. +// +// As with DB, given keys and values should be considered read-only, and must not be modified after +// passing them to the batch. +type Batch interface { + // Set sets a key/value pair. + // CONTRACT: key, value readonly []byte + Set(key, value []byte) error + + // Delete deletes a key/value pair. + // CONTRACT: key readonly []byte + Delete(key []byte) error + + // Write writes the batch, possibly without flushing to disk. Only Close() can be called after, + // other methods will error. + Write() error + + // WriteSync writes the batch and flushes it to disk. Only Close() can be called after, other + // methods will error. + WriteSync() error + + // Close closes the batch. It is idempotent, but calls to other methods afterwards will error. + Close() error +} + +// Iterator represents an iterator over a domain of keys. Callers must call Close when done. +// No writes can happen to a domain while there exists an iterator over it, some backends may take +// out database locks to ensure this will not happen. +// +// Callers must make sure the iterator is valid before calling any methods on it, otherwise +// these methods will panic. This is in part caused by most backend databases using this convention. +// +// As with DB, keys and values should be considered read-only, and must be copied before they are +// modified. +// +// Typical usage: +// +// var itr Iterator = ... +// defer itr.Close() +// +// for ; itr.Valid(); itr.Next() { +// k, v := itr.Key(); itr.Value() +// ... +// } +// if err := itr.Error(); err != nil { +// ... +// } +type Iterator interface { + // Domain returns the start (inclusive) and end (exclusive) limits of the iterator. + // CONTRACT: start, end readonly []byte + Domain() (start []byte, end []byte) + + // Valid returns whether the current iterator is valid. Once invalid, the Iterator remains + // invalid forever. + Valid() bool + + // Next moves the iterator to the next key in the database, as defined by order of iteration. + // If Valid returns false, this method will panic. + Next() + + // Key returns the key at the current position. Panics if the iterator is invalid. + // CONTRACT: key readonly []byte + Key() (key []byte) + + // Value returns the value at the current position. Panics if the iterator is invalid. + // CONTRACT: value readonly []byte + Value() (value []byte) + + // Error returns the last error encountered by the iterator, if any. + Error() error + + // Close closes the iterator, relasing any allocated resources. + Close() error +} diff --git a/libs/db/util.go b/libs/db/util.go new file mode 100644 index 0000000000..753970dd3c --- /dev/null +++ b/libs/db/util.go @@ -0,0 +1,30 @@ +package db + +func cp(bz []byte) (ret []byte) { + ret = make([]byte, len(bz)) + copy(ret, bz) + return ret +} + +// Returns a slice of the same length (big endian) +// except incremented by one. +// Returns nil on overflow (e.g. if bz bytes are all 0xFF) +// CONTRACT: len(bz) > 0 +func increment(bz []byte) (ret []byte) { + if len(bz) == 0 { + panic("cpIncr expects non-zero bz length") + } + ret = cp(bz) + for i := len(bz) - 1; i >= 0; i-- { + if ret[i] < byte(0xFF) { + ret[i]++ + return + } + ret[i] = byte(0x00) + if i == 0 { + // Overflow + return nil + } + } + return nil +} diff --git a/light/client_benchmark_test.go b/light/client_benchmark_test.go index ed514707ac..f7936b5d06 100644 --- a/light/client_benchmark_test.go +++ b/light/client_benchmark_test.go @@ -5,8 +5,7 @@ import ( "testing" "time" - dbm "github.com/tendermint/tm-db" - + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/light" "github.com/lazyledger/lazyledger-core/light/provider" @@ -37,7 +36,7 @@ func BenchmarkSequence(b *testing.B) { }, benchmarkFullNode, []provider.Provider{benchmarkFullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.SequentialVerification(), ) @@ -65,7 +64,7 @@ func BenchmarkBisection(b *testing.B) { }, benchmarkFullNode, []provider.Provider{benchmarkFullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) if err != nil { @@ -93,7 +92,7 @@ func BenchmarkBackwards(b *testing.B) { }, benchmarkFullNode, []provider.Provider{benchmarkFullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) if err != nil { diff --git a/light/client_test.go b/light/client_test.go index fe0ae354f6..2bd05b8f45 100644 --- a/light/client_test.go +++ b/light/client_test.go @@ -9,8 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/light" "github.com/lazyledger/lazyledger-core/light/provider" @@ -231,7 +230,7 @@ func TestClient_SequentialVerification(t *testing.T) { tc.otherHeaders, tc.vals, )}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.SequentialVerification(), light.Logger(log.TestingLogger()), ) @@ -356,7 +355,7 @@ func TestClient_SkippingVerification(t *testing.T) { tc.otherHeaders, tc.vals, )}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.SkippingVerification(light.DefaultTrustLevel), light.Logger(log.TestingLogger()), ) @@ -394,7 +393,7 @@ func TestClientLargeBisectionVerification(t *testing.T) { }, veryLargeFullNode, []provider.Provider{veryLargeFullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.SkippingVerification(light.DefaultTrustLevel), ) require.NoError(t, err) @@ -416,7 +415,7 @@ func TestClientBisectionBetweenTrustedHeaders(t *testing.T) { }, fullNode, []provider.Provider{fullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.SkippingVerification(light.DefaultTrustLevel), ) require.NoError(t, err) @@ -440,7 +439,7 @@ func TestClient_Cleanup(t *testing.T) { trustOptions, fullNode, []provider.Provider{fullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) require.NoError(t, err) @@ -460,7 +459,7 @@ func TestClient_Cleanup(t *testing.T) { func TestClientRestoresTrustedHeaderAfterStartup1(t *testing.T) { // 1. options.Hash == trustedHeader.Hash { - trustedStore := dbs.New(dbm.NewMemDB(), chainID) + trustedStore := dbs.New(memdb.NewDB(), chainID) err := trustedStore.SaveLightBlock(l1) require.NoError(t, err) @@ -484,7 +483,7 @@ func TestClientRestoresTrustedHeaderAfterStartup1(t *testing.T) { // 2. options.Hash != trustedHeader.Hash { - trustedStore := dbs.New(dbm.NewMemDB(), chainID) + trustedStore := dbs.New(memdb.NewDB(), chainID) err := trustedStore.SaveLightBlock(l1) require.NoError(t, err) @@ -529,7 +528,7 @@ func TestClientRestoresTrustedHeaderAfterStartup1(t *testing.T) { func TestClientRestoresTrustedHeaderAfterStartup2(t *testing.T) { // 1. options.Hash == trustedHeader.Hash { - trustedStore := dbs.New(dbm.NewMemDB(), chainID) + trustedStore := dbs.New(memdb.NewDB(), chainID) err := trustedStore.SaveLightBlock(l1) require.NoError(t, err) @@ -559,7 +558,7 @@ func TestClientRestoresTrustedHeaderAfterStartup2(t *testing.T) { // 2. options.Hash != trustedHeader.Hash // This could happen if previous provider was lying to us. { - trustedStore := dbs.New(dbm.NewMemDB(), chainID) + trustedStore := dbs.New(memdb.NewDB(), chainID) err := trustedStore.SaveLightBlock(l1) require.NoError(t, err) @@ -606,7 +605,7 @@ func TestClientRestoresTrustedHeaderAfterStartup3(t *testing.T) { // 1. options.Hash == trustedHeader.Hash { // load the first three headers into the trusted store - trustedStore := dbs.New(dbm.NewMemDB(), chainID) + trustedStore := dbs.New(memdb.NewDB(), chainID) err := trustedStore.SaveLightBlock(l1) require.NoError(t, err) @@ -644,7 +643,7 @@ func TestClientRestoresTrustedHeaderAfterStartup3(t *testing.T) { // 2. options.Hash != trustedHeader.Hash // This could happen if previous provider was lying to us. { - trustedStore := dbs.New(dbm.NewMemDB(), chainID) + trustedStore := dbs.New(memdb.NewDB(), chainID) err := trustedStore.SaveLightBlock(l1) require.NoError(t, err) @@ -704,7 +703,7 @@ func TestClient_Update(t *testing.T) { trustOptions, fullNode, []provider.Provider{fullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) require.NoError(t, err) @@ -725,7 +724,7 @@ func TestClient_Concurrency(t *testing.T) { trustOptions, fullNode, []provider.Provider{fullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) require.NoError(t, err) @@ -766,7 +765,7 @@ func TestClientReplacesPrimaryWithWitnessIfPrimaryIsUnavailable(t *testing.T) { trustOptions, deadNode, []provider.Provider{fullNode, fullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.MaxRetryAttempts(1), ) @@ -792,7 +791,7 @@ func TestClient_BackwardsVerification(t *testing.T) { }, largeFullNode, []provider.Provider{largeFullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) require.NoError(t, err) @@ -874,7 +873,7 @@ func TestClient_BackwardsVerification(t *testing.T) { }, tc.provider, []provider.Provider{tc.provider}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) require.NoError(t, err, idx) @@ -887,7 +886,7 @@ func TestClient_BackwardsVerification(t *testing.T) { func TestClient_NewClientFromTrustedStore(t *testing.T) { // 1) Initiate DB and fill with a "trusted" header - db := dbs.New(dbm.NewMemDB(), chainID) + db := dbs.New(memdb.NewDB(), chainID) err := db.SaveLightBlock(l1) require.NoError(t, err) @@ -944,7 +943,7 @@ func TestClientRemovesWitnessIfItSendsUsIncorrectHeader(t *testing.T) { trustOptions, fullNode, []provider.Provider{badProvider1, badProvider2}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.MaxRetryAttempts(1), ) @@ -994,7 +993,7 @@ func TestClient_TrustedValidatorSet(t *testing.T) { trustOptions, fullNode, []provider.Provider{badValSetNode, fullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), ) require.NoError(t, err) @@ -1012,7 +1011,7 @@ func TestClientPrunesHeadersAndValidatorSets(t *testing.T) { trustOptions, fullNode, []provider.Provider{fullNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.PruningSize(1), ) @@ -1085,7 +1084,7 @@ func TestClientEnsureValidHeadersAndValSets(t *testing.T) { trustOptions, badNode, []provider.Provider{badNode, badNode}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.MaxRetryAttempts(1), ) require.NoError(t, err) diff --git a/light/detector_test.go b/light/detector_test.go index f19ea4d567..bdf7e5faee 100644 --- a/light/detector_test.go +++ b/light/detector_test.go @@ -7,8 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/light" "github.com/lazyledger/lazyledger-core/light/provider" @@ -54,7 +53,7 @@ func TestLightClientAttackEvidence_Lunatic(t *testing.T) { }, primary, []provider.Provider{witness}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.MaxRetryAttempts(1), ) @@ -136,7 +135,7 @@ func TestLightClientAttackEvidence_Equivocation(t *testing.T) { }, primary, []provider.Provider{witness}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.MaxRetryAttempts(1), verificationOption, @@ -191,7 +190,7 @@ func TestClientDivergentTraces1(t *testing.T) { }, primary, []provider.Provider{witness}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.MaxRetryAttempts(1), ) @@ -215,7 +214,7 @@ func TestClientDivergentTraces2(t *testing.T) { }, primary, []provider.Provider{deadNode, deadNode, primary}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.MaxRetryAttempts(1), ) @@ -250,7 +249,7 @@ func TestClientDivergentTraces3(t *testing.T) { }, primary, []provider.Provider{witness}, - dbs.New(dbm.NewMemDB(), chainID), + dbs.New(memdb.NewDB(), chainID), light.Logger(log.TestingLogger()), light.MaxRetryAttempts(1), ) diff --git a/light/example_test.go b/light/example_test.go index 745cd2feb7..6aa600694a 100644 --- a/light/example_test.go +++ b/light/example_test.go @@ -9,9 +9,8 @@ import ( "testing" "time" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/abci/example/kvstore" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/light" "github.com/lazyledger/lazyledger-core/light/provider" @@ -46,7 +45,7 @@ func ExampleClient_Update() { stdlog.Fatal(err) } - db, err := dbm.NewGoLevelDB("light-client-db", dbDir) + db, err := badgerdb.NewDB("light-client-db", dbDir) if err != nil { stdlog.Fatal(err) } @@ -114,7 +113,7 @@ func ExampleClient_VerifyLightBlockAtHeight() { stdlog.Fatal(err) } - db, err := dbm.NewGoLevelDB("light-client-db", dbDir) + db, err := badgerdb.NewDB("light-client-db", dbDir) if err != nil { stdlog.Fatal(err) } diff --git a/light/store/db/db.go b/light/store/db/db.go index de37a1c319..5eb9b3f721 100644 --- a/light/store/db/db.go +++ b/light/store/db/db.go @@ -6,8 +6,7 @@ import ( "regexp" "strconv" - dbm "github.com/tendermint/tm-db" - + dbm "github.com/lazyledger/lazyledger-core/libs/db" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" "github.com/lazyledger/lazyledger-core/light/store" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" diff --git a/light/store/db/db_test.go b/light/store/db/db_test.go index 5360baa913..4ecd8eb987 100644 --- a/light/store/db/db_test.go +++ b/light/store/db/db_test.go @@ -8,10 +8,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/tmhash" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" tmversion "github.com/lazyledger/lazyledger-core/proto/tendermint/version" "github.com/lazyledger/lazyledger-core/types" @@ -19,7 +18,7 @@ import ( ) func TestLast_FirstLightBlockHeight(t *testing.T) { - dbStore := New(dbm.NewMemDB(), "TestLast_FirstLightBlockHeight") + dbStore := New(memdb.NewDB(), "TestLast_FirstLightBlockHeight") // Empty store height, err := dbStore.LastLightBlockHeight() @@ -44,7 +43,7 @@ func TestLast_FirstLightBlockHeight(t *testing.T) { } func Test_SaveLightBlock(t *testing.T) { - dbStore := New(dbm.NewMemDB(), "Test_SaveLightBlockAndValidatorSet") + dbStore := New(memdb.NewDB(), "Test_SaveLightBlockAndValidatorSet") // Empty store h, err := dbStore.LightBlock(1) @@ -74,7 +73,7 @@ func Test_SaveLightBlock(t *testing.T) { } func Test_LightBlockBefore(t *testing.T) { - dbStore := New(dbm.NewMemDB(), "Test_LightBlockBefore") + dbStore := New(memdb.NewDB(), "Test_LightBlockBefore") assert.Panics(t, func() { _, _ = dbStore.LightBlockBefore(0) @@ -92,7 +91,7 @@ func Test_LightBlockBefore(t *testing.T) { } func Test_Prune(t *testing.T) { - dbStore := New(dbm.NewMemDB(), "Test_Prune") + dbStore := New(memdb.NewDB(), "Test_Prune") // Empty store assert.EqualValues(t, 0, dbStore.Size()) @@ -129,7 +128,7 @@ func Test_Prune(t *testing.T) { } func Test_Concurrency(t *testing.T) { - dbStore := New(dbm.NewMemDB(), "Test_Prune") + dbStore := New(memdb.NewDB(), "Test_Prune") var wg sync.WaitGroup for i := 1; i <= 100; i++ { diff --git a/mempool/clist_mempool_test.go b/mempool/clist_mempool_test.go index 723cf42fdc..6993b55b3c 100644 --- a/mempool/clist_mempool_test.go +++ b/mempool/clist_mempool_test.go @@ -546,7 +546,7 @@ func TestMempoolTxsBytes(t *testing.T) { func checksumIt(data []byte) string { h := sha256.New() - h.Write(data) //nolint: errcheck // ignore errcheck + h.Write(data) return fmt.Sprintf("%x", h.Sum(nil)) } diff --git a/node/node.go b/node/node.go index 80febbbea9..1e3b88f7fb 100644 --- a/node/node.go +++ b/node/node.go @@ -22,8 +22,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" bcv0 "github.com/lazyledger/lazyledger-core/blockchain/v0" bcv2 "github.com/lazyledger/lazyledger-core/blockchain/v2" @@ -31,6 +29,8 @@ import ( cs "github.com/lazyledger/lazyledger-core/consensus" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/evidence" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" tmjson "github.com/lazyledger/lazyledger-core/libs/json" "github.com/lazyledger/lazyledger-core/libs/log" tmpubsub "github.com/lazyledger/lazyledger-core/libs/pubsub" @@ -69,8 +69,7 @@ type DBProvider func(*DBContext) (dbm.DB, error) // DefaultDBProvider returns a database using the DBBackend and DBDir // specified in the ctx.Config. func DefaultDBProvider(ctx *DBContext) (dbm.DB, error) { - dbType := dbm.BackendType(ctx.Config.DBBackend) - return dbm.NewDB(ctx.ID, dbType, ctx.Config.DBDir()) + return badgerdb.NewDB(ctx.ID, ctx.Config.DBDir()) } // GenesisDocProvider returns a GenesisDoc. diff --git a/node/node_test.go b/node/node_test.go index ab2916819c..e29a82aacd 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -13,14 +13,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/abci/example/kvstore" cfg "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/ed25519" "github.com/lazyledger/lazyledger-core/crypto/tmhash" "github.com/lazyledger/lazyledger-core/evidence" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" mempl "github.com/lazyledger/lazyledger-core/mempool" @@ -257,8 +257,8 @@ func TestCreateProposalBlock(t *testing.T) { mempool.SetLogger(logger) // Make EvidencePool - evidenceDB := dbm.NewMemDB() - blockStore := store.NewBlockStore(dbm.NewMemDB()) + evidenceDB := memdb.NewDB() + blockStore := store.NewBlockStore(memdb.NewDB()) evidencePool, err := evidence.NewPool(evidenceDB, stateStore, blockStore) require.NoError(t, err) evidencePool.SetLogger(logger) @@ -551,7 +551,7 @@ func state(nVals int, height int64) (sm.State, dbm.DB, []types.PrivValidator) { }) // save validators to db for 2 heights - stateDB := dbm.NewMemDB() + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) if err := stateStore.Save(s); err != nil { panic(err) diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 8feed382e1..10101a9060 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -941,6 +941,6 @@ func (a *addrBook) hash(b []byte) ([]byte, error) { if err != nil { return nil, err } - hasher.Write(b) //nolint:errcheck // ignore error + hasher.Write(b) return hasher.Sum(nil), nil } diff --git a/p2p/trust/store.go b/p2p/trust/store.go index de0f9c483a..75c54249df 100644 --- a/p2p/trust/store.go +++ b/p2p/trust/store.go @@ -8,8 +8,7 @@ import ( "fmt" "time" - dbm "github.com/tendermint/tm-db" - + dbm "github.com/lazyledger/lazyledger-core/libs/db" "github.com/lazyledger/lazyledger-core/libs/service" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" ) diff --git a/p2p/trust/store_test.go b/p2p/trust/store_test.go index c3a8d322d5..b54b9b640f 100644 --- a/p2p/trust/store_test.go +++ b/p2p/trust/store_test.go @@ -9,15 +9,16 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" ) func TestTrustMetricStoreSaveLoad(t *testing.T) { dir := t.TempDir() - historyDB, err := dbm.NewDB("trusthistory", "goleveldb", dir) + historyDB, err := badgerdb.NewDB("trusthistory", dir) require.NoError(t, err) // 0 peers saved @@ -80,8 +81,7 @@ func TestTrustMetricStoreSaveLoad(t *testing.T) { } func TestTrustMetricStoreConfig(t *testing.T) { - historyDB, err := dbm.NewDB("", "memdb", "") - require.NoError(t, err) + historyDB := memdb.NewDB() config := MetricConfig{ ProportionalWeight: 0.5, @@ -91,7 +91,7 @@ func TestTrustMetricStoreConfig(t *testing.T) { // Create a store with custom config store := NewTrustMetricStore(historyDB, config) store.SetLogger(log.TestingLogger()) - err = store.Start() + err := store.Start() require.NoError(t, err) // Have the store make us a metric with the config @@ -105,12 +105,11 @@ func TestTrustMetricStoreConfig(t *testing.T) { } func TestTrustMetricStoreLookup(t *testing.T) { - historyDB, err := dbm.NewDB("", "memdb", "") - require.NoError(t, err) + historyDB := memdb.NewDB() store := NewTrustMetricStore(historyDB, DefaultConfig()) store.SetLogger(log.TestingLogger()) - err = store.Start() + err := store.Start() require.NoError(t, err) // Create 100 peers in the trust metric store @@ -128,12 +127,11 @@ func TestTrustMetricStoreLookup(t *testing.T) { } func TestTrustMetricStorePeerScore(t *testing.T) { - historyDB, err := dbm.NewDB("", "memdb", "") - require.NoError(t, err) + historyDB := memdb.NewDB() store := NewTrustMetricStore(historyDB, DefaultConfig()) store.SetLogger(log.TestingLogger()) - err = store.Start() + err := store.Start() require.NoError(t, err) key := "TestKey" diff --git a/rpc/core/blocks_test.go b/rpc/core/blocks_test.go index d45b727981..996c83c43e 100644 --- a/rpc/core/blocks_test.go +++ b/rpc/core/blocks_test.go @@ -7,9 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" tmstate "github.com/lazyledger/lazyledger-core/proto/tendermint/state" ctypes "github.com/lazyledger/lazyledger-core/rpc/core/types" rpctypes "github.com/lazyledger/lazyledger-core/rpc/jsonrpc/types" @@ -81,7 +80,7 @@ func TestBlockResults(t *testing.T) { } env = &Environment{} - env.StateStore = sm.NewStore(dbm.NewMemDB()) + env.StateStore = sm.NewStore(memdb.NewDB()) err := env.StateStore.SaveABCIResponses(100, results) require.NoError(t, err) env.BlockStore = mockBlockStore{height: 100} diff --git a/state/export_test.go b/state/export_test.go index 0d64c67039..54e50ec125 100644 --- a/state/export_test.go +++ b/state/export_test.go @@ -1,9 +1,8 @@ package state import ( - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" + dbm "github.com/lazyledger/lazyledger-core/libs/db" tmstate "github.com/lazyledger/lazyledger-core/proto/tendermint/state" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" "github.com/lazyledger/lazyledger-core/types" diff --git a/state/helpers_test.go b/state/helpers_test.go index 4f8688122e..32a3c78cf8 100644 --- a/state/helpers_test.go +++ b/state/helpers_test.go @@ -5,11 +5,11 @@ import ( "fmt" "time" - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/ed25519" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" tmstate "github.com/lazyledger/lazyledger-core/proto/tendermint/state" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" @@ -122,7 +122,7 @@ func makeState(nVals, height int) (sm.State, dbm.DB, map[string]types.PrivValida AppHash: nil, }) - stateDB := dbm.NewMemDB() + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) if err := stateStore.Save(s); err != nil { panic(err) diff --git a/state/state_test.go b/state/state_test.go index f6a3de0e27..d78ec1ef34 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -11,12 +11,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" cfg "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto/ed25519" cryptoenc "github.com/lazyledger/lazyledger-core/crypto/encoding" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" tmstate "github.com/lazyledger/lazyledger-core/proto/tendermint/state" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" @@ -27,8 +27,7 @@ import ( // setupTestCase does setup common to all test cases. func setupTestCase(t *testing.T) (func(t *testing.T), dbm.DB, sm.State) { config := cfg.ResetTestRoot("state_") - dbType := dbm.BackendType(config.DBBackend) - stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) + stateDB, err := badgerdb.NewDB("state", config.DBDir()) stateStore := sm.NewStore(stateDB) require.NoError(t, err) state, err := stateStore.LoadFromDBOrGenesisFile(config.GenesisFile()) diff --git a/state/store.go b/state/store.go index 9e9c3f1916..24439b1ace 100644 --- a/state/store.go +++ b/state/store.go @@ -5,9 +5,9 @@ import ( "fmt" "github.com/gogo/protobuf/proto" - dbm "github.com/tendermint/tm-db" abci "github.com/lazyledger/lazyledger-core/abci/types" + dbm "github.com/lazyledger/lazyledger-core/libs/db" tmmath "github.com/lazyledger/lazyledger-core/libs/math" tmos "github.com/lazyledger/lazyledger-core/libs/os" tmstate "github.com/lazyledger/lazyledger-core/proto/tendermint/state" diff --git a/state/store_test.go b/state/store_test.go index e225235daf..4e80cb3c63 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -8,12 +8,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" cfg "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto" "github.com/lazyledger/lazyledger-core/crypto/ed25519" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" tmstate "github.com/lazyledger/lazyledger-core/proto/tendermint/state" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" @@ -22,7 +22,7 @@ import ( ) func TestStoreLoadValidators(t *testing.T) { - stateDB := dbm.NewMemDB() + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) val, _ := types.RandValidator(true, 10) vals := types.NewValidatorSet([]*types.Validator{val}) @@ -51,8 +51,7 @@ func BenchmarkLoadValidators(b *testing.B) { config := cfg.ResetTestRoot("state_") defer os.RemoveAll(config.RootDir) - dbType := dbm.BackendType(config.DBBackend) - stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) + stateDB, err := badgerdb.NewDB("state", config.DBDir()) require.NoError(b, err) stateStore := sm.NewStore(stateDB) state, err := stateStore.LoadFromDBOrGenesisFile(config.GenesisFile()) @@ -106,7 +105,7 @@ func TestPruneStates(t *testing.T) { for name, tc := range testcases { tc := tc t.Run(name, func(t *testing.T) { - db := dbm.NewMemDB() + db := memdb.NewDB() stateStore := sm.NewStore(db) pk := ed25519.GenPrivKey().PubKey() diff --git a/state/tx_filter_test.go b/state/tx_filter_test.go index 2a2ab23b47..91645f27e5 100644 --- a/state/tx_filter_test.go +++ b/state/tx_filter_test.go @@ -1,14 +1,12 @@ package state_test import ( - "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - + "github.com/lazyledger/lazyledger-core/libs/db/memdb" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" sm "github.com/lazyledger/lazyledger-core/state" "github.com/lazyledger/lazyledger-core/types" @@ -31,8 +29,7 @@ func TestTxFilter(t *testing.T) { } for i, tc := range testCases { - stateDB, err := dbm.NewDB("state", "memdb", os.TempDir()) - require.NoError(t, err) + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc) require.NoError(t, err) diff --git a/state/txindex/indexer_service_test.go b/state/txindex/indexer_service_test.go index dea51814f7..0cd469fb53 100644 --- a/state/txindex/indexer_service_test.go +++ b/state/txindex/indexer_service_test.go @@ -7,9 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - db "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" "github.com/lazyledger/lazyledger-core/state/txindex" "github.com/lazyledger/lazyledger-core/state/txindex/kv" @@ -29,7 +28,7 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { }) // tx indexer - store := db.NewMemDB() + store := memdb.NewDB() txIndexer := kv.NewTxIndex(store) service := txindex.NewIndexerService(txIndexer, eventBus) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index c351f562b4..3b5c015bf6 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -10,9 +10,9 @@ import ( "time" "github.com/gogo/protobuf/proto" - dbm "github.com/tendermint/tm-db" abci "github.com/lazyledger/lazyledger-core/abci/types" + dbm "github.com/lazyledger/lazyledger-core/libs/db" "github.com/lazyledger/lazyledger-core/libs/pubsub/query" "github.com/lazyledger/lazyledger-core/state/txindex" "github.com/lazyledger/lazyledger-core/types" diff --git a/state/txindex/kv/kv_bench_test.go b/state/txindex/kv/kv_bench_test.go index 6c9b1e1b05..98cae930bd 100644 --- a/state/txindex/kv/kv_bench_test.go +++ b/state/txindex/kv/kv_bench_test.go @@ -7,9 +7,8 @@ import ( "io/ioutil" "testing" - dbm "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" "github.com/lazyledger/lazyledger-core/libs/pubsub/query" "github.com/lazyledger/lazyledger-core/types" ) @@ -20,7 +19,7 @@ func BenchmarkTxSearch(b *testing.B) { b.Errorf("failed to create temporary directory: %s", err) } - db, err := dbm.NewGoLevelDB("benchmark_tx_search_test", dbDir) + db, err := badgerdb.NewDB("benchmark_tx_search_test", dbDir) if err != nil { b.Errorf("failed to create database: %s", err) } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index c233333c1d..ec59face4d 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -11,9 +11,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - db "github.com/tendermint/tm-db" - abci "github.com/lazyledger/lazyledger-core/abci/types" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/pubsub/query" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" "github.com/lazyledger/lazyledger-core/state/txindex" @@ -21,7 +21,7 @@ import ( ) func TestTxIndex(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndex(memdb.NewDB()) tx := types.Tx("HELLO WORLD") txResult := &abci.TxResult{ @@ -67,7 +67,7 @@ func TestTxIndex(t *testing.T) { } func TestTxSearch(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndex(memdb.NewDB()) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1"), Index: true}}}, @@ -141,7 +141,7 @@ func TestTxSearch(t *testing.T) { } func TestTxSearchWithCancelation(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndex(memdb.NewDB()) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1"), Index: true}}}, @@ -159,7 +159,7 @@ func TestTxSearchWithCancelation(t *testing.T) { } func TestTxSearchDeprecatedIndexing(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndex(memdb.NewDB()) // index tx using events indexing (composite key) txResult1 := txResultWithEvents([]abci.Event{ @@ -238,7 +238,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { } func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndex(memdb.NewDB()) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1"), Index: true}}}, @@ -260,7 +260,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { } func TestTxSearchMultipleTxs(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndex(memdb.NewDB()) // indexed first, but bigger height (to test the order of transactions) txResult := txResultWithEvents([]abci.Event{ @@ -333,7 +333,7 @@ func benchmarkTxIndex(txsCount int64, b *testing.B) { require.NoError(b, err) defer os.RemoveAll(dir) - store, err := db.NewDB("tx_index", "goleveldb", dir) + store, err := badgerdb.NewDB("tx_index", dir) require.NoError(b, err) indexer := NewTxIndex(store) diff --git a/statesync/snapshots.go b/statesync/snapshots.go index 77d6b4d24f..97df705e52 100644 --- a/statesync/snapshots.go +++ b/statesync/snapshots.go @@ -33,9 +33,9 @@ type snapshot struct { func (s *snapshot) Key() snapshotKey { // Hash.Write() never returns an error. hasher := sha256.New() - hasher.Write([]byte(fmt.Sprintf("%v:%v:%v", s.Height, s.Format, s.Chunks))) //nolint:errcheck // ignore error - hasher.Write(s.Hash) //nolint:errcheck // ignore error - hasher.Write(s.Metadata) //nolint:errcheck // ignore error + hasher.Write([]byte(fmt.Sprintf("%v:%v:%v", s.Height, s.Format, s.Chunks))) + hasher.Write(s.Hash) + hasher.Write(s.Metadata) var key snapshotKey copy(key[:], hasher.Sum(nil)) return key diff --git a/statesync/stateprovider.go b/statesync/stateprovider.go index f56e0e0d6f..75f37aa081 100644 --- a/statesync/stateprovider.go +++ b/statesync/stateprovider.go @@ -6,8 +6,7 @@ import ( "strings" "time" - dbm "github.com/tendermint/tm-db" - + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" "github.com/lazyledger/lazyledger-core/light" @@ -72,7 +71,7 @@ func NewLightClientStateProvider( } lc, err := light.NewClient(ctx, chainID, trustOptions, providers[0], providers[1:], - lightdb.New(dbm.NewMemDB(), ""), light.Logger(logger), light.MaxRetryAttempts(5)) + lightdb.New(memdb.NewDB(), ""), light.Logger(logger), light.MaxRetryAttempts(5)) if err != nil { return nil, err } diff --git a/store/store.go b/store/store.go index d03dab5bd4..fa5dca3421 100644 --- a/store/store.go +++ b/store/store.go @@ -5,8 +5,8 @@ import ( "strconv" "github.com/gogo/protobuf/proto" - dbm "github.com/tendermint/tm-db" + dbm "github.com/lazyledger/lazyledger-core/libs/db" tmsync "github.com/lazyledger/lazyledger-core/libs/sync" tmstore "github.com/lazyledger/lazyledger-core/proto/tendermint/store" tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types" diff --git a/store/store_test.go b/store/store_test.go index c7f9947924..1f45a018ec 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -13,10 +13,11 @@ import ( "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" cfg "github.com/lazyledger/lazyledger-core/config" "github.com/lazyledger/lazyledger-core/crypto" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" tmstore "github.com/lazyledger/lazyledger-core/proto/tendermint/store" @@ -63,10 +64,10 @@ func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Bl func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore, cleanupFunc) { config := cfg.ResetTestRoot("blockchain_reactor_test") - // blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB()) - // stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB()) - blockDB := dbm.NewMemDB() - stateDB := dbm.NewMemDB() + // blockDB := dbm.NewDebugDB("blockDB", memdb.NewDB()) + // stateDB := dbm.NewDebugDB("stateDB", memdb.NewDB()) + blockDB := memdb.NewDB() + stateDB := memdb.NewDB() stateStore := sm.NewStore(stateDB) state, err := stateStore.LoadFromDBOrGenesisFile(config.GenesisFile()) if err != nil { @@ -91,7 +92,7 @@ func TestLoadBlockStoreState(t *testing.T) { } for _, tc := range testCases { - db := dbm.NewMemDB() + db := memdb.NewDB() SaveBlockStoreState(tc.bss, db) retrBSJ := LoadBlockStoreState(db) assert.Equal(t, tc.want, retrBSJ, "expected the retrieved DBs to match: %s", tc.testName) @@ -99,7 +100,7 @@ func TestLoadBlockStoreState(t *testing.T) { } func TestNewBlockStore(t *testing.T) { - db := dbm.NewMemDB() + db := memdb.NewDB() bss := tmstore.BlockStoreState{Base: 100, Height: 10000} bz, _ := proto.Marshal(&bss) err := db.Set(blockStoreKey, bz) @@ -136,7 +137,7 @@ func TestNewBlockStore(t *testing.T) { } func freshBlockStore() (*BlockStore, dbm.DB) { - db := dbm.NewMemDB() + db := memdb.NewDB() return NewBlockStore(db), db } @@ -376,10 +377,10 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { func TestLoadBaseMeta(t *testing.T) { config := cfg.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) - stateStore := sm.NewStore(dbm.NewMemDB()) + stateStore := sm.NewStore(memdb.NewDB()) state, err := stateStore.LoadFromDBOrGenesisFile(config.GenesisFile()) require.NoError(t, err) - bs := NewBlockStore(dbm.NewMemDB()) + bs := NewBlockStore(memdb.NewDB()) for h := int64(1); h <= 10; h++ { block := makeBlock(h, state, new(types.Commit)) @@ -432,10 +433,10 @@ func TestLoadBlockPart(t *testing.T) { func TestPruneBlocks(t *testing.T) { config := cfg.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) - stateStore := sm.NewStore(dbm.NewMemDB()) + stateStore := sm.NewStore(memdb.NewDB()) state, err := stateStore.LoadFromDBOrGenesisFile(config.GenesisFile()) require.NoError(t, err) - db := dbm.NewMemDB() + db := memdb.NewDB() bs := NewBlockStore(db) assert.EqualValues(t, 0, bs.Base()) assert.EqualValues(t, 0, bs.Height()) diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 038798d5db..281e32af1e 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -27,9 +27,6 @@ var ( } // The following specify randomly chosen values for testnet nodes. - // TODO(ismail): do we want to stick to badgerdb? - // see also: https://github.com/lazyledger/lazyledger-core/issues/211 - nodeDatabases = uniformChoice{"badgerdb"} nodeABCIProtocols = uniformChoice{"builtin"} // we only care about builtin apps here nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp"} nodeFastSyncs = uniformChoice{"", "v0"} // disable v2 due to bugs @@ -195,7 +192,6 @@ func generateNode( node := e2e.ManifestNode{ Mode: string(mode), StartAt: startAt, - Database: nodeDatabases.Choose(r).(string), ABCIProtocol: nodeABCIProtocols.Choose(r).(string), PrivvalProtocol: nodePrivvalProtocols.Choose(r).(string), FastSync: nodeFastSyncs.Choose(r).(string), diff --git a/test/e2e/networks/ci.toml b/test/e2e/networks/ci.toml index 44dae0a05b..a8a4b97da5 100644 --- a/test/e2e/networks/ci.toml +++ b/test/e2e/networks/ci.toml @@ -40,7 +40,6 @@ misbehaviors = { 1018 = "double-prevote" } [node.validator02] seeds = ["seed02"] -database = "badgerdb" abci_protocol = "builtin" privval_protocol = "tcp" persist_interval = 0 @@ -48,7 +47,6 @@ perturb = ["restart"] [node.validator03] seeds = ["seed01"] -database = "badgerdb" privval_protocol = "unix" persist_interval = 3 retain_blocks = 3 @@ -56,14 +54,12 @@ perturb = ["kill"] [node.validator04] persistent_peers = ["validator01"] -database = "badgerdb" abci_protocol = "builtin" perturb = ["pause"] [node.validator05] start_at = 1005 # Becomes part of the validator set at 1010 seeds = ["seed02"] -database = "badgerdb" fast_sync = "v0" privval_protocol = "tcp" perturb = ["kill", "pause", "disconnect", "restart"] diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index 41db78137a..7ca48483e2 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -67,10 +67,6 @@ type ManifestNode struct { // this defaults to all other nodes in the network. PersistentPeers []string `toml:"persistent_peers"` - // TODO: remove this config flag completely -> only badgerdb allowed now anyways - // Database specifies the database backend. In LazyLedger we only test with "badgerdb". - Database string `toml:"database"` - // TODO: remove this config flag completely -> only builtin allowed now anyways // ABCIProtocol specifies the protocol used to communicate with the ABCI // application: "unix", "tcp", "grpc", or "builtin". Defaults to builtin. diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index 42f9f8a228..b286320bd9 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -74,7 +74,6 @@ type Node struct { StartAt int64 FastSync string StateSync bool - Database string ABCIProtocol Protocol PrivvalProtocol Protocol PersistInterval uint64 @@ -148,7 +147,6 @@ func LoadTestnet(file string) (*Testnet, error) { IP: ipGen.Next(), ProxyPort: proxyPortGen.Next(), Mode: ModeValidator, - Database: "badgerdb", ABCIProtocol: ProtocolBuiltin, PrivvalProtocol: ProtocolFile, StartAt: nodeManifest.StartAt, @@ -166,9 +164,6 @@ func LoadTestnet(file string) (*Testnet, error) { if nodeManifest.Mode != "" { node.Mode = Mode(nodeManifest.Mode) } - if nodeManifest.Database != "" { - node.Database = nodeManifest.Database - } if nodeManifest.ABCIProtocol != "" { node.ABCIProtocol = Protocol(nodeManifest.ABCIProtocol) } @@ -311,11 +306,6 @@ func (n Node) Validate(testnet Testnet) error { default: return fmt.Errorf("invalid fast sync setting %q", n.FastSync) } - switch n.Database { - case "badgerdb": - default: - return fmt.Errorf("invalid database setting %q", n.Database) - } switch n.ABCIProtocol { case ProtocolBuiltin, ProtocolUNIX, ProtocolTCP, ProtocolGRPC: default: diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 6e4599cc57..ca170f5d16 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -236,7 +236,6 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657" cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false)) cfg.P2P.AddrBookStrict = false - cfg.DBBackend = node.Database cfg.StateSync.DiscoveryTime = 5 * time.Second switch node.ABCIProtocol { diff --git a/test/maverick/consensus/replay_file.go b/test/maverick/consensus/replay_file.go index bf96abab16..96b782f44d 100644 --- a/test/maverick/consensus/replay_file.go +++ b/test/maverick/consensus/replay_file.go @@ -10,9 +10,8 @@ import ( "strconv" "strings" - dbm "github.com/tendermint/tm-db" - cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" "github.com/lazyledger/lazyledger-core/libs/log" tmos "github.com/lazyledger/lazyledger-core/libs/os" "github.com/lazyledger/lazyledger-core/proxy" @@ -284,16 +283,15 @@ func (pb *playback) replayConsoleLoop() int { // convenience for replay mode func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *State { - dbType := dbm.BackendType(config.DBBackend) // Get BlockStore - blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir()) + blockStoreDB, err := badgerdb.NewDB("blockstore", config.DBDir()) if err != nil { tmos.Exit(err.Error()) } blockStore := store.NewBlockStore(blockStoreDB) // Get State - stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) + stateDB, err := badgerdb.NewDB("state", config.DBDir()) if err != nil { tmos.Exit(err.Error()) } diff --git a/test/maverick/consensus/wal_generator.go b/test/maverick/consensus/wal_generator.go index e2a905c288..40e50f0468 100644 --- a/test/maverick/consensus/wal_generator.go +++ b/test/maverick/consensus/wal_generator.go @@ -9,10 +9,9 @@ import ( "testing" "time" - db "github.com/tendermint/tm-db" - "github.com/lazyledger/lazyledger-core/abci/example/kvstore" cfg "github.com/lazyledger/lazyledger-core/config" + "github.com/lazyledger/lazyledger-core/libs/db/memdb" "github.com/lazyledger/lazyledger-core/libs/log" tmrand "github.com/lazyledger/lazyledger-core/libs/rand" "github.com/lazyledger/lazyledger-core/privval" @@ -49,7 +48,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) { if err != nil { return fmt.Errorf("failed to read genesis file: %w", err) } - blockStoreDB := db.NewMemDB() + blockStoreDB := memdb.NewDB() stateDB := blockStoreDB stateStore := sm.NewStore(stateDB) state, err := sm.MakeGenesisState(genDoc) diff --git a/test/maverick/node/node.go b/test/maverick/node/node.go index 96857ab9bf..9ba03d581a 100644 --- a/test/maverick/node/node.go +++ b/test/maverick/node/node.go @@ -16,7 +16,8 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" - dbm "github.com/tendermint/tm-db" + dbm "github.com/lazyledger/lazyledger-core/libs/db" + "github.com/lazyledger/lazyledger-core/libs/db/badgerdb" abci "github.com/lazyledger/lazyledger-core/abci/types" bcv0 "github.com/lazyledger/lazyledger-core/blockchain/v0" @@ -94,8 +95,7 @@ type DBProvider func(*DBContext) (dbm.DB, error) // DefaultDBProvider returns a database using the DBBackend and DBDir // specified in the ctx.Config. func DefaultDBProvider(ctx *DBContext) (dbm.DB, error) { - dbType := dbm.BackendType(ctx.Config.DBBackend) - return dbm.NewDB(ctx.ID, dbType, ctx.Config.DBDir()) + return badgerdb.NewDB(ctx.ID, ctx.Config.DBDir()) } // GenesisDocProvider returns a GenesisDoc.