From 80d33b9914e74d53be61c0e7c39a454390bf62ed Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 31 Jan 2024 16:18:26 +0200 Subject: [PATCH] moved new db from config to factory --- factory/db.go | 30 +++++++++ factory/db_test.go | 63 +++++++++++++++++++ factory/storageUnit.go | 32 ++++++++++ factory/storageUnit_test.go | 105 ++++++++++++++++++++++++++++++++ storageUnit/nilStorer.go | 3 + storageUnit/storageunit.go | 27 -------- storageUnit/storageunit_test.go | 102 ------------------------------- 7 files changed, 233 insertions(+), 129 deletions(-) create mode 100644 factory/db_test.go create mode 100644 factory/storageUnit.go create mode 100644 factory/storageUnit_test.go diff --git a/factory/db.go b/factory/db.go index 7312cd2e..3dc5c20d 100644 --- a/factory/db.go +++ b/factory/db.go @@ -1 +1,31 @@ package factory + +import ( + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/leveldb" + "github.com/multiversx/mx-chain-storage-go/memorydb" + "github.com/multiversx/mx-chain-storage-go/types" +) + +// ArgDB is a structure that is used to create a new storage.Persister implementation +type ArgDB struct { + DBType common.DBType + Path string + BatchDelaySeconds int + MaxBatchSize int + MaxOpenFiles int +} + +// NewDB creates a new database from database config +func NewDB(argDB ArgDB) (types.Persister, error) { + switch argDB.DBType { + case common.LvlDB: + return leveldb.NewDB(argDB.Path, argDB.BatchDelaySeconds, argDB.MaxBatchSize, argDB.MaxOpenFiles) + case common.LvlDBSerial: + return leveldb.NewSerialDB(argDB.Path, argDB.BatchDelaySeconds, argDB.MaxBatchSize, argDB.MaxOpenFiles) + case common.MemoryDB: + return memorydb.New(), nil + default: + return nil, common.ErrNotSupportedDBType + } +} diff --git a/factory/db_test.go b/factory/db_test.go new file mode 100644 index 00000000..73a65719 --- /dev/null +++ b/factory/db_test.go @@ -0,0 +1,63 @@ +package factory_test + +import ( + "testing" + + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/factory" + "github.com/stretchr/testify/assert" +) + +func TestCreateDBFromConfWrongType(t *testing.T) { + t.Parallel() + + argsDB := factory.ArgDB{ + DBType: "NotLvlDB", + Path: "test", + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) +} + +func TestCreateDBFromConfWrongFileNameLvlDB(t *testing.T) { + if testing.Short() { + t.Skip("this is not a short test") + } + + path := "" + argsDB := factory.ArgDB{ + DBType: common.LvlDB, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + assert.NotNil(t, err, "error expected") + assert.Nil(t, persister, "persister expected to be nil, but got %s", persister) +} + +func TestCreateDBFromConfLvlDBOk(t *testing.T) { + t.Parallel() + + path := t.TempDir() + + argsDB := factory.ArgDB{ + DBType: common.LvlDB, + Path: path, + BatchDelaySeconds: 10, + MaxBatchSize: 10, + MaxOpenFiles: 10, + } + persister, err := factory.NewDB(argsDB) + assert.Nil(t, err, "no error expected") + assert.NotNil(t, persister, "valid persister expected but got nil") + + err = persister.Destroy() + assert.Nil(t, err, "no error expected destroying the persister") +} diff --git a/factory/storageUnit.go b/factory/storageUnit.go new file mode 100644 index 00000000..e9305414 --- /dev/null +++ b/factory/storageUnit.go @@ -0,0 +1,32 @@ +package factory + +import ( + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/storageUnit" +) + +// NewStorageUnitFromConf creates a new storage unit from a storage unit config +func NewStorageUnitFromConf(cacheConf common.CacheConfig, dbConf common.DBConfig) (*storageUnit.Unit, error) { + if dbConf.MaxBatchSize > int(cacheConf.Capacity) { + return nil, common.ErrCacheSizeIsLowerThanBatchSize + } + + cache, err := NewCache(cacheConf) + if err != nil { + return nil, err + } + + argDB := ArgDB{ + DBType: dbConf.Type, + Path: dbConf.FilePath, + BatchDelaySeconds: dbConf.BatchDelaySeconds, + MaxBatchSize: dbConf.MaxBatchSize, + MaxOpenFiles: dbConf.MaxOpenFiles, + } + db, err := NewDB(argDB) + if err != nil { + return nil, err + } + + return storageUnit.NewStorageUnit(cache, db) +} diff --git a/factory/storageUnit_test.go b/factory/storageUnit_test.go new file mode 100644 index 00000000..80bf30ce --- /dev/null +++ b/factory/storageUnit_test.go @@ -0,0 +1,105 @@ +package factory_test + +import ( + "testing" + + "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/factory" + "github.com/stretchr/testify/assert" +) + +func TestNewStorageUnitFromConf_WrongCacheSizeVsBatchSize(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + MaxBatchSize: 11, + BatchDelaySeconds: 1, + MaxOpenFiles: 10, + }, + ) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, storer, "storer expected to be nil but got %s", storer) +} + +func TestNewStorageUnitFromConf_WrongCacheConfig(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: "NotLRU", + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + BatchDelaySeconds: 1, + MaxBatchSize: 1, + MaxOpenFiles: 10, + }, + ) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, storer, "storer expected to be nil but got %s", storer) +} + +func TestNewStorageUnitFromConf_WrongDBConfig(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: "NotLvlDB", + }, + ) + + assert.NotNil(t, err, "error expected") + assert.Nil(t, storer, "storer expected to be nil but got %s", storer) +} + +func TestNewStorageUnitFromConf_LvlDBOk(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + MaxBatchSize: 1, + BatchDelaySeconds: 1, + MaxOpenFiles: 10, + }, + ) + + assert.Nil(t, err, "no error expected but got %s", err) + assert.NotNil(t, storer, "valid storer expected but got nil") + err = storer.DestroyUnit() + assert.Nil(t, err, "no error expected destroying the persister") +} + +func TestNewStorageUnitFromConf_ShouldWorkLvlDB(t *testing.T) { + t.Parallel() + + storer, err := factory.NewStorageUnitFromConf(common.CacheConfig{ + Capacity: 10, + Type: common.LRUCache, + }, common.DBConfig{ + FilePath: "Blocks", + Type: common.LvlDB, + BatchDelaySeconds: 1, + MaxBatchSize: 1, + MaxOpenFiles: 10, + }, + ) + + assert.Nil(t, err, "no error expected but got %s", err) + assert.NotNil(t, storer, "valid storer expected but got nil") + err = storer.DestroyUnit() + assert.Nil(t, err, "no error expected destroying the persister") +} diff --git a/storageUnit/nilStorer.go b/storageUnit/nilStorer.go index 2330b452..63fdcc3c 100644 --- a/storageUnit/nilStorer.go +++ b/storageUnit/nilStorer.go @@ -3,8 +3,11 @@ package storageUnit import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-storage-go/common" + "github.com/multiversx/mx-chain-storage-go/types" ) +var _ types.Storer = (*NilStorer)(nil) + // NilStorer resembles a disabled implementation of the Storer interface type NilStorer struct { } diff --git a/storageUnit/storageunit.go b/storageUnit/storageunit.go index a71501fd..cf01ad4b 100644 --- a/storageUnit/storageunit.go +++ b/storageUnit/storageunit.go @@ -9,7 +9,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data" logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-storage-go/common" - "github.com/multiversx/mx-chain-storage-go/factory" "github.com/multiversx/mx-chain-storage-go/types" ) @@ -50,32 +49,6 @@ func NewStorageUnit(c types.Cacher, p types.Persister) (*Unit, error) { return sUnit, nil } -// NewStorageUnitFromConf creates a new storage unit from a storage unit config -func NewStorageUnitFromConf(cacheConf common.CacheConfig, dbConf common.DBConfig, persisterFactory PersisterFactoryHandler) (*Unit, error) { - var cache types.Cacher - var db types.Persister - var err error - - // TODO: if there will be a differentiation between the creation or opening of a DB, the DB could be destroyed - // in case of a failure while creating (not opening). - - if dbConf.MaxBatchSize > int(cacheConf.Capacity) { - return nil, common.ErrCacheSizeIsLowerThanBatchSize - } - - cache, err = factory.NewCache(cacheConf) - if err != nil { - return nil, err - } - - db, err = persisterFactory.CreateWithRetries(dbConf.FilePath) - if err != nil { - return nil, err - } - - return NewStorageUnit(cache, db) -} - // Put adds data to both cache and persistence medium func (u *Unit) Put(key, data []byte) error { u.lock.Lock() diff --git a/storageUnit/storageunit_test.go b/storageUnit/storageunit_test.go index 6014a52b..dd219dce 100644 --- a/storageUnit/storageunit_test.go +++ b/storageUnit/storageunit_test.go @@ -7,7 +7,6 @@ import ( "github.com/multiversx/mx-chain-storage-go/lrucache" "github.com/multiversx/mx-chain-storage-go/memorydb" "github.com/multiversx/mx-chain-storage-go/storageUnit" - "github.com/multiversx/mx-chain-storage-go/testscommon" "github.com/stretchr/testify/assert" ) @@ -262,104 +261,3 @@ func TestDestroyUnitNoError(t *testing.T) { err := s.DestroyUnit() assert.Nil(t, err, "no error expected, but got %s", err) } - -func TestNewStorageUnit_FromConfWrongCacheSizeVsBatchSize(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - MaxBatchSize: 11, - BatchDelaySeconds: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 11, 1, 10), - ) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, storer, "storer expected to be nil but got %s", storer) -} - -func TestNewStorageUnit_FromConfWrongCacheConfig(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: "NotLRU", - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - BatchDelaySeconds: 1, - MaxBatchSize: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), - ) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, storer, "storer expected to be nil but got %s", storer) -} - -func TestNewStorageUnit_FromConfWrongDBConfig(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: "NotLvlDB", - }, - testscommon.NewPersisterFactoryHandlerMock("NotLvlDB", 0, 0, 0), - ) - - assert.NotNil(t, err, "error expected") - assert.Nil(t, storer, "storer expected to be nil but got %s", storer) -} - -func TestNewStorageUnit_FromConfLvlDBOk(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - MaxBatchSize: 1, - BatchDelaySeconds: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), - ) - - assert.Nil(t, err, "no error expected but got %s", err) - assert.NotNil(t, storer, "valid storer expected but got nil") - err = storer.DestroyUnit() - assert.Nil(t, err, "no error expected destroying the persister") -} - -func TestNewStorageUnit_ShouldWorkLvlDB(t *testing.T) { - t.Parallel() - - storer, err := storageUnit.NewStorageUnitFromConf(common.CacheConfig{ - Capacity: 10, - Type: common.LRUCache, - }, common.DBConfig{ - FilePath: "Blocks", - Type: common.LvlDB, - BatchDelaySeconds: 1, - MaxBatchSize: 1, - MaxOpenFiles: 10, - }, - testscommon.NewPersisterFactoryHandlerMock(common.LvlDB, 1, 1, 10), - ) - - assert.Nil(t, err, "no error expected but got %s", err) - assert.NotNil(t, storer, "valid storer expected but got nil") - err = storer.DestroyUnit() - assert.Nil(t, err, "no error expected destroying the persister") -}