From e9df3c2a3b43090405a7a5b5fce0bb18ea0dd619 Mon Sep 17 00:00:00 2001 From: dahn510 Date: Fri, 30 Aug 2024 17:21:32 -0700 Subject: [PATCH 1/9] adding flatfs for large file storage --- config/config.go | 23 ++++++++++++++++++++++- config/types.go | 16 +++++++++++++++- core/app.go | 18 +++++++++++++++++- file_system/types.go | 5 +++-- go.mod | 6 ++++-- go.sum | 9 +++++++++ ipfs/datastore.go | 16 ++++++++++++++++ ipfs/ipfs.go | 11 ++--------- 8 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 ipfs/datastore.go diff --git a/config/config.go b/config/config.go index 1a369e3..69a7614 100644 --- a/config/config.go +++ b/config/config.go @@ -1,11 +1,32 @@ package config import ( + "errors" "strings" "gopkg.in/yaml.v3" ) +func (c Config) Validate() error { + if c.DataDirectory == "" { + return errors.New("invalid data directory") + } + + switch c.DataStoreConfig.Backend { + case "flatfs": + case "badgerdb": + if c.DataStoreConfig.Directory != c.DataDirectory { + return errors.New("badger ds directory must be the same as data directory") + } + default: + return errors.New("invalid data store backend") + } + + return nil +} + +// ReadConfig parses data and returns Config. +// Error during parsing or an invalid configuration in the Config will return an error. func ReadConfig(data []byte) (*Config, error) { config := Config{} @@ -14,7 +35,7 @@ func ReadConfig(data []byte) (*Config, error) { return nil, err } - return &config, nil + return &config, config.Validate() } func (c Config) Export() ([]byte, error) { diff --git a/config/types.go b/config/types.go index 73649e6..a7f2b57 100644 --- a/config/types.go +++ b/config/types.go @@ -19,6 +19,7 @@ type Config struct { DataDirectory string `yaml:"data_directory"` APICfg APIConfig `yaml:"api_config"` ProofThreads int64 `yaml:"proof_threads"` + DataStoreConfig DataStoreConfig `yaml:"data_store_config"` } type StrayManagerConfig struct { @@ -33,6 +34,14 @@ type APIConfig struct { IPFSDomain string `yaml:"ipfs_domain"` } +type DataStoreConfig struct { + // *choosing badgerdb as data store will need to use the same directory + // for data directory + Directory string `yaml:"directory"` + // data store options: flatfs, badgerdb + Backend string `yaml:"backend"` +} + // LegacyWallet handles keys from earlier versions of storage providers. // v3 and earlier providers used private key to sign txs // and by design it can't derive mnemonic seed which made @@ -67,6 +76,10 @@ func DefaultConfig() *Config { IPFSDomain: "dns4/ipfs.example.com/tcp/4001", }, ProofThreads: 1000, + DataStoreConfig: DataStoreConfig{ + Directory: "$HOME/.sequoia/datastore", + Backend: "flatfs", + }, } } @@ -86,5 +99,6 @@ func (c Config) MarshalZerologObject(e *zerolog.Event) { Int64("APIPort", c.APICfg.Port). Int("APIIPFSPort", c.APICfg.IPFSPort). Str("APIIPFSDomain", c.APICfg.IPFSDomain). - Int64("ProofThreads", c.ProofThreads) + Int64("ProofThreads", c.ProofThreads). + Str("DatastoreBackend", c.DataStoreConfig.Backend) } diff --git a/core/app.go b/core/app.go index f92038c..2728945 100644 --- a/core/app.go +++ b/core/app.go @@ -10,6 +10,8 @@ import ( "time" "github.com/JackalLabs/sequoia/file_system" + "github.com/JackalLabs/sequoia/ipfs" + "github.com/ipfs/go-datastore" "github.com/JackalLabs/sequoia/monitoring" @@ -67,9 +69,23 @@ func NewApp(home string) (*App, error) { return nil, err } + var ds datastore.Batching + switch cfg.DataStoreConfig.Backend { + case "badgerds": + ds, err = ipfs.NewBadgerDataStore(db) + if err != nil { + return nil, err + } + case "flatfs": + ds, err = ipfs.NewFlatfsDataStore(cfg.DataStoreConfig.Directory) + if err != nil { + return nil, err + } + } + apiServer := api.NewAPI(cfg.APICfg.Port) - f, err := file_system.NewFileSystem(ctx, db, cfg.APICfg.IPFSPort, cfg.APICfg.IPFSDomain) + f, err := file_system.NewFileSystem(ctx, db, ds, cfg.APICfg.IPFSPort, cfg.APICfg.IPFSDomain) if err != nil { return nil, err } diff --git a/file_system/types.go b/file_system/types.go index 94e692c..e792afb 100644 --- a/file_system/types.go +++ b/file_system/types.go @@ -3,6 +3,7 @@ package file_system import ( "context" + "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" ipfs2 "github.com/JackalLabs/sequoia/ipfs" @@ -17,8 +18,8 @@ type FileSystem struct { ipfsHost host.Host } -func NewFileSystem(ctx context.Context, db *badger.DB, ipfsPort int, ipfsDomain string) (*FileSystem, error) { - ipfs, host, err := ipfs2.MakeIPFS(ctx, db, ipfsPort, ipfsDomain) +func NewFileSystem(ctx context.Context, db *badger.DB, ds datastore.Batching, ipfsPort int, ipfsDomain string) (*FileSystem, error) { + ipfs, host, err := ipfs2.MakeIPFS(ctx, ds, ipfsPort, ipfsDomain) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 7af8443..fed637e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,9 @@ require ( github.com/hsanjuan/ipfs-lite v1.8.2 github.com/ipfs/boxo v0.17.0 github.com/ipfs/go-cid v0.4.1 + github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 + github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ipld-format v0.6.0 github.com/jackalLabs/canine-chain/v4 v4.0.3 github.com/json-iterator/go v1.1.12 @@ -26,6 +28,7 @@ require ( github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 + github.com/tendermint/tendermint v0.34.27 github.com/wealdtech/go-merkletree/v2 v2.6.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -44,6 +47,7 @@ require ( github.com/Jorropo/jsync v1.0.1 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect + github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -137,7 +141,6 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect - github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect @@ -232,7 +235,6 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect - github.com/tendermint/tendermint v0.34.27 // indirect github.com/tendermint/tm-db v0.6.7 // indirect github.com/tidwall/btree v1.5.0 // indirect github.com/wealdtech/go-merkletree v1.0.0 // indirect diff --git a/go.sum b/go.sum index d9f3bc6..9d5a55b 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -811,12 +813,16 @@ github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= +github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ds-flatfs v0.5.1 h1:ZCIO/kQOS/PSh3vcF1H6a8fkRGS7pOfwfPdx4n/KJH4= +github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= @@ -829,8 +835,10 @@ github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= +github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= @@ -947,6 +955,7 @@ github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoK github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/ipfs/datastore.go b/ipfs/datastore.go new file mode 100644 index 0000000..5451e1a --- /dev/null +++ b/ipfs/datastore.go @@ -0,0 +1,16 @@ +package ipfs + +import ( + "github.com/dgraph-io/badger/v4" + ds "github.com/ipfs/go-datastore" + bds "github.com/ipfs/go-ds-badger2" + fds "github.com/ipfs/go-ds-flatfs" +) + +func NewFlatfsDataStore(path string) (ds.Batching, error) { + return fds.CreateOrOpen(path, fds.IPFS_DEF_SHARD, true) +} + +func NewBadgerDataStore(db *badger.DB) (ds.Batching, error) { + return bds.NewDatastoreFromDB(db) +} diff --git a/ipfs/ipfs.go b/ipfs/ipfs.go index 818b3e3..c4f9236 100644 --- a/ipfs/ipfs.go +++ b/ipfs/ipfs.go @@ -7,20 +7,13 @@ import ( "github.com/libp2p/go-libp2p/core/host" - "github.com/dgraph-io/badger/v4" ipfslite "github.com/hsanjuan/ipfs-lite" + datastore "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/crypto" "github.com/multiformats/go-multiaddr" - - bds "github.com/ipfs/go-ds-badger2" ) -func MakeIPFS(ctx context.Context, db *badger.DB, port int, customDomain string) (*ipfslite.Peer, host.Host, error) { - ds, err := bds.NewDatastoreFromDB(db) - if err != nil { - return nil, nil, err - } - +func MakeIPFS(ctx context.Context, ds datastore.Batching, port int, customDomain string) (*ipfslite.Peer, host.Host, error) { priv, _, err := crypto.GenerateKeyPair(crypto.RSA, 2048) if err != nil { return nil, nil, err From d8446d561b4a0d73e9e6a84abac99c10384b7d3f Mon Sep 17 00:00:00 2001 From: dahn510 Date: Thu, 5 Sep 2024 17:07:36 -0700 Subject: [PATCH 2/9] read config with missing data store config --- config/config.go | 10 ++++++++-- config/types.go | 5 ++++- core/app.go | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 69a7614..70e93e5 100644 --- a/config/config.go +++ b/config/config.go @@ -13,8 +13,8 @@ func (c Config) Validate() error { } switch c.DataStoreConfig.Backend { - case "flatfs": - case "badgerdb": + case OptFlatFS: + case OptBadgerDS: if c.DataStoreConfig.Directory != c.DataDirectory { return errors.New("badger ds directory must be the same as data directory") } @@ -28,6 +28,7 @@ func (c Config) Validate() error { // ReadConfig parses data and returns Config. // Error during parsing or an invalid configuration in the Config will return an error. func ReadConfig(data []byte) (*Config, error) { + //not using a default config to detect badger ds users config := Config{} err := yaml.Unmarshal(data, &config) @@ -35,6 +36,11 @@ func ReadConfig(data []byte) (*Config, error) { return nil, err } + if config.DataStoreConfig.Backend == "" && config.DataStoreConfig.Directory == "" { + config.DataStoreConfig.Backend = OptBadgerDS + config.DataStoreConfig.Directory = config.DataDirectory + } + return &config, config.Validate() } diff --git a/config/types.go b/config/types.go index a7f2b57..7720f18 100644 --- a/config/types.go +++ b/config/types.go @@ -34,6 +34,9 @@ type APIConfig struct { IPFSDomain string `yaml:"ipfs_domain"` } +const OptBadgerDS = "badgerds" +const OptFlatFS = "flatfs" + type DataStoreConfig struct { // *choosing badgerdb as data store will need to use the same directory // for data directory @@ -78,7 +81,7 @@ func DefaultConfig() *Config { ProofThreads: 1000, DataStoreConfig: DataStoreConfig{ Directory: "$HOME/.sequoia/datastore", - Backend: "flatfs", + Backend: OptFlatFS, }, } } diff --git a/core/app.go b/core/app.go index 2728945..610fb2d 100644 --- a/core/app.go +++ b/core/app.go @@ -71,12 +71,12 @@ func NewApp(home string) (*App, error) { var ds datastore.Batching switch cfg.DataStoreConfig.Backend { - case "badgerds": + case config.OptBadgerDS: ds, err = ipfs.NewBadgerDataStore(db) if err != nil { return nil, err } - case "flatfs": + case config.OptFlatFS: ds, err = ipfs.NewFlatfsDataStore(cfg.DataStoreConfig.Directory) if err != nil { return nil, err From cf01d48c7ce76a7ab7fccbe173f275b0abe41253 Mon Sep 17 00:00:00 2001 From: dahn510 Date: Thu, 5 Sep 2024 21:51:35 -0700 Subject: [PATCH 3/9] add config guide for data store --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f78037..277dfda 100644 --- a/README.md +++ b/README.md @@ -29,4 +29,49 @@ Once the wallet is funded, running `sequoia start` again will start the provider ### Earning Rewards -In order for your provider to run correctly, you will need to set up a domain for your provider pointed at the port your provider is running on and set that up in the `config.yaml`. You will also need to make sure you have port `4005` (or whatever you specified in the config) open on TCP and UDP for IPFS support, or you could be penalized by the reporting system. \ No newline at end of file +In order for your provider to run correctly, you will need to set up a domain for your provider pointed at the port your provider is running on and set that up in the `config.yaml`. You will also need to make sure you have port `4005` (or whatever you specified in the config) open on TCP and UDP for IPFS support, or you could be penalized by the reporting system. + +### Configuration + +Default config looks like this: +```yaml +###################### +### Sequoia Config ### +###################### + +queue_interval: 10 +proof_interval: 120 +stray_manager: + check_interval: 30 + refresh_interval: 120 + hands: 2 +chain_config: + bech32_prefix: jkl + rpc_addr: http://localhost:26657 + grpc_addr: localhost:9090 + gas_price: 0.02ujkl + gas_adjustment: 1.5 +domain: https://example.com +total_bytes_offered: 1092616192 +data_directory: $HOME/.sequoia/data +api_config: + port: 3333 + ipfs_port: 4005 + ipfs_domain: dns4/ipfs.example.com/tcp/4001 +proof_threads: 1000 +data_store_config: + directory: $HOME/.sequoia/datastore + backend: flatfs + +###################### +``` + +| `data_directory` | file path | directory to store database (badger db) files | +##### `data_store_config` +sequoia uses ipfs go-datastore interface to store files +`flatfs` stores raw block contents on disk +`badgerds` a key value database that uses LSM tree to store and manage data +currently only supports `flatfs` and `badgerds` data store backends +| `directory` | file path | directory for data store files | +> Using `badgerds` as backend requires data store directory to be same as `data_directory` as both uses badger db to store data. +| `backend` | `flatfs` or `badgerds` | data store backend to store files | From f68183007b4bfaf311b3d738ebcd780f16b3fab3 Mon Sep 17 00:00:00 2001 From: dahn510 Date: Thu, 5 Sep 2024 21:56:17 -0700 Subject: [PATCH 4/9] update format --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 277dfda..85449b2 100644 --- a/README.md +++ b/README.md @@ -65,13 +65,14 @@ data_store_config: ###################### ``` - -| `data_directory` | file path | directory to store database (badger db) files | -##### `data_store_config` +`data_directory`: file path | directory to store database (badger db) files +#### `data_store_config` sequoia uses ipfs go-datastore interface to store files `flatfs` stores raw block contents on disk `badgerds` a key value database that uses LSM tree to store and manage data currently only supports `flatfs` and `badgerds` data store backends -| `directory` | file path | directory for data store files | +`directory`: file path | directory for data store files +`backend`: `flatfs` or `badgerds` | data store backend to store files + > Using `badgerds` as backend requires data store directory to be same as `data_directory` as both uses badger db to store data. -| `backend` | `flatfs` or `badgerds` | data store backend to store files | + From 7f19db322e7622983303db519ef92a5f20a4113f Mon Sep 17 00:00:00 2001 From: dahn510 Date: Wed, 11 Sep 2024 07:29:53 -0700 Subject: [PATCH 5/9] wrap flatfs ds as blockstore use flatfs to only store blocks and everything else with badger --- config/config.go | 12 ++++++------ config/types.go | 36 +++++++++++++++++++----------------- core/app.go | 20 +++++++++++--------- file_system/types.go | 5 +++-- ipfs/datastore.go | 10 ++++++++-- ipfs/ipfs.go | 5 +++-- 6 files changed, 50 insertions(+), 38 deletions(-) diff --git a/config/config.go b/config/config.go index 70e93e5..a6a3df8 100644 --- a/config/config.go +++ b/config/config.go @@ -12,10 +12,10 @@ func (c Config) Validate() error { return errors.New("invalid data directory") } - switch c.DataStoreConfig.Backend { + switch c.BlockStoreConfig.Backend { case OptFlatFS: case OptBadgerDS: - if c.DataStoreConfig.Directory != c.DataDirectory { + if c.BlockStoreConfig.Directory != c.DataDirectory { return errors.New("badger ds directory must be the same as data directory") } default: @@ -28,7 +28,7 @@ func (c Config) Validate() error { // ReadConfig parses data and returns Config. // Error during parsing or an invalid configuration in the Config will return an error. func ReadConfig(data []byte) (*Config, error) { - //not using a default config to detect badger ds users + // not using a default config to detect badger ds users config := Config{} err := yaml.Unmarshal(data, &config) @@ -36,9 +36,9 @@ func ReadConfig(data []byte) (*Config, error) { return nil, err } - if config.DataStoreConfig.Backend == "" && config.DataStoreConfig.Directory == "" { - config.DataStoreConfig.Backend = OptBadgerDS - config.DataStoreConfig.Directory = config.DataDirectory + if config.BlockStoreConfig.Backend == "" && config.BlockStoreConfig.Directory == "" { + config.BlockStoreConfig.Backend = OptBadgerDS + config.BlockStoreConfig.Directory = config.DataDirectory } return &config, config.Validate() diff --git a/config/types.go b/config/types.go index 7720f18..100d336 100644 --- a/config/types.go +++ b/config/types.go @@ -10,16 +10,16 @@ type Seed struct { DerivationPath string `json:"derivation_path"` } type Config struct { - QueueInterval int64 `yaml:"queue_interval"` - ProofInterval int64 `yaml:"proof_interval"` - StrayManagerCfg StrayManagerConfig `yaml:"stray_manager"` - ChainCfg types.ChainConfig `yaml:"chain_config"` - Ip string `yaml:"domain"` - TotalSpace int64 `yaml:"total_bytes_offered"` - DataDirectory string `yaml:"data_directory"` - APICfg APIConfig `yaml:"api_config"` - ProofThreads int64 `yaml:"proof_threads"` - DataStoreConfig DataStoreConfig `yaml:"data_store_config"` + QueueInterval int64 `yaml:"queue_interval"` + ProofInterval int64 `yaml:"proof_interval"` + StrayManagerCfg StrayManagerConfig `yaml:"stray_manager"` + ChainCfg types.ChainConfig `yaml:"chain_config"` + Ip string `yaml:"domain"` + TotalSpace int64 `yaml:"total_bytes_offered"` + DataDirectory string `yaml:"data_directory"` + APICfg APIConfig `yaml:"api_config"` + ProofThreads int64 `yaml:"proof_threads"` + BlockStoreConfig BlockStoreConfig `yaml:"data_store_config"` } type StrayManagerConfig struct { @@ -34,11 +34,13 @@ type APIConfig struct { IPFSDomain string `yaml:"ipfs_domain"` } -const OptBadgerDS = "badgerds" -const OptFlatFS = "flatfs" +const ( + OptBadgerDS = "badgerds" + OptFlatFS = "flatfs" +) -type DataStoreConfig struct { - // *choosing badgerdb as data store will need to use the same directory +type BlockStoreConfig struct { + // *choosing badgerdb as block store will need to use the same directory // for data directory Directory string `yaml:"directory"` // data store options: flatfs, badgerdb @@ -79,8 +81,8 @@ func DefaultConfig() *Config { IPFSDomain: "dns4/ipfs.example.com/tcp/4001", }, ProofThreads: 1000, - DataStoreConfig: DataStoreConfig{ - Directory: "$HOME/.sequoia/datastore", + BlockStoreConfig: BlockStoreConfig{ + Directory: "$HOME/.sequoia/blockstore", Backend: OptFlatFS, }, } @@ -103,5 +105,5 @@ func (c Config) MarshalZerologObject(e *zerolog.Event) { Int("APIIPFSPort", c.APICfg.IPFSPort). Str("APIIPFSDomain", c.APICfg.IPFSDomain). Int64("ProofThreads", c.ProofThreads). - Str("DatastoreBackend", c.DataStoreConfig.Backend) + Str("BlockstoreBackend", c.BlockStoreConfig.Backend) } diff --git a/core/app.go b/core/app.go index 610fb2d..5d324a9 100644 --- a/core/app.go +++ b/core/app.go @@ -11,7 +11,7 @@ import ( "github.com/JackalLabs/sequoia/file_system" "github.com/JackalLabs/sequoia/ipfs" - "github.com/ipfs/go-datastore" + "github.com/ipfs/boxo/blockstore" "github.com/JackalLabs/sequoia/monitoring" @@ -69,15 +69,17 @@ func NewApp(home string) (*App, error) { return nil, err } - var ds datastore.Batching - switch cfg.DataStoreConfig.Backend { + ds, err := ipfs.NewBadgerDataStore(db) + if err != nil { + return nil, err + } + + var bs blockstore.Blockstore + bs = nil + switch cfg.BlockStoreConfig.Backend { case config.OptBadgerDS: - ds, err = ipfs.NewBadgerDataStore(db) - if err != nil { - return nil, err - } case config.OptFlatFS: - ds, err = ipfs.NewFlatfsDataStore(cfg.DataStoreConfig.Directory) + bs, err = ipfs.NewFlatfsBlockStore(cfg.BlockStoreConfig.Directory) if err != nil { return nil, err } @@ -85,7 +87,7 @@ func NewApp(home string) (*App, error) { apiServer := api.NewAPI(cfg.APICfg.Port) - f, err := file_system.NewFileSystem(ctx, db, ds, cfg.APICfg.IPFSPort, cfg.APICfg.IPFSDomain) + f, err := file_system.NewFileSystem(ctx, db, ds, bs, cfg.APICfg.IPFSPort, cfg.APICfg.IPFSDomain) if err != nil { return nil, err } diff --git a/file_system/types.go b/file_system/types.go index e792afb..f4ee2fd 100644 --- a/file_system/types.go +++ b/file_system/types.go @@ -3,6 +3,7 @@ package file_system import ( "context" + "github.com/ipfs/boxo/blockstore" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" @@ -18,8 +19,8 @@ type FileSystem struct { ipfsHost host.Host } -func NewFileSystem(ctx context.Context, db *badger.DB, ds datastore.Batching, ipfsPort int, ipfsDomain string) (*FileSystem, error) { - ipfs, host, err := ipfs2.MakeIPFS(ctx, ds, ipfsPort, ipfsDomain) +func NewFileSystem(ctx context.Context, db *badger.DB, ds datastore.Batching, bs blockstore.Blockstore, ipfsPort int, ipfsDomain string) (*FileSystem, error) { + ipfs, host, err := ipfs2.MakeIPFS(ctx, ds, bs, ipfsPort, ipfsDomain) if err != nil { return nil, err } diff --git a/ipfs/datastore.go b/ipfs/datastore.go index 5451e1a..b307183 100644 --- a/ipfs/datastore.go +++ b/ipfs/datastore.go @@ -2,13 +2,19 @@ package ipfs import ( "github.com/dgraph-io/badger/v4" + "github.com/ipfs/boxo/blockstore" ds "github.com/ipfs/go-datastore" bds "github.com/ipfs/go-ds-badger2" fds "github.com/ipfs/go-ds-flatfs" ) -func NewFlatfsDataStore(path string) (ds.Batching, error) { - return fds.CreateOrOpen(path, fds.IPFS_DEF_SHARD, true) +func NewFlatfsBlockStore(path string) (blockstore.Blockstore, error) { + ds, err := fds.CreateOrOpen(path, fds.IPFS_DEF_SHARD, true) + if err != nil { + return nil, err + } + + return blockstore.NewBlockstore(ds, blockstore.Option{}), nil } func NewBadgerDataStore(db *badger.DB) (ds.Batching, error) { diff --git a/ipfs/ipfs.go b/ipfs/ipfs.go index c4f9236..b156722 100644 --- a/ipfs/ipfs.go +++ b/ipfs/ipfs.go @@ -8,12 +8,13 @@ import ( "github.com/libp2p/go-libp2p/core/host" ipfslite "github.com/hsanjuan/ipfs-lite" + "github.com/ipfs/boxo/blockstore" datastore "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/crypto" "github.com/multiformats/go-multiaddr" ) -func MakeIPFS(ctx context.Context, ds datastore.Batching, port int, customDomain string) (*ipfslite.Peer, host.Host, error) { +func MakeIPFS(ctx context.Context, ds datastore.Batching, bs blockstore.Blockstore, port int, customDomain string) (*ipfslite.Peer, host.Host, error) { priv, _, err := crypto.GenerateKeyPair(crypto.RSA, 2048) if err != nil { return nil, nil, err @@ -53,7 +54,7 @@ func MakeIPFS(ctx context.Context, ds datastore.Batching, port int, customDomain return nil, h, err } - lite, err := ipfslite.New(ctx, ds, nil, h, dht, nil) + lite, err := ipfslite.New(ctx, ds, bs, h, dht, nil) if err != nil { return nil, h, err } From 56dbbcf4ca4858727c0bfa29306c276f31d9ff29 Mon Sep 17 00:00:00 2001 From: dahn510 Date: Thu, 12 Sep 2024 16:15:51 -0700 Subject: [PATCH 6/9] rename config fields datastore > blockstore Backend > Type --- api/file_handler.go | 2 +- config/config.go | 8 ++++---- config/types.go | 8 ++++---- config/wallet.go | 2 +- core/app.go | 4 ++-- file_system/file_system_test.go | 2 +- ipfs/datastore.go | 2 +- ipfs/ipfs.go | 2 +- proofs/proofs.go | 2 +- proofs/types.go | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/api/file_handler.go b/api/file_handler.go index fed090a..1df9b87 100644 --- a/api/file_handler.go +++ b/api/file_handler.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/ipfs/go-cid" + cid "github.com/ipfs/go-cid" "github.com/JackalLabs/sequoia/proofs" diff --git a/config/config.go b/config/config.go index a6a3df8..fc63a76 100644 --- a/config/config.go +++ b/config/config.go @@ -4,7 +4,7 @@ import ( "errors" "strings" - "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v3" ) func (c Config) Validate() error { @@ -12,7 +12,7 @@ func (c Config) Validate() error { return errors.New("invalid data directory") } - switch c.BlockStoreConfig.Backend { + switch c.BlockStoreConfig.Type { case OptFlatFS: case OptBadgerDS: if c.BlockStoreConfig.Directory != c.DataDirectory { @@ -36,8 +36,8 @@ func ReadConfig(data []byte) (*Config, error) { return nil, err } - if config.BlockStoreConfig.Backend == "" && config.BlockStoreConfig.Directory == "" { - config.BlockStoreConfig.Backend = OptBadgerDS + if config.BlockStoreConfig.Type == "" && config.BlockStoreConfig.Directory == "" { + config.BlockStoreConfig.Type = OptBadgerDS config.BlockStoreConfig.Directory = config.DataDirectory } diff --git a/config/types.go b/config/types.go index 100d336..e38df6a 100644 --- a/config/types.go +++ b/config/types.go @@ -19,7 +19,7 @@ type Config struct { DataDirectory string `yaml:"data_directory"` APICfg APIConfig `yaml:"api_config"` ProofThreads int64 `yaml:"proof_threads"` - BlockStoreConfig BlockStoreConfig `yaml:"data_store_config"` + BlockStoreConfig BlockStoreConfig `yaml:"block_store_config"` } type StrayManagerConfig struct { @@ -44,7 +44,7 @@ type BlockStoreConfig struct { // for data directory Directory string `yaml:"directory"` // data store options: flatfs, badgerdb - Backend string `yaml:"backend"` + Type string `yaml:"type"` } // LegacyWallet handles keys from earlier versions of storage providers. @@ -83,7 +83,7 @@ func DefaultConfig() *Config { ProofThreads: 1000, BlockStoreConfig: BlockStoreConfig{ Directory: "$HOME/.sequoia/blockstore", - Backend: OptFlatFS, + Type: OptFlatFS, }, } } @@ -105,5 +105,5 @@ func (c Config) MarshalZerologObject(e *zerolog.Event) { Int("APIIPFSPort", c.APICfg.IPFSPort). Str("APIIPFSDomain", c.APICfg.IPFSDomain). Int64("ProofThreads", c.ProofThreads). - Str("BlockstoreBackend", c.BlockStoreConfig.Backend) + Str("BlockstoreBackend", c.BlockStoreConfig.Type) } diff --git a/config/wallet.go b/config/wallet.go index 9b11574..9820200 100644 --- a/config/wallet.go +++ b/config/wallet.go @@ -6,7 +6,7 @@ import ( "path" sequoiaWallet "github.com/JackalLabs/sequoia/wallet" - "github.com/cosmos/go-bip39" + bip39 "github.com/cosmos/go-bip39" "github.com/desmos-labs/cosmos-go-wallet/wallet" jsoniter "github.com/json-iterator/go" diff --git a/core/app.go b/core/app.go index 5d324a9..646f9e4 100644 --- a/core/app.go +++ b/core/app.go @@ -25,7 +25,7 @@ import ( "github.com/JackalLabs/sequoia/strays" walletTypes "github.com/desmos-labs/cosmos-go-wallet/types" "github.com/desmos-labs/cosmos-go-wallet/wallet" - "github.com/dgraph-io/badger/v4" + badger "github.com/dgraph-io/badger/v4" "github.com/jackalLabs/canine-chain/v4/x/storage/types" storageTypes "github.com/jackalLabs/canine-chain/v4/x/storage/types" "github.com/rs/zerolog/log" @@ -76,7 +76,7 @@ func NewApp(home string) (*App, error) { var bs blockstore.Blockstore bs = nil - switch cfg.BlockStoreConfig.Backend { + switch cfg.BlockStoreConfig.Type { case config.OptBadgerDS: case config.OptFlatFS: bs, err = ipfs.NewFlatfsBlockStore(cfg.BlockStoreConfig.Directory) diff --git a/file_system/file_system_test.go b/file_system/file_system_test.go index 3bae0df..ecedd6c 100644 --- a/file_system/file_system_test.go +++ b/file_system/file_system_test.go @@ -51,7 +51,7 @@ func BenchmarkFileWrites(b *testing.B) { err = db.DropAll() require.NoError(b, err) - f, err := NewFileSystem(context.Background(), db, 4005, "/dns4/ipfs.example.com/tcp/4001") + f, err := NewFileSystem(context.Background(), db, nil, 4005, "/dns4/ipfs.example.com/tcp/4001") require.NoError(b, err) defer db.Close() server := &http.Server{ diff --git a/ipfs/datastore.go b/ipfs/datastore.go index b307183..a797c04 100644 --- a/ipfs/datastore.go +++ b/ipfs/datastore.go @@ -1,7 +1,7 @@ package ipfs import ( - "github.com/dgraph-io/badger/v4" + badger "github.com/dgraph-io/badger/v4" "github.com/ipfs/boxo/blockstore" ds "github.com/ipfs/go-datastore" bds "github.com/ipfs/go-ds-badger2" diff --git a/ipfs/ipfs.go b/ipfs/ipfs.go index b156722..97f57f4 100644 --- a/ipfs/ipfs.go +++ b/ipfs/ipfs.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/boxo/blockstore" datastore "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/crypto" - "github.com/multiformats/go-multiaddr" + multiaddr "github.com/multiformats/go-multiaddr" ) func MakeIPFS(ctx context.Context, ds datastore.Batching, bs blockstore.Blockstore, port int, customDomain string) (*ipfslite.Peer, host.Host, error) { diff --git a/proofs/proofs.go b/proofs/proofs.go index 67134b6..b0eed48 100644 --- a/proofs/proofs.go +++ b/proofs/proofs.go @@ -15,7 +15,7 @@ import ( "github.com/desmos-labs/cosmos-go-wallet/wallet" "github.com/jackalLabs/canine-chain/v4/x/storage/types" "github.com/rs/zerolog/log" - "github.com/wealdtech/go-merkletree/v2" + merkletree "github.com/wealdtech/go-merkletree/v2" "github.com/wealdtech/go-merkletree/v2/sha3" ) import jsoniter "github.com/json-iterator/go" diff --git a/proofs/types.go b/proofs/types.go index cc70073..18171e6 100644 --- a/proofs/types.go +++ b/proofs/types.go @@ -3,7 +3,7 @@ package proofs import ( "time" - "github.com/wealdtech/go-merkletree/v2" + merkletree "github.com/wealdtech/go-merkletree/v2" "github.com/JackalLabs/sequoia/queue" "github.com/desmos-labs/cosmos-go-wallet/wallet" From 9c6fa62ddc20cfb4f909699a9edadcc1e8e58a4e Mon Sep 17 00:00:00 2001 From: dahn510 Date: Thu, 12 Sep 2024 16:17:26 -0700 Subject: [PATCH 7/9] Update block store config detail --- README.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 85449b2..0adec0d 100644 --- a/README.md +++ b/README.md @@ -59,20 +59,18 @@ api_config: ipfs_port: 4005 ipfs_domain: dns4/ipfs.example.com/tcp/4001 proof_threads: 1000 -data_store_config: +block_store_config: directory: $HOME/.sequoia/datastore - backend: flatfs + type: flatfs ###################### ``` -`data_directory`: file path | directory to store database (badger db) files -#### `data_store_config` -sequoia uses ipfs go-datastore interface to store files -`flatfs` stores raw block contents on disk -`badgerds` a key value database that uses LSM tree to store and manage data -currently only supports `flatfs` and `badgerds` data store backends -`directory`: file path | directory for data store files -`backend`: `flatfs` or `badgerds` | data store backend to store files - -> Using `badgerds` as backend requires data store directory to be same as `data_directory` as both uses badger db to store data. +`data_directory`: directory for database files +#### `block_store_config` +`directory`: directory for block store files +`type`: `flatfs` or `badgerds` +There are two types of block store available to sequoia: +`badgerds` is a key value database that uses LSM tree to store and manage data. The storage limit is < 11TB. +`flatfs` stores raw block contents on disk. Relies on underlying file system for stability and performance. +> Using `badgerds` requires the block store directory to be same as `data_directory` because badgerdb is used for database as well. From 8c516d5e599f71ab9db231473f4ce851c0a47abc Mon Sep 17 00:00:00 2001 From: dahn510 Date: Thu, 12 Sep 2024 17:00:46 -0700 Subject: [PATCH 8/9] add block store option --- core/app.go | 3 ++- ipfs/{datastore.go => blockstore.go} | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) rename ipfs/{datastore.go => blockstore.go} (88%) diff --git a/core/app.go b/core/app.go index 646f9e4..4ad728d 100644 --- a/core/app.go +++ b/core/app.go @@ -74,12 +74,13 @@ func NewApp(home string) (*App, error) { return nil, err } + bsDir := os.ExpandEnv(cfg.BlockStoreConfig.Directory) var bs blockstore.Blockstore bs = nil switch cfg.BlockStoreConfig.Type { case config.OptBadgerDS: case config.OptFlatFS: - bs, err = ipfs.NewFlatfsBlockStore(cfg.BlockStoreConfig.Directory) + bs, err = ipfs.NewFlatfsBlockStore(bsDir) if err != nil { return nil, err } diff --git a/ipfs/datastore.go b/ipfs/blockstore.go similarity index 88% rename from ipfs/datastore.go rename to ipfs/blockstore.go index a797c04..dfef1a4 100644 --- a/ipfs/datastore.go +++ b/ipfs/blockstore.go @@ -14,7 +14,7 @@ func NewFlatfsBlockStore(path string) (blockstore.Blockstore, error) { return nil, err } - return blockstore.NewBlockstore(ds, blockstore.Option{}), nil + return blockstore.NewBlockstore(ds, blockstore.NoPrefix()), nil } func NewBadgerDataStore(db *badger.DB) (ds.Batching, error) { From 5fa3bcd67fd5147ed57e03ebfb06d55aeb74aa5e Mon Sep 17 00:00:00 2001 From: marston Date: Thu, 12 Sep 2024 22:43:54 -0400 Subject: [PATCH 9/9] linting --- file_system/file_system_test.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/file_system/file_system_test.go b/file_system/file_system_test.go index ecedd6c..113a55c 100644 --- a/file_system/file_system_test.go +++ b/file_system/file_system_test.go @@ -1,7 +1,11 @@ package file_system //nolint:all -import "io/ioutil" +import ( + "io/ioutil" + + "github.com/JackalLabs/sequoia/ipfs" +) import ( "bytes" @@ -51,7 +55,10 @@ func BenchmarkFileWrites(b *testing.B) { err = db.DropAll() require.NoError(b, err) - f, err := NewFileSystem(context.Background(), db, nil, 4005, "/dns4/ipfs.example.com/tcp/4001") + ds, err := ipfs.NewBadgerDataStore(db) + require.NoError(b, err) + + f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "/dns4/ipfs.example.com/tcp/4001") require.NoError(b, err) defer db.Close() server := &http.Server{ @@ -91,7 +98,11 @@ func TestWriteFile(t *testing.T) { err = db.DropAll() require.NoError(t, err) - f, err := NewFileSystem(context.Background(), db, 4005, "/dns4/ipfs.example.com/tcp/4001") + + ds, err := ipfs.NewBadgerDataStore(db) + require.NoError(t, err) + + f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "/dns4/ipfs.example.com/tcp/4001") require.NoError(t, err) defer db.Close() @@ -137,7 +148,11 @@ func TestWriteFileWithDomain(t *testing.T) { err = db.DropAll() require.NoError(t, err) - f, err := NewFileSystem(context.Background(), db, 4005, "dns4/jackal-testnet-v4-storage.p2p.brocha.in/tcp/30506") + + ds, err := ipfs.NewBadgerDataStore(db) + require.NoError(t, err) + + f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "dns4/jackal-testnet-v4-storage.p2p.brocha.in/tcp/30506") require.NoError(t, err) defer db.Close() @@ -186,7 +201,11 @@ func TestWriteAndProveFiles(t *testing.T) { err = db.DropAll() require.NoError(t, err) - f, err := NewFileSystem(context.Background(), db, 4005, "/dns4/ipfs.example.com/tcp/4001") + + ds, err := ipfs.NewBadgerDataStore(db) + require.NoError(t, err) + + f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "/dns4/ipfs.example.com/tcp/4001") require.NoError(t, err) size := 1024 * 255 // 255 kbs var chunkSize int64 = 1024