Merge branch 'main' into marston/shutdown

README.md

@@ -29,4 +29,48 @@ Once the wallet is funded, running `sequoia start` again will start the provider
 
 ### Earning Rewards
 
-In order for your provider to run correctly, you will need to set up a domain for your provider pointed at the port your provider is running on and set that up in the `config.yaml`. You will also need to make sure you have port `4005` (or whatever you specified in the config) open on TCP and UDP for IPFS support, or you could be penalized by the reporting system.
+In order for your provider to run correctly, you will need to set up a domain for your provider pointed at the port your provider is running on and set that up in the `config.yaml`. You will also need to make sure you have port `4005` (or whatever you specified in the config) open on TCP and UDP for IPFS support, or you could be penalized by the reporting system.
+
+### Configuration
+
+Default config looks like this:  
+```yaml
+######################
+### Sequoia Config ###
+######################
+
+queue_interval: 10
+proof_interval: 120
+stray_manager:
+    check_interval: 30
+    refresh_interval: 120
+    hands: 2
+chain_config:
+    bech32_prefix: jkl
+    rpc_addr: http://localhost:26657
+    grpc_addr: localhost:9090
+    gas_price: 0.02ujkl
+    gas_adjustment: 1.5
+domain: https://example.com
+total_bytes_offered: 1092616192
+data_directory: $HOME/.sequoia/data
+api_config:
+    port: 3333
+    ipfs_port: 4005
+    ipfs_domain: dns4/ipfs.example.com/tcp/4001
+proof_threads: 1000
+block_store_config:
+    directory: $HOME/.sequoia/datastore
+    type: flatfs
+
+######################
+```  
+`data_directory`: directory for database files
+#### `block_store_config`
+`directory`: directory for block store files  
+`type`: `flatfs` or `badgerds`  
+There are two types of block store available to sequoia:  
+`badgerds` is a key value database that uses LSM tree to store and manage data. The storage limit is < 11TB.  
+`flatfs` stores raw block contents on disk. Relies on underlying file system for stability and performance.  
+> Using `badgerds` requires the block store directory to be same as `data_directory` because badgerdb is used for database as well.
+

api/file_handler.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ipfs/go-cid"
+	cid "github.com/ipfs/go-cid"
 
 	"github.com/JackalLabs/sequoia/proofs"
 

config/config.go

@@ -1,20 +1,47 @@
 package config
 
 import (
+	"errors"
 	"strings"
 
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
+func (c Config) Validate() error {
+	if c.DataDirectory == "" {
+		return errors.New("invalid data directory")
+	}
+
+	switch c.BlockStoreConfig.Type {
+	case OptFlatFS:
+	case OptBadgerDS:
+		if c.BlockStoreConfig.Directory != c.DataDirectory {
+			return errors.New("badger ds directory must be the same as data directory")
+		}
+	default:
+		return errors.New("invalid data store backend")
+	}
+
+	return nil
+}
+
+// ReadConfig parses data and returns Config.
+// Error during parsing or an invalid configuration in the Config will return an error.
 func ReadConfig(data []byte) (*Config, error) {
+	// not using a default config to detect badger ds users
 	config := Config{}
 
 	err := yaml.Unmarshal(data, &config)
 	if err != nil {
 		return nil, err
 	}
 
-	return &config, nil
+	if config.BlockStoreConfig.Type == "" && config.BlockStoreConfig.Directory == "" {
+		config.BlockStoreConfig.Type = OptBadgerDS
+		config.BlockStoreConfig.Directory = config.DataDirectory
+	}
+
+	return &config, config.Validate()
 }
 
 func (c Config) Export() ([]byte, error) {

config/types.go

@@ -10,15 +10,16 @@ type Seed struct {
 	DerivationPath string `json:"derivation_path"`
 }
 type Config struct {
-	QueueInterval   int64              `yaml:"queue_interval"`
-	ProofInterval   int64              `yaml:"proof_interval"`
-	StrayManagerCfg StrayManagerConfig `yaml:"stray_manager"`
-	ChainCfg        types.ChainConfig  `yaml:"chain_config"`
-	Ip              string             `yaml:"domain"`
-	TotalSpace      int64              `yaml:"total_bytes_offered"`
-	DataDirectory   string             `yaml:"data_directory"`
-	APICfg          APIConfig          `yaml:"api_config"`
-	ProofThreads    int64              `yaml:"proof_threads"`
+	QueueInterval    int64              `yaml:"queue_interval"`
+	ProofInterval    int64              `yaml:"proof_interval"`
+	StrayManagerCfg  StrayManagerConfig `yaml:"stray_manager"`
+	ChainCfg         types.ChainConfig  `yaml:"chain_config"`
+	Ip               string             `yaml:"domain"`
+	TotalSpace       int64              `yaml:"total_bytes_offered"`
+	DataDirectory    string             `yaml:"data_directory"`
+	APICfg           APIConfig          `yaml:"api_config"`
+	ProofThreads     int64              `yaml:"proof_threads"`
+	BlockStoreConfig BlockStoreConfig   `yaml:"block_store_config"`
 }
 
 type StrayManagerConfig struct {
@@ -33,6 +34,19 @@ type APIConfig struct {
 	IPFSDomain string `yaml:"ipfs_domain"`
 }
 
+const (
+	OptBadgerDS = "badgerds"
+	OptFlatFS   = "flatfs"
+)
+
+type BlockStoreConfig struct {
+	// *choosing badgerdb as block store will need to use the same directory
+	// for data directory
+	Directory string `yaml:"directory"`
+	// data store options: flatfs, badgerdb
+	Type string `yaml:"type"`
+}
+
 // LegacyWallet handles keys from earlier versions of storage providers.
 // v3 and earlier providers used private key to sign txs
 // and by design it can't derive mnemonic seed which made
@@ -67,6 +81,10 @@ func DefaultConfig() *Config {
 			IPFSDomain: "dns4/ipfs.example.com/tcp/4001",
 		},
 		ProofThreads: 1000,
+		BlockStoreConfig: BlockStoreConfig{
+			Directory: "$HOME/.sequoia/blockstore",
+			Type:      OptFlatFS,
+		},
 	}
 }
 
@@ -86,5 +104,6 @@ func (c Config) MarshalZerologObject(e *zerolog.Event) {
 		Int64("APIPort", c.APICfg.Port).
 		Int("APIIPFSPort", c.APICfg.IPFSPort).
 		Str("APIIPFSDomain", c.APICfg.IPFSDomain).
-		Int64("ProofThreads", c.ProofThreads)
+		Int64("ProofThreads", c.ProofThreads).
+		Str("BlockstoreBackend", c.BlockStoreConfig.Type)
 }

config/wallet.go

@@ -6,7 +6,7 @@ import (
 	"path"
 
 	sequoiaWallet "github.com/JackalLabs/sequoia/wallet"
-	"github.com/cosmos/go-bip39"
+	bip39 "github.com/cosmos/go-bip39"
 	"github.com/desmos-labs/cosmos-go-wallet/wallet"
 
 	jsoniter "github.com/json-iterator/go"

core/app.go

@@ -10,6 +10,8 @@ import (
 	"time"
 
 	"github.com/JackalLabs/sequoia/file_system"
+	"github.com/JackalLabs/sequoia/ipfs"
+	"github.com/ipfs/boxo/blockstore"
 
 	"github.com/JackalLabs/sequoia/monitoring"
 
@@ -23,7 +25,7 @@ import (
 	"github.com/JackalLabs/sequoia/strays"
 	walletTypes "github.com/desmos-labs/cosmos-go-wallet/types"
 	"github.com/desmos-labs/cosmos-go-wallet/wallet"
-	"github.com/dgraph-io/badger/v4"
+	badger "github.com/dgraph-io/badger/v4"
 	"github.com/jackalLabs/canine-chain/v4/x/storage/types"
 	storageTypes "github.com/jackalLabs/canine-chain/v4/x/storage/types"
 	"github.com/rs/zerolog/log"
@@ -67,9 +69,26 @@ func NewApp(home string) (*App, error) {
 		return nil, err
 	}
 
+	ds, err := ipfs.NewBadgerDataStore(db)
+	if err != nil {
+		return nil, err
+	}
+
+	bsDir := os.ExpandEnv(cfg.BlockStoreConfig.Directory)
+	var bs blockstore.Blockstore
+	bs = nil
+	switch cfg.BlockStoreConfig.Type {
+	case config.OptBadgerDS:
+	case config.OptFlatFS:
+		bs, err = ipfs.NewFlatfsBlockStore(bsDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	apiServer := api.NewAPI(cfg.APICfg.Port)
 
-	f, err := file_system.NewFileSystem(ctx, db, cfg.APICfg.IPFSPort, cfg.APICfg.IPFSDomain)
+	f, err := file_system.NewFileSystem(ctx, db, ds, bs, cfg.APICfg.IPFSPort, cfg.APICfg.IPFSDomain)
 	if err != nil {
 		return nil, err
 	}

file_system/file_system_test.go

@@ -1,7 +1,11 @@
 package file_system
 
 //nolint:all
-import "io/ioutil"
+import (
+	"io/ioutil"
+
+	"github.com/JackalLabs/sequoia/ipfs"
+)
 
 import (
 	"bytes"
@@ -51,7 +55,10 @@ func BenchmarkFileWrites(b *testing.B) {
 	err = db.DropAll()
 	require.NoError(b, err)
 
-	f, err := NewFileSystem(context.Background(), db, 4005, "/dns4/ipfs.example.com/tcp/4001")
+	ds, err := ipfs.NewBadgerDataStore(db)
+	require.NoError(b, err)
+
+	f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "/dns4/ipfs.example.com/tcp/4001")
 	require.NoError(b, err)
 	defer db.Close()
 	server := &http.Server{
@@ -91,7 +98,11 @@ func TestWriteFile(t *testing.T) {
 
 	err = db.DropAll()
 	require.NoError(t, err)
-	f, err := NewFileSystem(context.Background(), db, 4005, "/dns4/ipfs.example.com/tcp/4001")
+
+	ds, err := ipfs.NewBadgerDataStore(db)
+	require.NoError(t, err)
+
+	f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "/dns4/ipfs.example.com/tcp/4001")
 	require.NoError(t, err)
 	defer db.Close()
 
@@ -137,7 +148,11 @@ func TestWriteFileWithDomain(t *testing.T) {
 
 	err = db.DropAll()
 	require.NoError(t, err)
-	f, err := NewFileSystem(context.Background(), db, 4005, "dns4/jackal-testnet-v4-storage.p2p.brocha.in/tcp/30506")
+
+	ds, err := ipfs.NewBadgerDataStore(db)
+	require.NoError(t, err)
+
+	f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "dns4/jackal-testnet-v4-storage.p2p.brocha.in/tcp/30506")
 	require.NoError(t, err)
 	defer db.Close()
 
@@ -186,7 +201,11 @@ func TestWriteAndProveFiles(t *testing.T) {
 
 	err = db.DropAll()
 	require.NoError(t, err)
-	f, err := NewFileSystem(context.Background(), db, 4005, "/dns4/ipfs.example.com/tcp/4001")
+
+	ds, err := ipfs.NewBadgerDataStore(db)
+	require.NoError(t, err)
+
+	f, err := NewFileSystem(context.Background(), db, ds, nil, 4005, "/dns4/ipfs.example.com/tcp/4001")
 	require.NoError(t, err)
 	size := 1024 * 255 // 255 kbs
 	var chunkSize int64 = 1024

file_system/types.go

@@ -3,6 +3,8 @@ package file_system
 import (
 	"context"
 
+	"github.com/ipfs/boxo/blockstore"
+	"github.com/ipfs/go-datastore"
 	"github.com/libp2p/go-libp2p/core/host"
 
 	ipfs2 "github.com/JackalLabs/sequoia/ipfs"
@@ -17,8 +19,8 @@ type FileSystem struct {
 	ipfsHost host.Host
 }
 
-func NewFileSystem(ctx context.Context, db *badger.DB, ipfsPort int, ipfsDomain string) (*FileSystem, error) {
-	ipfs, host, err := ipfs2.MakeIPFS(ctx, db, ipfsPort, ipfsDomain)
+func NewFileSystem(ctx context.Context, db *badger.DB, ds datastore.Batching, bs blockstore.Blockstore, ipfsPort int, ipfsDomain string) (*FileSystem, error) {
+	ipfs, host, err := ipfs2.MakeIPFS(ctx, ds, bs, ipfsPort, ipfsDomain)
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -15,7 +15,9 @@ require (
 	github.com/hsanjuan/ipfs-lite v1.8.2
 	github.com/ipfs/boxo v0.17.0
 	github.com/ipfs/go-cid v0.4.1
+	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-badger2 v0.1.3
+	github.com/ipfs/go-ds-flatfs v0.5.1
 	github.com/ipfs/go-ipld-format v0.6.0
 	github.com/jackalLabs/canine-chain/v4 v4.0.3
 	github.com/json-iterator/go v1.1.12
@@ -45,6 +47,7 @@ require (
 	github.com/Jorropo/jsync v1.0.1 // indirect
 	github.com/Workiva/go-datastructures v1.0.53 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
+	github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -138,7 +141,6 @@ require (
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-block-format v0.2.0 // indirect
 	github.com/ipfs/go-cidutil v0.1.0 // indirect
-	github.com/ipfs/go-datastore v0.6.0 // indirect
 	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
 	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.3 // indirect

go.sum

@@ -142,6 +142,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs=
 github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
@@ -811,12 +813,16 @@ github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
 github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
 github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q=
 github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA=
+github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk=
 github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk=
 github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8=
 github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
 github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
+github.com/ipfs/go-ds-flatfs v0.5.1 h1:ZCIO/kQOS/PSh3vcF1H6a8fkRGS7pOfwfPdx4n/KJH4=
+github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4=
 github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
 github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
+github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
 github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE=
@@ -829,8 +835,10 @@ github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten
 github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg=
 github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk=
 github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM=
+github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
 github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8=
 github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo=
+github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
 github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
 github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY=
 github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI=
@@ -947,6 +955,7 @@ github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoK
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=