From ef60b9a873ed826d59ec537d9cf424a599c1a904 Mon Sep 17 00:00:00 2001 From: Daniel Ahn Date: Tue, 24 Sep 2024 08:56:38 -0700 Subject: [PATCH 01/16] bind cmd flags to config types --- cmd/start.go | 13 +++++++++++++ config/config.go | 3 +-- config/files.go | 17 +++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 7257843..46515ca 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -9,6 +9,7 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/spf13/viper" ) const defaultMaxRestartAttempt = 60 @@ -62,5 +63,17 @@ func StartCmd() *cobra.Command { } cmd.Flags().Int("restart-attempt", defaultMaxRestartAttempt, "attempt to restart times when the provider fails to start") + cmd.Flags().String("ip", "http://example.com", "provider comain") + cmd.Flags().Int64("apicfg.port", 3333, "port to serve api requests") + cmd.Flags().Int("apicfg.ipfsport", 4005, "port for IPFS") + cmd.Flags().String("apicfg.ipfsdomain", "dns4/ipfs.example.com/tcp/4001", "IPFS domain") + cmd.Flags().Int64("proofthreads", 1000, "maximum threads for proofs") + cmd.Flags().String("datadirectory", "$HOME/.sequoia/data", "directory to store database files") + cmd.Flags().Int64("queueinterval", 10, "seconds to wait until next cycle to flush the transaction queue") + cmd.Flags().Int64("proofinterval", 120, "seconds to wait until next cycle to post proofs") + cmd.Flags().Int64("totalspace", 1092616192, "maximum storage space to provide in bytes") + + viper.BindPFlags(cmd.Flags()) + return cmd } diff --git a/config/config.go b/config/config.go index fc63a76..0213252 100644 --- a/config/config.go +++ b/config/config.go @@ -31,8 +31,7 @@ func ReadConfig(data []byte) (*Config, error) { // not using a default config to detect badger ds users config := Config{} - err := yaml.Unmarshal(data, &config) - if err != nil { + if err := yaml.Unmarshal(data, &config); err != nil { return nil, err } diff --git a/config/files.go b/config/files.go index 93979b8..50c715e 100644 --- a/config/files.go +++ b/config/files.go @@ -4,6 +4,9 @@ import ( "errors" "os" "path" + + "github.com/rs/zerolog/log" + "github.com/spf13/viper" ) const ConfigFileName = "config.yaml" @@ -62,7 +65,6 @@ func ReadConfigFile(directory string) (*Config, error) { if err != nil { return nil, err } - return config, nil } @@ -79,5 +81,16 @@ func Init(home string) (*Config, error) { return nil, err } - return ReadConfigFile(directory) + config, err := ReadConfigFile(directory) + if err != nil { + return nil, err + } + + if err := viper.Unmarshal(&config); err != nil { + return nil, err + } + + log.Debug().Object("config", config) + + return config, nil } From 4c2329ae6778ba17633460ce1650673f974a985f Mon Sep 17 00:00:00 2001 From: dahn510 Date: Wed, 25 Sep 2024 09:19:10 -0700 Subject: [PATCH 02/16] config file --- config/files.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/config/files.go b/config/files.go index 50c715e..0dc5b41 100644 --- a/config/files.go +++ b/config/files.go @@ -76,9 +76,16 @@ func Init(home string) (*Config, error) { return nil, err } - err = createFiles(directory) - if err != nil { - return nil, err + viper.SetConfigName(ConfigFileName) + viper.AddConfigPath(directory) + + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + err := createFiles(directory) + if err != nil { + return nil, err + } + } } config, err := ReadConfigFile(directory) From e9bea2fc73d297f95446ca808f5cccc068ecda0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 18:22:10 +0000 Subject: [PATCH 03/16] Bump actions/checkout from 4.1.7 to 4.2.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.7 to 4.2.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.7...v4.2.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- .github/workflows/cloc.yml | 2 +- .github/workflows/cov.yml | 2 +- .github/workflows/golangci.yml | 2 +- .github/workflows/release.yml | 4 ++-- .github/workflows/test-unit.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d129e85..9e07bc1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.0 - uses: actions/setup-go@v5 with: go-version: 1.22.2 # The Go version to download (if necessary) and use. diff --git a/.github/workflows/cloc.yml b/.github/workflows/cloc.yml index 0922f71..ee0a312 100644 --- a/.github/workflows/cloc.yml +++ b/.github/workflows/cloc.yml @@ -18,7 +18,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.0 # Runs djdefi/cloc-action - name: Count Lines of Code (cloc) diff --git a/.github/workflows/cov.yml b/.github/workflows/cov.yml index d9f4a21..aa922c7 100644 --- a/.github/workflows/cov.yml +++ b/.github/workflows/cov.yml @@ -18,7 +18,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.0 - uses: actions/setup-go@v5 with: go-version: 1.22.0 # The Go version to download (if necessary) and use. diff --git a/.github/workflows/golangci.yml b/.github/workflows/golangci.yml index 1781fa0..c9f5582 100644 --- a/.github/workflows/golangci.yml +++ b/.github/workflows/golangci.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-go@v5 with: go-version: 1.22.2 - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.0 - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fe3b1e..f98ea17 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: matrix: os: [ubuntu-20.04, macos-latest] steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.0 - uses: actions/setup-go@v5 with: go-version: 1.22.2 # The Go version to download (if necessary) and use. @@ -36,7 +36,7 @@ jobs: [native-build-cli] runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.0 - name: Get the version id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/} diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2d45a59..fcd4857df 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -10,7 +10,7 @@ jobs: name: Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4.2.0 - uses: actions/setup-go@v5 with: go-version: 1.22.2 # The Go version to download (if necessary) and use. From 74884d95d53e2b7b071790b6ec686c0b09b4199e Mon Sep 17 00:00:00 2001 From: Daniel Ahn Date: Wed, 25 Sep 2024 14:01:31 -0700 Subject: [PATCH 04/16] make config struct compatible with viper config file --- config/files.go | 25 +++++--- config/types.go | 157 +++++++++++++++++++++++++++++++++-------------- config/wallet.go | 5 +- proofs/proofs.go | 2 +- proofs/types.go | 4 +- 5 files changed, 133 insertions(+), 60 deletions(-) diff --git a/config/files.go b/config/files.go index 0dc5b41..be43961 100644 --- a/config/files.go +++ b/config/files.go @@ -9,7 +9,11 @@ import ( "github.com/spf13/viper" ) -const ConfigFileName = "config.yaml" +const ( + ConfigName = "config" + ConfigType = "yaml" + ConfigFileName = ConfigName + "." + ConfigType +) func createIfNotExists(directory string, fileName string, contents []byte) (bool, error) { filePath := path.Join(directory, fileName) @@ -76,7 +80,8 @@ func Init(home string) (*Config, error) { return nil, err } - viper.SetConfigName(ConfigFileName) + viper.SetConfigName(ConfigName) + viper.SetConfigType(ConfigType) viper.AddConfigPath(directory) if err := viper.ReadInConfig(); err != nil { @@ -85,13 +90,17 @@ func Init(home string) (*Config, error) { if err != nil { return nil, err } + } else { + return nil, err } } - - config, err := ReadConfigFile(directory) - if err != nil { - return nil, err - } + /* + config, err := ReadConfigFile(directory) + if err != nil { + return nil, err + } + */ + var config Config if err := viper.Unmarshal(&config); err != nil { return nil, err @@ -99,5 +108,5 @@ func Init(home string) (*Config, error) { log.Debug().Object("config", config) - return config, nil + return &config, nil } diff --git a/config/types.go b/config/types.go index e38df6a..e53cec2 100644 --- a/config/types.go +++ b/config/types.go @@ -1,37 +1,87 @@ package config import ( - "github.com/desmos-labs/cosmos-go-wallet/types" "github.com/rs/zerolog" + "github.com/spf13/viper" ) type Seed struct { SeedPhrase string `json:"seed_phrase"` DerivationPath string `json:"derivation_path"` } + +// required for the mapstructure tag +type ChainConfig struct { + Bech32Prefix string `yaml:"bech32_prefix" mapstructure:"bech32_prefix"` + RPCAddr string `yaml:"rpc_addr" mapstructure:"rpc_addr"` + GRPCAddr string `yaml:"grpc_addr" mapstructure:"grpc_addr"` + GasPrice string `yaml:"gas_price" mapstructure:"gas_price"` + GasAdjustment float64 `yaml:"gas_adjustment" mapstructure:"gas_adjustment"` +} + type Config struct { - QueueInterval int64 `yaml:"queue_interval"` - ProofInterval int64 `yaml:"proof_interval"` - StrayManagerCfg StrayManagerConfig `yaml:"stray_manager"` - ChainCfg types.ChainConfig `yaml:"chain_config"` - Ip string `yaml:"domain"` - TotalSpace int64 `yaml:"total_bytes_offered"` - DataDirectory string `yaml:"data_directory"` - APICfg APIConfig `yaml:"api_config"` - ProofThreads int64 `yaml:"proof_threads"` - BlockStoreConfig BlockStoreConfig `yaml:"block_store_config"` + QueueInterval int64 `yaml:"queue_interval" mapstructure:"queue_interval"` + ProofInterval int64 `yaml:"proof_interval" mapstructure:"proof_interval"` + StrayManagerCfg StrayManagerConfig `yaml:"stray_manager" mapstructure:"stray_manager"` + ChainCfg ChainConfig `yaml:"chain_config" mapstructure:"chain_config"` + Ip string `yaml:"domain" mapstructure:"domain"` + TotalSpace int64 `yaml:"total_bytes_offered" mapstructure:"total_bytes_offered"` + DataDirectory string `yaml:"data_directory" mapstructure:"data_directory"` + APICfg APIConfig `yaml:"api_config" mapstructure:"api_config"` + ProofThreads int16 `yaml:"proof_threads" mapstructure:"proof_threads"` + BlockStoreConfig BlockStoreConfig `yaml:"block_store_config" mapstructure:"block_store_config"` +} + +func DefaultQueueInterval() int64 { + return 10 +} + +func DefaultProofInterval() int64 { + return 120 +} + +func DefaultIP() string { + return "https://example.com" +} + +func DefaultTotalSpace() int64 { + return 1092616192 +} + +func DefaultDataDirectory() string { + return "$HOME/.sequoia/data" +} + +func DefaultProofThreads() int16 { + return 1000 } type StrayManagerConfig struct { - CheckInterval int64 `yaml:"check_interval"` - RefreshInterval int64 `yaml:"refresh_interval"` - HandCount int `yaml:"hands"` + CheckInterval int64 `yaml:"check_interval" mapstructure:"check_interval"` + RefreshInterval int64 `yaml:"refresh_interval" mapstructure:"refresh_interval"` + HandCount int `yaml:"hands" mapstructure:"hands"` +} + +func DefaultStrayManagerConfig() StrayManagerConfig { + return StrayManagerConfig{ + CheckInterval: 30, + RefreshInterval: 120, + HandCount: 2, + } } type APIConfig struct { - Port int64 `yaml:"port"` - IPFSPort int `yaml:"ipfs_port"` - IPFSDomain string `yaml:"ipfs_domain"` + Port int64 `yaml:"port" mapstructure:"port"` + IPFSPort int `yaml:"ipfs_port" mapstructure:"ipfs_port"` + IPFSDomain string `yaml:"ipfs_domain" mapstructure:"ipfs_domain"` +} + +func DefaultAPIConfig() APIConfig { + return APIConfig{ + Port: 3333, + IPFSPort: 4005, + IPFSDomain: "dns4/ipfs.example.com/tcp/4001", + } } const ( @@ -42,9 +92,16 @@ const ( type BlockStoreConfig struct { // *choosing badgerdb as block store will need to use the same directory // for data directory - Directory string `yaml:"directory"` + Directory string `yaml:"directory" mapstructure:"directory"` // data store options: flatfs, badgerdb - Type string `yaml:"type"` + Type string `yaml:"type" mapstructure:"type"` +} + +func DefaultBlockStoreConfig() BlockStoreConfig { + return BlockStoreConfig{ + Directory: "$HOME/.sequoia/blockstore", + Type: OptFlatFS, + } } // LegacyWallet handles keys from earlier versions of storage providers. @@ -56,35 +113,28 @@ type LegacyWallet struct { Address string `json:"address"` } +func DefaultChainConfig() ChainConfig { + return ChainConfig{ + RPCAddr: "http://localhost:26657", + GRPCAddr: "localhost:9090", + GasPrice: "0.02ujkl", + GasAdjustment: 1.5, + Bech32Prefix: "jkl", + } +} + func DefaultConfig() *Config { return &Config{ - QueueInterval: 10, - ProofInterval: 120, - StrayManagerCfg: StrayManagerConfig{ - CheckInterval: 30, - RefreshInterval: 120, - HandCount: 2, - }, - ChainCfg: types.ChainConfig{ - RPCAddr: "http://localhost:26657", - GRPCAddr: "localhost:9090", - GasPrice: "0.02ujkl", - GasAdjustment: 1.5, - Bech32Prefix: "jkl", - }, - Ip: "https://example.com", - TotalSpace: 1092616192, // 1 gib default - DataDirectory: "$HOME/.sequoia/data", - APICfg: APIConfig{ - Port: 3333, - IPFSPort: 4005, - IPFSDomain: "dns4/ipfs.example.com/tcp/4001", - }, - ProofThreads: 1000, - BlockStoreConfig: BlockStoreConfig{ - Directory: "$HOME/.sequoia/blockstore", - Type: OptFlatFS, - }, + QueueInterval: DefaultQueueInterval(), + ProofInterval: DefaultProofInterval(), + StrayManagerCfg: DefaultStrayManagerConfig(), + ChainCfg: DefaultChainConfig(), + Ip: DefaultIP(), + TotalSpace: DefaultTotalSpace(), // 1 gib default + DataDirectory: DefaultDataDirectory(), + APICfg: DefaultAPIConfig(), + ProofThreads: DefaultProofThreads(), + BlockStoreConfig: DefaultBlockStoreConfig(), } } @@ -104,6 +154,19 @@ func (c Config) MarshalZerologObject(e *zerolog.Event) { Int64("APIPort", c.APICfg.Port). Int("APIIPFSPort", c.APICfg.IPFSPort). Str("APIIPFSDomain", c.APICfg.IPFSDomain). - Int64("ProofThreads", c.ProofThreads). + Int16("ProofThreads", c.ProofThreads). Str("BlockstoreBackend", c.BlockStoreConfig.Type) } + +func init() { + viper.SetDefault("QueueInterval", DefaultQueueInterval()) + viper.SetDefault("ProofInterval", DefaultProofInterval()) + viper.SetDefault("StrayManagerCfg", DefaultStrayManagerConfig()) + viper.SetDefault("ChainCfg", DefaultChainConfig()) + viper.SetDefault("Ip", DefaultIP()) + viper.SetDefault("TotalSpace", DefaultTotalSpace()) // 1 gib defaul + viper.SetDefault("DataDirectory", DefaultDataDirectory()) + viper.SetDefault("APICfg", DefaultAPIConfig()) + viper.SetDefault("ProofThreads", DefaultProofThreads()) + viper.SetDefault("BlockStoreConfig", DefaultBlockStoreConfig()) +} diff --git a/config/wallet.go b/config/wallet.go index 9820200..f46998d 100644 --- a/config/wallet.go +++ b/config/wallet.go @@ -7,6 +7,7 @@ import ( sequoiaWallet "github.com/JackalLabs/sequoia/wallet" bip39 "github.com/cosmos/go-bip39" + "github.com/desmos-labs/cosmos-go-wallet/types" "github.com/desmos-labs/cosmos-go-wallet/wallet" jsoniter "github.com/json-iterator/go" @@ -94,7 +95,7 @@ func InitWallet(home string) (*wallet.Wallet, error) { legacyWallet, err := detectLegacyWallet(home) if err == nil { log.Info().Msg("legacy wallet detected") - return sequoiaWallet.CreateWalletPrivKey(legacyWallet.Key, config.ChainCfg) + return sequoiaWallet.CreateWalletPrivKey(legacyWallet.Key, types.ChainConfig(config.ChainCfg)) } err = createWallet(directory) @@ -112,7 +113,7 @@ func InitWallet(home string) (*wallet.Wallet, error) { return nil, err } - return sequoiaWallet.CreateWallet(seed.SeedPhrase, seed.DerivationPath, config.ChainCfg) + return sequoiaWallet.CreateWallet(seed.SeedPhrase, seed.DerivationPath, types.ChainConfig(config.ChainCfg)) } // returns LegacyWallet if "priv_storkey.json" is found at sequoia home directory, diff --git a/proofs/proofs.go b/proofs/proofs.go index b0eed48..5777f13 100644 --- a/proofs/proofs.go +++ b/proofs/proofs.go @@ -298,7 +298,7 @@ func (p *Prover) Stop() { p.running = false } -func NewProver(wallet *wallet.Wallet, q *queue.Queue, io FileSystem, interval int64, threads int64, chunkSize int) *Prover { +func NewProver(wallet *wallet.Wallet, q *queue.Queue, io FileSystem, interval int64, threads int16, chunkSize int) *Prover { p := Prover{ running: false, wallet: wallet, diff --git a/proofs/types.go b/proofs/types.go index 18171e6..a86eedf 100644 --- a/proofs/types.go +++ b/proofs/types.go @@ -16,8 +16,8 @@ type Prover struct { processed time.Time interval int64 io FileSystem - threads int64 - currentThreads int64 + threads int16 + currentThreads int16 chunkSize int } From ce031467b4402015e661533660d529843d448430 Mon Sep 17 00:00:00 2001 From: Daniel Ahn Date: Wed, 25 Sep 2024 14:20:29 -0700 Subject: [PATCH 05/16] rename flags to match the config tags --- cmd/start.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 46515ca..d42af2c 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -63,15 +63,15 @@ func StartCmd() *cobra.Command { } cmd.Flags().Int("restart-attempt", defaultMaxRestartAttempt, "attempt to restart times when the provider fails to start") - cmd.Flags().String("ip", "http://example.com", "provider comain") - cmd.Flags().Int64("apicfg.port", 3333, "port to serve api requests") - cmd.Flags().Int("apicfg.ipfsport", 4005, "port for IPFS") - cmd.Flags().String("apicfg.ipfsdomain", "dns4/ipfs.example.com/tcp/4001", "IPFS domain") - cmd.Flags().Int64("proofthreads", 1000, "maximum threads for proofs") - cmd.Flags().String("datadirectory", "$HOME/.sequoia/data", "directory to store database files") - cmd.Flags().Int64("queueinterval", 10, "seconds to wait until next cycle to flush the transaction queue") - cmd.Flags().Int64("proofinterval", 120, "seconds to wait until next cycle to post proofs") - cmd.Flags().Int64("totalspace", 1092616192, "maximum storage space to provide in bytes") + cmd.Flags().String("domain", "http://example.com", "provider comain") + cmd.Flags().Int64("api_config.port", 3333, "port to serve api requests") + cmd.Flags().Int("api_config.ipfs_port", 4005, "port for IPFS") + cmd.Flags().String("api_config.ipfs_domain", "dns4/ipfs.example.com/tcp/4001", "IPFS domain") + cmd.Flags().Int64("proof_threads", 1000, "maximum threads for proofs") + cmd.Flags().String("data_directory", "$HOME/.sequoia/data", "directory to store database files") + cmd.Flags().Int64("queue_interval", 10, "seconds to wait until next cycle to flush the transaction queue") + cmd.Flags().Int64("proof_interval", 120, "seconds to wait until next cycle to post proofs") + cmd.Flags().Int64("total_bytes_offered", 1092616192, "maximum storage space to provide in bytes") viper.BindPFlags(cmd.Flags()) From 420d0ae1c34895c9c531c7620dc6543ccaf71c98 Mon Sep 17 00:00:00 2001 From: Daniel Ahn Date: Wed, 25 Sep 2024 14:29:20 -0700 Subject: [PATCH 06/16] move flag define to root --- cmd/root.go | 13 +++++++++++++ cmd/start.go | 14 -------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 121c21b..7d5badc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,6 +16,7 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/spf13/viper" ) func init() { @@ -156,6 +157,18 @@ func RootCmd() *cobra.Command { r.PersistentFlags().String(types.FlagHome, types.DefaultHome, "sets the home directory for sequoia") r.PersistentFlags().String(types.FlagLogLevel, types.DefaultLogLevel, "log level. info|error|debug") + r.PersistentFlags().Int("restart-attempt", defaultMaxRestartAttempt, "attempt to restart times when the provider fails to start") + r.PersistentFlags().String("domain", "http://example.com", "provider comain") + r.PersistentFlags().Int64("api_config.port", 3333, "port to serve api requests") + r.PersistentFlags().Int("api_config.ipfs_port", 4005, "port for IPFS") + r.PersistentFlags().String("api_config.ipfs_domain", "dns4/ipfs.example.com/tcp/4001", "IPFS domain") + r.PersistentFlags().Int64("proof_threads", 1000, "maximum threads for proofs") + r.PersistentFlags().String("data_directory", "$HOME/.sequoia/data", "directory to store database files") + r.PersistentFlags().Int64("queue_interval", 10, "seconds to wait until next cycle to flush the transaction queue") + r.PersistentFlags().Int64("proof_interval", 120, "seconds to wait until next cycle to post proofs") + r.PersistentFlags().Int64("total_bytes_offered", 1092616192, "maximum storage space to provide in bytes") + + viper.BindPFlags(r.PersistentFlags()) r.AddCommand(StartCmd(), wallet.WalletCmd(), InitCmd(), VersionCmd(), IPFSCmd(), SalvageCmd(), ShutdownCmd()) diff --git a/cmd/start.go b/cmd/start.go index d42af2c..91d182f 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -9,7 +9,6 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/spf13/viper" ) const defaultMaxRestartAttempt = 60 @@ -62,18 +61,5 @@ func StartCmd() *cobra.Command { }, } - cmd.Flags().Int("restart-attempt", defaultMaxRestartAttempt, "attempt to restart times when the provider fails to start") - cmd.Flags().String("domain", "http://example.com", "provider comain") - cmd.Flags().Int64("api_config.port", 3333, "port to serve api requests") - cmd.Flags().Int("api_config.ipfs_port", 4005, "port for IPFS") - cmd.Flags().String("api_config.ipfs_domain", "dns4/ipfs.example.com/tcp/4001", "IPFS domain") - cmd.Flags().Int64("proof_threads", 1000, "maximum threads for proofs") - cmd.Flags().String("data_directory", "$HOME/.sequoia/data", "directory to store database files") - cmd.Flags().Int64("queue_interval", 10, "seconds to wait until next cycle to flush the transaction queue") - cmd.Flags().Int64("proof_interval", 120, "seconds to wait until next cycle to post proofs") - cmd.Flags().Int64("total_bytes_offered", 1092616192, "maximum storage space to provide in bytes") - - viper.BindPFlags(cmd.Flags()) - return cmd } From 9ef76934ee4f0e3340f1866f5b5e0f347c2148f4 Mon Sep 17 00:00:00 2001 From: Daniel Ahn Date: Wed, 25 Sep 2024 14:48:07 -0700 Subject: [PATCH 07/16] lint --- cmd/root.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 7d5badc..e5a579a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -168,7 +168,10 @@ func RootCmd() *cobra.Command { r.PersistentFlags().Int64("proof_interval", 120, "seconds to wait until next cycle to post proofs") r.PersistentFlags().Int64("total_bytes_offered", 1092616192, "maximum storage space to provide in bytes") - viper.BindPFlags(r.PersistentFlags()) + err := viper.BindPFlags(r.PersistentFlags()) + if err != nil { + panic(err) + } r.AddCommand(StartCmd(), wallet.WalletCmd(), InitCmd(), VersionCmd(), IPFSCmd(), SalvageCmd(), ShutdownCmd()) From e6952462757d28db23f0f970aa4f7b33b9a78fa0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 18:57:44 +0000 Subject: [PATCH 08/16] Bump codecov/codecov-action from 4.5.0 to 4.6.0 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4.5.0...v4.6.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/cov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cov.yml b/.github/workflows/cov.yml index aa922c7..4a9ad2b 100644 --- a/.github/workflows/cov.yml +++ b/.github/workflows/cov.yml @@ -30,7 +30,7 @@ jobs: go tool cover -func coverage3.out rm coverage2.out rm coverage.out - - uses: codecov/codecov-action@v4.5.0 + - uses: codecov/codecov-action@v4.6.0 with: token: ${{ secrets.CODECOV }} From b9d382fefe631a39a360bbca05e5a282fa64c89c Mon Sep 17 00:00:00 2001 From: marston Date: Fri, 11 Oct 2024 15:58:39 -0400 Subject: [PATCH 09/16] adding pagination fix --- strays/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/strays/manager.go b/strays/manager.go index f8039be..eaa92d5 100644 --- a/strays/manager.go +++ b/strays/manager.go @@ -182,7 +182,7 @@ func (s *StrayManager) RefreshList() error { s.strays = append(s.strays, &newStray) } - s.lastSize = res.Size() + s.lastSize = int(res.Pagination.Total) return nil } From fa9987c3a3fd0a30e13ffa9d4e3831b3135f0c10 Mon Sep 17 00:00:00 2001 From: marston Date: Wed, 16 Oct 2024 14:13:17 -0400 Subject: [PATCH 10/16] updating canine --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index fed637e..224fb61 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ipld-format v0.6.0 - github.com/jackalLabs/canine-chain/v4 v4.0.3 + github.com/jackalLabs/canine-chain/v4 v4.1.0 github.com/json-iterator/go v1.1.12 github.com/libp2p/go-libp2p v0.32.2 github.com/multiformats/go-multiaddr v0.12.3 @@ -27,6 +27,7 @@ require ( github.com/rs/cors v1.11.0 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.0 + github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.9.0 github.com/tendermint/tendermint v0.34.27 github.com/wealdtech/go-merkletree/v2 v2.6.0 @@ -230,7 +231,6 @@ require ( github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.16.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect diff --git a/go.sum b/go.sum index 9d5a55b..147a5b6 100644 --- a/go.sum +++ b/go.sum @@ -859,8 +859,8 @@ github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5Uybo github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= -github.com/jackalLabs/canine-chain/v4 v4.0.3 h1:cfn0qm3fEdL+XYaXGdP/S33bkt8jTxGAQnBwOEy+0OI= -github.com/jackalLabs/canine-chain/v4 v4.0.3/go.mod h1:xl48on9+oEDnE2/1A1miPVXhYvxf10ApJUHunKZeiHI= +github.com/jackalLabs/canine-chain/v4 v4.1.0 h1:3PA7UNn2Nu7HHY+LAnpCMQxg32GyRt+F6ntQTLIve/A= +github.com/jackalLabs/canine-chain/v4 v4.1.0/go.mod h1:td0cUHEvmWwqTA+pS8ItQzZxZqFVx/RGqb8MMgOAq+w= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= From 3833a4cf75c8b1a17921b88fccecc17956154a9e Mon Sep 17 00:00:00 2001 From: marston Date: Thu, 17 Oct 2024 14:25:56 -0400 Subject: [PATCH 11/16] changing to page request --- go.mod | 2 +- go.sum | 4 ++-- recycle/recycle.go | 13 +++++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 224fb61..b0d22e0 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ipld-format v0.6.0 - github.com/jackalLabs/canine-chain/v4 v4.1.0 + github.com/jackalLabs/canine-chain/v4 v4.1.1 github.com/json-iterator/go v1.1.12 github.com/libp2p/go-libp2p v0.32.2 github.com/multiformats/go-multiaddr v0.12.3 diff --git a/go.sum b/go.sum index 147a5b6..b54d78f 100644 --- a/go.sum +++ b/go.sum @@ -859,8 +859,8 @@ github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5Uybo github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= -github.com/jackalLabs/canine-chain/v4 v4.1.0 h1:3PA7UNn2Nu7HHY+LAnpCMQxg32GyRt+F6ntQTLIve/A= -github.com/jackalLabs/canine-chain/v4 v4.1.0/go.mod h1:td0cUHEvmWwqTA+pS8ItQzZxZqFVx/RGqb8MMgOAq+w= +github.com/jackalLabs/canine-chain/v4 v4.1.1 h1:dUkk9kclOLC2Ku3QjNcnya5B7fo13MWPZEVy6qJ8lF8= +github.com/jackalLabs/canine-chain/v4 v4.1.1/go.mod h1:td0cUHEvmWwqTA+pS8ItQzZxZqFVx/RGqb8MMgOAq+w= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= diff --git a/recycle/recycle.go b/recycle/recycle.go index 3ec547c..382e840 100644 --- a/recycle/recycle.go +++ b/recycle/recycle.go @@ -6,11 +6,14 @@ import ( "fmt" "io" "io/fs" + "math/rand" "os" "path/filepath" "strings" "time" + "github.com/cosmos/cosmos-sdk/types/query" + "github.com/JackalLabs/jackal-provider/jprov/archive" "github.com/jackalLabs/canine-chain/v4/x/storage/types" "github.com/rs/zerolog/log" @@ -174,13 +177,19 @@ func record(file io.Writer, merkle []byte, size int, fid string) error { func (r *RecycleDepot) collectOpenFiles() ([]types.UnifiedFile, error) { req := &types.QueryOpenFiles{ ProviderAddress: r.address, - Pagination: nil, + Pagination: &query.PageRequest{ + Key: nil, + Offset: 0, + Limit: 300, + CountTotal: true, + Reverse: rand.Int63n(2) == 0, + }, } resp, err := r.queryClient.OpenFiles(context.Background(), req) if err != nil { return nil, err } - log.Info().Msgf("We found %d files open", len(resp.Files)) + log.Info().Msgf("We found %d files open", resp.Pagination.Total) return resp.Files, nil } From b934ee68fd0e07d31c585413b1dcf77a0a48d4f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:02:34 +0000 Subject: [PATCH 12/16] Bump actions/checkout from 4.2.0 to 4.2.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.0 to 4.2.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.2.0...v4.2.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- .github/workflows/cloc.yml | 2 +- .github/workflows/cov.yml | 2 +- .github/workflows/golangci.yml | 2 +- .github/workflows/release.yml | 4 ++-- .github/workflows/test-unit.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e07bc1..96b226d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-go@v5 with: go-version: 1.22.2 # The Go version to download (if necessary) and use. diff --git a/.github/workflows/cloc.yml b/.github/workflows/cloc.yml index ee0a312..1caeb3a 100644 --- a/.github/workflows/cloc.yml +++ b/.github/workflows/cloc.yml @@ -18,7 +18,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.2 # Runs djdefi/cloc-action - name: Count Lines of Code (cloc) diff --git a/.github/workflows/cov.yml b/.github/workflows/cov.yml index aa922c7..1473434 100644 --- a/.github/workflows/cov.yml +++ b/.github/workflows/cov.yml @@ -18,7 +18,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-go@v5 with: go-version: 1.22.0 # The Go version to download (if necessary) and use. diff --git a/.github/workflows/golangci.yml b/.github/workflows/golangci.yml index c9f5582..aebb620 100644 --- a/.github/workflows/golangci.yml +++ b/.github/workflows/golangci.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-go@v5 with: go-version: 1.22.2 - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.2 - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f98ea17..dd1a00a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: matrix: os: [ubuntu-20.04, macos-latest] steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-go@v5 with: go-version: 1.22.2 # The Go version to download (if necessary) and use. @@ -36,7 +36,7 @@ jobs: [native-build-cli] runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.2 - name: Get the version id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/} diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index fcd4857df..210c45a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -10,7 +10,7 @@ jobs: name: Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.0 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-go@v5 with: go-version: 1.22.2 # The Go version to download (if necessary) and use. From abf5d6961052be01ddebb6a1247d310465bcddbf Mon Sep 17 00:00:00 2001 From: marston Date: Thu, 24 Oct 2024 11:09:40 -0400 Subject: [PATCH 13/16] fixing hands --- strays/hands.go | 4 ++-- strays/manager.go | 4 ++-- strays/types.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/strays/hands.go b/strays/hands.go index e510758..6b894b7 100644 --- a/strays/hands.go +++ b/strays/hands.go @@ -24,8 +24,8 @@ func (h *Hand) Stop() { func (h *Hand) Start(f *file_system.FileSystem, wallet *wallet.Wallet, myUrl string, chunkSize int64) { h.running = true defer log.Info().Msg("Hand stopped") - for !h.running { - if h.running { + for h.running { + if !h.running { return } diff --git a/strays/manager.go b/strays/manager.go index eaa92d5..e259527 100644 --- a/strays/manager.go +++ b/strays/manager.go @@ -155,7 +155,7 @@ func (s *StrayManager) RefreshList() error { var val uint64 if s.lastSize > 300 { - val = uint64(s.rand.Int63n(int64(s.lastSize))) + val = uint64(s.rand.Int63n(s.lastSize)) } page := &query.PageRequest{ // more randomly pick from the stray pile @@ -182,7 +182,7 @@ func (s *StrayManager) RefreshList() error { s.strays = append(s.strays, &newStray) } - s.lastSize = int(res.Pagination.Total) + s.lastSize = int64(res.Pagination.Total) return nil } diff --git a/strays/types.go b/strays/types.go index 604c288..a5cebcf 100644 --- a/strays/types.go +++ b/strays/types.go @@ -18,7 +18,7 @@ type Hand struct { type StrayManager struct { strays []*types.UnifiedFile wallet *wallet.Wallet - lastSize int + lastSize int64 rand *rand.Rand interval time.Duration refreshInterval time.Duration From 03a0c8696a8bfa02fa46ddd44d9b2b08ca7a6907 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:02:37 +0000 Subject: [PATCH 14/16] Bump codecov/codecov-action from 4.6.0 to 5.0.0 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.6.0 to 5.0.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4.6.0...v5.0.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cov.yml b/.github/workflows/cov.yml index d890fa4..551220b 100644 --- a/.github/workflows/cov.yml +++ b/.github/workflows/cov.yml @@ -30,7 +30,7 @@ jobs: go tool cover -func coverage3.out rm coverage2.out rm coverage.out - - uses: codecov/codecov-action@v4.6.0 + - uses: codecov/codecov-action@v5.0.0 with: token: ${{ secrets.CODECOV }} From fa41e4955aba1a9ca46074c132699f3a82f5c30b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 18:11:03 +0000 Subject: [PATCH 15/16] Bump codecov/codecov-action from 5.0.0 to 5.0.2 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.0.0 to 5.0.2. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.0.0...v5.0.2) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/cov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cov.yml b/.github/workflows/cov.yml index 551220b..3e38624 100644 --- a/.github/workflows/cov.yml +++ b/.github/workflows/cov.yml @@ -30,7 +30,7 @@ jobs: go tool cover -func coverage3.out rm coverage2.out rm coverage.out - - uses: codecov/codecov-action@v5.0.0 + - uses: codecov/codecov-action@v5.0.2 with: token: ${{ secrets.CODECOV }} From e195ad421656ecf4307fef6fb8c3711bfaead87a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:01:20 +0000 Subject: [PATCH 16/16] Bump codecov/codecov-action from 5.0.2 to 5.0.7 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.0.2 to 5.0.7. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.0.2...v5.0.7) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/cov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cov.yml b/.github/workflows/cov.yml index 3e38624..8dc45db 100644 --- a/.github/workflows/cov.yml +++ b/.github/workflows/cov.yml @@ -30,7 +30,7 @@ jobs: go tool cover -func coverage3.out rm coverage2.out rm coverage.out - - uses: codecov/codecov-action@v5.0.2 + - uses: codecov/codecov-action@v5.0.7 with: token: ${{ secrets.CODECOV }}