Merge branch 'main' into blanket-integrations

.github/workflows/build.yml

@@ -10,7 +10,7 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
       - uses: actions/setup-go@v4
         with:
           go-version: 1.19 # The Go version to download (if necessary) and use.

.github/workflows/cloc.yml

@@ -18,7 +18,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/[email protected].0
+    - uses: actions/[email protected].2
 
     # Runs djdefi/cloc-action
     - name: Count Lines of Code (cloc)

.github/workflows/cov.yml

@@ -18,7 +18,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-    - uses: actions/[email protected].0
+    - uses: actions/[email protected].2
     - uses: actions/setup-go@v4
       with:
         go-version: 1.19 # The Go version to download (if necessary) and use.

.github/workflows/golangci.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: 1.19
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:

.github/workflows/nightly.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         os: [ubuntu-20.04, macos-latest]
     steps:
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
       - uses: actions/setup-go@v4
         with:
           go-version: 1.18.1 # The Go version to download (if necessary) and use.
@@ -29,7 +29,7 @@ jobs:
       [native-build-cli]
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
       - uses: actions/download-artifact@v3
         with:
           name: jprovd-Linux

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
         os: [ubuntu-20.04, macos-latest]
     steps:
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
       - uses: actions/setup-go@v4
         with:
           go-version: 1.18.1 # The Go version to download (if necessary) and use.
@@ -31,7 +31,7 @@ jobs:
   # MacOS-ARM64-CLI:
   #   runs-on: ubuntu-20.04
   #   steps:
-  #     - uses: actions/[email protected].0
+  #     - uses: actions/[email protected].2
   #     - uses: actions/setup-go@v4
   #       with:
   #         go-version: 1.19 # The Go version to download (if necessary) and use.
@@ -54,7 +54,7 @@ jobs:
       [native-build-cli]
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
       - name: Get the version
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/}

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
       - uses: actions/setup-go@v4
         with:
           go-version: 1.19 # The Go version to download (if necessary) and use.

go.mod

@@ -7,11 +7,11 @@ require (
 	github.com/cosmos/cosmos-sdk v0.45.11
 	github.com/cosmos/go-bip39 v1.0.0
 	github.com/huin/goupnp v1.0.3
-	github.com/jackalLabs/canine-chain v1.2.0-beta.7
+	github.com/jackalLabs/canine-chain v0.0.0-00010101000000-000000000000
 	github.com/julienschmidt/httprouter v1.3.0
-	github.com/rs/cors v1.8.2
-	github.com/rs/zerolog v1.27.0
-	github.com/spf13/cobra v1.6.1
+	github.com/rs/cors v1.9.0
+	github.com/rs/zerolog v1.29.1
+	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.14.0
 	github.com/stretchr/testify v1.8.2
@@ -90,7 +90,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect
 	github.com/improbable-eng/grpc-web v0.15.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.15 // indirect

go.sum

@@ -231,7 +231,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44=
@@ -632,8 +632,8 @@ github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/C
 github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ=
 github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
-github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY=
 github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI=
 github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8=
@@ -1024,11 +1024,11 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
-github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
-github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
-github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
-github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
+github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
+github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
+github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -1082,8 +1082,8 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
-github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
-github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=

jprov/jprovd/provider_commands.go

@@ -35,16 +35,17 @@ func StartServerCommand() *cobra.Command {
 	cmd.Flags().Int(types.FlagPort, 3333, "Port to host the server on.")
 	cmd.Flags().String(types.VersionFlag, "", "The value exposed by the version api to allow for custom deployments.")
 	cmd.Flags().Bool(types.HaltStraysFlag, false, "Debug flag to stop picking up strays.")
-	cmd.Flags().Uint16(types.FlagInterval, 32, "The interval in seconds for which to check proofs.")
+	cmd.Flags().Uint16(types.FlagInterval, 32, "The interval in seconds for which to check proofs. Must be >=1800 if you need a custom interval")
 	cmd.Flags().Uint(types.FlagThreads, 10, "The amount of stray threads.")
 	cmd.Flags().Int(types.FlagMaxMisses, 16, "The amount of intervals a provider can miss their proofs before removing a file.")
-	cmd.Flags().Int64(types.FlagChunkSize, 10240, "The size of a single chunk.")
+	cmd.Flags().Int64(types.FlagChunkSize, 10240, "The size of a single file chunk.")
 	cmd.Flags().Int64(types.FlagStrayInterval, 20, "The interval in seconds to check for new strays.")
-	cmd.Flags().Int(types.FlagMessageSize, 500_000, "The max message size in bytes to submit to the chain at one time.")
+	cmd.Flags().Int(types.FlagMessageSize, 500_000, "The max size of all messages in bytes to submit to the chain at one time.")
 	cmd.Flags().Int(types.FlagGasCap, 20_000, "The maximum gas to be used per message.")
 	cmd.Flags().Int(types.FlagMaxFileSize, 32000, "The maximum size allowed to be sent to this provider in mbs. (only for monitoring services)")
 	cmd.Flags().Int64(types.FlagQueueInterval, 4, "The time, in seconds, between running a queue loop.")
 	cmd.Flags().String(types.FlagProviderName, "A Storage Provider", "The name to identify this provider in block explorers.")
+	cmd.Flags().Int64(types.FlagSleep, 250, "The time, in milliseconds, before adding another proof msg to the queue.")
 	return cmd
 }
 

jprov/queue/queue.go

@@ -83,6 +83,7 @@ func (q *UploadQueue) listenOnce(cmd *cobra.Command, providerName string) {
 	}
 
 	clientCtx := client.GetClientContextFromCmd(cmd)
+	ctx.Logger.Debug(fmt.Sprintf("total no. of msgs in proof transaction is: %d", len(msgs)))
 
 	res, err := utils.SendTx(clientCtx, cmd.Flags(), fmt.Sprintf("Storage Provided by %s", providerName), msgs...)
 	for _, v := range uploads {

jprov/server/http.go

@@ -53,15 +53,19 @@ func checkVersion(cmd *cobra.Command, w http.ResponseWriter, ctx *utils.Context)
 		ctx.Logger.Error(err.Error())
 	}
 
-	var v types.VersionResponse
+	clientCtx, error := client.GetClientTxContext(cmd)
+	if error != nil {
+		ctx.Logger.Error(err.Error())
+		return
+	}
+	chainID := clientCtx.ChainID
+
+	v := types.VersionResponse{
+		Version: version.Version,
+		ChainID: chainID,
+	}
 	if len(res) > 0 {
-		v = types.VersionResponse{
-			Version: res,
-		}
-	} else {
-		v = types.VersionResponse{
-			Version: version.Version,
-		}
+		v.Version = res
 	}
 
 	err = json.NewEncoder(w).Encode(v)

jprov/server/proofs.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math/rand"
 	"os"
 	"strconv"
 	"strings"
@@ -144,7 +145,7 @@ func postProof(clientCtx client.Context, cid string, block string, db *leveldb.D
 }
 
 func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *utils.Context) {
-	interval, err := cmd.Flags().GetUint16(types.FlagInterval)
+	intervalFromCMD, err := cmd.Flags().GetUint16(types.FlagInterval)
 	if err != nil {
 		ctx.Logger.Error(err.Error())
 		return
@@ -168,28 +169,19 @@ func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *u
 		return
 	}
 
-	//type IterWrap struct {
-	//	Key   []byte
-	//	Value []byte
-	//}
-
 	for {
+		interval := intervalFromCMD
+
+		if interval < 1800 { // If the provider picked an interval that's less than 30 minutes, we generate a random interval for them anyways
+
+			r := rand.New(rand.NewSource(time.Now().UnixNano()))
+			interval = uint16(r.Intn(901) + 900) // Generate interval between 15-30 minutes
+
+		}
+		ctx.Logger.Debug(fmt.Sprintf("The interval between proofs is now %d", interval))
 		start := time.Now()
-		// ctx.Logger.Info(fmt.Sprintf("Starting proof commitment at %s", start.Format("2006-01-02 15:04:05.000000")))
-		// m := []IterWrap{}
+
 		iter := db.NewIterator(nil, nil)
-		//for iter.Next() {
-		//	mm := IterWrap{
-		//		Key:   iter.Key(),
-		//		Value: iter.Value(),
-		//	}
-		//	m = append(m, mm)
-		//}
-		//iter.Release()
-		//err = iter.Error()
-		//if err != nil {
-		//	ctx.Logger.Error("Iterator Error: %s", err.Error())
-		//}
 
 		for iter.Next() {
 			cid := string(iter.Key())
@@ -198,7 +190,6 @@ func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *u
 			if cid[:len(utils.FileKey)] != utils.FileKey {
 				continue
 			}
-			fid := value
 
 			cid = cid[len(utils.FileKey):]
 
@@ -245,7 +236,7 @@ func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *u
 							break
 						}
 					}
-					ctx.Logger.Info(fmt.Sprintf("%s is being removed", value))
+					ctx.Logger.Info(fmt.Sprintf("%s is being removed", cid))
 
 					if !duplicate {
 						ctx.Logger.Info("And we are removing the file on disk.")
@@ -254,17 +245,19 @@ func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *u
 						if err != nil {
 							ctx.Logger.Error(err.Error())
 						}
+
+						err = os.Remove(utils.GetStoragePathForTree(clientCtx, value))
+						if err != nil {
+							ctx.Logger.Error(err.Error())
+							continue
+						}
 					}
 					err = db.Delete(utils.MakeFileKey(cid), nil)
 					if err != nil {
 						ctx.Logger.Error(err.Error())
 						continue
 					}
-					err = os.Remove(utils.GetStoragePathForTree(clientCtx, fid))
-					if err != nil {
-						ctx.Logger.Error(err.Error())
-						continue
-					}
+
 					err = db.Delete(utils.MakeDowntimeKey(cid), nil)
 					if err != nil {
 						ctx.Logger.Error(err.Error())
@@ -317,6 +310,12 @@ func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *u
 				ctx.Logger.Error(fmt.Sprintf("Posting Proof Error: %v", err))
 				continue
 			}
+			sleep, err := cmd.Flags().GetInt64(types.FlagSleep)
+			if err != nil {
+				ctx.Logger.Error(err.Error())
+				return
+			}
+			time.Sleep(time.Duration(sleep) * time.Millisecond)
 
 		}
 

jprov/strays/hand_process.go

@@ -68,19 +68,19 @@ func (h *LittleHand) Process(ctx *utils.Context, m *StrayManager) { // process t
 			return // If we don't have it and nobody else does, there is nothing we can do.
 		}
 	} else { // If there are providers with this file, we will download it from them instead to keep things consistent
-		if _, err := os.Stat(utils.GetStoragePath(h.ClientContext, h.Stray.Fid)); !os.IsNotExist(err) {
-			ctx.Logger.Info("Already have this file")
-			return
+
+		for _, prov := range arr {
+			if prov == m.Ip { // Ignore ourselves
+				return
+			}
 		}
 
 		found := false
 		for _, prov := range arr { // Check every provider for the file, not just trust chain data.
 			if found {
 				continue
 			}
-			if prov == m.Ip { // Ignore ourselves
-				return
-			}
+
 			_, err = utils.DownloadFileFromURL(h.Cmd, prov, h.Stray.Fid, h.Stray.Cid, h.Database, ctx.Logger)
 			if err != nil {
 				ctx.Logger.Error(err.Error())
@@ -327,8 +327,8 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command) {
 		return
 	}
 
-	rand.Seed(time.Now().UnixNano())
-	rand.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] })
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	r.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] })
 
 	for _, newStray := range s { // Only add new strays to the queue
 

jprov/types/flags.go

@@ -12,4 +12,5 @@ const (
 	FlagMaxFileSize   = "max-file-size"
 	FlagQueueInterval = "queue-interval"
 	FlagProviderName  = "moniker"
+	FlagSleep         = "sleep"
 )

jprov/types/types.go

@@ -31,6 +31,7 @@ type ErrorResponse struct {
 
 type VersionResponse struct {
 	Version string `json:"version"`
+	ChainID string `json:"chain-id"`
 }
 
 type Message interface{}