From 363b417bfc9f30b8402a17cb26fd10c7d5cf9ac6 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 11:48:40 -0400 Subject: [PATCH 01/28] fid now shows so you can manually check --- jprov/strays/hand_process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jprov/strays/hand_process.go b/jprov/strays/hand_process.go index ccde506..6b3ee93 100644 --- a/jprov/strays/hand_process.go +++ b/jprov/strays/hand_process.go @@ -64,7 +64,7 @@ func (h *LittleHand) Process(ctx *utils.Context, m *StrayManager) { // process t the cached file from our filesystem which keeps the file alive) */ if _, err := os.Stat(utils.GetStoragePath(h.ClientContext, h.Stray.Fid)); os.IsNotExist(err) { - ctx.Logger.Info("Nobody, not even I have the file.") + ctx.Logger.Info(fmt.Sprintf("Nobody, not even I have %s.", h.Stray.Fid)) return // If we don't have it and nobody else does, there is nothing we can do. } } else { // If there are providers with this file, we will download it from them instead to keep things consistent From 68d5b188a3aea802103a14b021fbcf15b62baa43 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 11:55:33 -0400 Subject: [PATCH 02/28] cleaning out stray list --- jprov/strays/hand_process.go | 31 ------------------------------- jprov/strays/stray_manager.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/jprov/strays/hand_process.go b/jprov/strays/hand_process.go index 6b3ee93..586139d 100644 --- a/jprov/strays/hand_process.go +++ b/jprov/strays/hand_process.go @@ -4,9 +4,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "math/rand" "os" - "time" "github.com/JackalLabs/jackal-provider/jprov/crypto" "github.com/JackalLabs/jackal-provider/jprov/utils" @@ -17,7 +15,6 @@ import ( "github.com/cosmos/cosmos-sdk/types/tx/signing" authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" storageTypes "github.com/jackalLabs/canine-chain/x/storage/types" - "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -309,31 +306,3 @@ func (h *LittleHand) Sign(txf txns.Factory, clientCtx client.Context, index byte prevSignatures = append(prevSignatures, sig) return txBuilder.SetSignatures(prevSignatures...) } - -func (m *StrayManager) CollectStrays(cmd *cobra.Command) { - m.Context.Logger.Info("Collecting strays from chain...") - qClient := storageTypes.NewQueryClient(m.ClientContext) - - res, err := qClient.StraysAll(cmd.Context(), &storageTypes.QueryAllStraysRequest{}) - if err != nil { - m.Context.Logger.Error(err.Error()) - return - } - - s := res.Strays - - if len(s) == 0 { // If there are no strays, the network has claimed them all. We will try again later. - m.Context.Logger.Info("No strays found.") - return - } - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - r.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] }) - - for _, newStray := range s { // Only add new strays to the queue - - k := newStray - m.Strays = append(m.Strays, &k) - - } -} diff --git a/jprov/strays/stray_manager.go b/jprov/strays/stray_manager.go index bfb32ea..b797c13 100644 --- a/jprov/strays/stray_manager.go +++ b/jprov/strays/stray_manager.go @@ -2,6 +2,7 @@ package strays import ( "fmt" + "math/rand" "time" "github.com/JackalLabs/jackal-provider/jprov/crypto" @@ -164,6 +165,36 @@ func (m *StrayManager) Init(cmd *cobra.Command, count uint, db *leveldb.DB) { // fmt.Println("Finished Initialization...") } +func (m *StrayManager) CollectStrays(cmd *cobra.Command) { + m.Context.Logger.Info("Collecting strays from chain...") + qClient := storageTypes.NewQueryClient(m.ClientContext) + + res, err := qClient.StraysAll(cmd.Context(), &storageTypes.QueryAllStraysRequest{}) + if err != nil { + m.Context.Logger.Error(err.Error()) + return + } + + s := res.Strays + + if len(s) == 0 { // If there are no strays, the network has claimed them all. We will try again later. + m.Context.Logger.Info("No strays found.") + return + } + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + r.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] }) + + m.Strays = make([]*storageTypes.Strays, 0) + + for _, newStray := range s { // Only add new strays to the queue + + k := newStray + m.Strays = append(m.Strays, &k) + + } +} + func (m *StrayManager) Start(cmd *cobra.Command) { // loop through stray system tm, err := cmd.Flags().GetInt64(types.FlagStrayInterval) if err != nil { From 99d129cf7cc1058c573a37a6a08b4f157d936547 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 12:59:08 -0400 Subject: [PATCH 03/28] won't start without other providers approving --- jprov/api/network/proxy.go | 51 ++++++++++++++++++++++++++++++++++++ jprov/server/file_server.go | 14 +++++++++- jprov/server/utils.go | 52 +++++++++++++++++++++++++++++++++++++ jprov/types/types.go | 4 +++ 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 jprov/api/network/proxy.go diff --git a/jprov/api/network/proxy.go b/jprov/api/network/proxy.go new file mode 100644 index 0000000..cb9ec8b --- /dev/null +++ b/jprov/api/network/proxy.go @@ -0,0 +1,51 @@ +package network + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/JackalLabs/jackal-provider/jprov/types" + "github.com/julienschmidt/httprouter" + "github.com/spf13/cobra" +) + +func GetProxy(cmd *cobra.Command, w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + + ok := true + var versionRes types.VersionResponse + var versionUrl *url.URL + var res *http.Response + + queries := r.URL.Query() + uri := queries.Get("route") + + u, err := url.Parse(uri) + if err != nil { + ok = false + goto skip + } + + versionUrl = u.JoinPath("version") + res, err = http.Get(versionUrl.String()) + if err != nil { + ok = false + goto skip + } + err = json.NewDecoder(res.Body).Decode(&versionRes) + if err != nil { + ok = false + goto skip + } + +skip: + + okRes := types.ProxyResponse{ + Ok: ok, + } + err = json.NewEncoder(w).Encode(okRes) + if err != nil { + fmt.Println(err) + } +} diff --git a/jprov/server/file_server.go b/jprov/server/file_server.go index 7087f36..5515442 100644 --- a/jprov/server/file_server.go +++ b/jprov/server/file_server.go @@ -159,12 +159,18 @@ func StartFileServer(cmd *cobra.Command) { Address: address, } - _, err = queryClient.Providers(context.Background(), params) + me, err := queryClient.Providers(context.Background(), params) if err != nil { fmt.Println("Provider not initialized on the blockchain, or connection to the RPC node has been lost. Please make sure your RPC node is available then run `jprovd init` to fix this.") return } + providers, err := queryClient.ProvidersAll(context.Background(), &storageTypes.QueryAllProvidersRequest{}) + if err != nil { + fmt.Println("Cannot connect to jackal blockchain.") + return + } + path := utils.GetDataPath(clientCtx) db, dberr := leveldb.OpenFile(path, nil) @@ -201,6 +207,12 @@ func StartFileServer(cmd *cobra.Command) { providerName = "A Storage Provider" } + connected := testConnection(providers.Providers, me.Providers.Ip) + if !connected { + fmt.Println("Domain not configured correctly, make sure your domain points to your provider.") + return + } + manager := strays.NewStrayManager(cmd) // creating and starting the stray management system if !strs { manager.Init(cmd, threads, db) diff --git a/jprov/server/utils.go b/jprov/server/utils.go index c8b23ec..e207393 100644 --- a/jprov/server/utils.go +++ b/jprov/server/utils.go @@ -2,15 +2,67 @@ package server import ( "context" + "encoding/json" "fmt" + "net/http" + "net/url" "strconv" + "github.com/JackalLabs/jackal-provider/jprov/types" "github.com/cosmos/cosmos-sdk/client" storageTypes "github.com/jackalLabs/canine-chain/x/storage/types" ) const ErrNotYours = "not your deal" +func testConnection(providers []storageTypes.Providers, ip string) bool { + onlineProviders := 0 + respondingProvider := 0 + outdatedProvider := 0 + + for _, provider := range providers { + u, err := url.Parse(provider.Ip) + if err != nil { + continue + } + versionUrl := u.JoinPath("version") + r, err := http.Get(versionUrl.String()) + var versionRes types.VersionResponse + err = json.NewDecoder(r.Body).Decode(&versionRes) + if err != nil { + continue + } + onlineProviders++ + + proxyUrl := u.JoinPath("checkme") + vals := proxyUrl.Query() + vals.Add("route", ip) + proxyUrl.RawQuery = vals.Encode() + r, err = http.Get(proxyUrl.String()) + if err != nil { + outdatedProvider++ + continue + } + var proxyRes types.ProxyResponse + err = json.NewDecoder(r.Body).Decode(&proxyRes) + if err != nil { + continue + } + + respondingProvider++ + } + + if respondingProvider < 2 && (onlineProviders-outdatedProvider) < 3 { + return true + } + + if respondingProvider < 2 { + return false + } + + return true +} + func queryBlock(clientCtx *client.Context, cid string) (string, error) { queryClient := storageTypes.NewQueryClient(clientCtx) diff --git a/jprov/types/types.go b/jprov/types/types.go index 343ed30..527c866 100644 --- a/jprov/types/types.go +++ b/jprov/types/types.go @@ -34,6 +34,10 @@ type VersionResponse struct { ChainID string `json:"chain-id"` } +type ProxyResponse struct { + Ok bool `json:"ok"` +} + type Message interface{} type Upload struct { From ebc4ebae0b779ff505a1e0c6af96d1f05e65f92c Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 13:04:18 -0400 Subject: [PATCH 04/28] fixing nil --- jprov/server/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jprov/server/utils.go b/jprov/server/utils.go index e207393..002857f 100644 --- a/jprov/server/utils.go +++ b/jprov/server/utils.go @@ -27,6 +27,9 @@ func testConnection(providers []storageTypes.Providers, ip string) bool { } versionUrl := u.JoinPath("version") r, err := http.Get(versionUrl.String()) + if err != nil { + continue + } var versionRes types.VersionResponse err = json.NewDecoder(r.Body).Decode(&versionRes) if err != nil { From 04454addc712b62880e6678f254530788473f4b8 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 13:05:40 -0400 Subject: [PATCH 05/28] preventing insanely long startup times --- jprov/server/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jprov/server/utils.go b/jprov/server/utils.go index 002857f..ef89dec 100644 --- a/jprov/server/utils.go +++ b/jprov/server/utils.go @@ -21,6 +21,9 @@ func testConnection(providers []storageTypes.Providers, ip string) bool { outdatedProvider := 0 for _, provider := range providers { + if onlineProviders > 20 { + continue + } u, err := url.Parse(provider.Ip) if err != nil { continue From ef72714d615766854768f88581cf28049676aff9 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 13:06:29 -0400 Subject: [PATCH 06/28] preventing insanely long startup times --- jprov/server/file_server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/jprov/server/file_server.go b/jprov/server/file_server.go index 5515442..86b8824 100644 --- a/jprov/server/file_server.go +++ b/jprov/server/file_server.go @@ -207,6 +207,7 @@ func StartFileServer(cmd *cobra.Command) { providerName = "A Storage Provider" } + fmt.Println("Testing connection...") connected := testConnection(providers.Providers, me.Providers.Ip) if !connected { fmt.Println("Domain not configured correctly, make sure your domain points to your provider.") From 61fc10f55a0cd325a945d3d5876c7940b0bd17a8 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 13:08:07 -0400 Subject: [PATCH 07/28] logging --- jprov/server/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jprov/server/utils.go b/jprov/server/utils.go index ef89dec..35e5697 100644 --- a/jprov/server/utils.go +++ b/jprov/server/utils.go @@ -19,11 +19,14 @@ func testConnection(providers []storageTypes.Providers, ip string) bool { onlineProviders := 0 respondingProvider := 0 outdatedProvider := 0 + checked := 0 for _, provider := range providers { if onlineProviders > 20 { continue } + checked++ + fmt.Printf("Checked with %d other providers...\n", checked) u, err := url.Parse(provider.Ip) if err != nil { continue From f4ca098e5671c39643575dc1342e2b705740df93 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 13:09:32 -0400 Subject: [PATCH 08/28] logging --- jprov/server/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/jprov/server/utils.go b/jprov/server/utils.go index 35e5697..1dd59e7 100644 --- a/jprov/server/utils.go +++ b/jprov/server/utils.go @@ -60,6 +60,7 @@ func testConnection(providers []storageTypes.Providers, ip string) bool { respondingProvider++ } + fmt.Printf("Total: %d | Online: %d | Outdated: %d| Responsive: %d\n", checked, onlineProviders, outdatedProvider, respondingProvider) if respondingProvider < 2 && (onlineProviders-outdatedProvider) < 3 { return true From c8e06b6eb718146d48dd3ddd4617ba707d8c0fdf Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 13:11:44 -0400 Subject: [PATCH 09/28] fixing calls --- jprov/server/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/jprov/server/utils.go b/jprov/server/utils.go index 1dd59e7..2456b3c 100644 --- a/jprov/server/utils.go +++ b/jprov/server/utils.go @@ -55,6 +55,7 @@ func testConnection(providers []storageTypes.Providers, ip string) bool { var proxyRes types.ProxyResponse err = json.NewDecoder(r.Body).Decode(&proxyRes) if err != nil { + outdatedProvider++ continue } From 9bf5adb7579a4008242a9dd25ffffe08f6b35236 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Mon, 8 May 2023 13:13:51 -0400 Subject: [PATCH 10/28] adding route --- jprov/api/api.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jprov/api/api.go b/jprov/api/api.go index 88066be..1c2d68f 100644 --- a/jprov/api/api.go +++ b/jprov/api/api.go @@ -54,4 +54,8 @@ func BuildApi(cmd *cobra.Command, q *queue.UploadQueue, router *httprouter.Route router.GET("/api/network/status", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { network.GetStatus(cmd, w, r, ps) }) + + router.GET("/checkme", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + network.GetProxy(cmd, w, r, ps) + }) } From ba004890092bf783e4236cb7381adc05887bb115 Mon Sep 17 00:00:00 2001 From: BiPhan4 Date: Tue, 9 May 2023 05:23:43 -0400 Subject: [PATCH 11/28] interval between 1 and 5 minutes --- jprov/server/proofs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jprov/server/proofs.go b/jprov/server/proofs.go index 9c5a545..9850892 100644 --- a/jprov/server/proofs.go +++ b/jprov/server/proofs.go @@ -172,10 +172,10 @@ func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *u for { interval := intervalFromCMD - if interval < 1800 { // If the provider picked an interval that's less than 30 minutes, we generate a random interval for them anyways + if interval < 300 { // If the provider picked an interval that's less than 5 minutes, we generate a random interval for them anyways r := rand.New(rand.NewSource(time.Now().UnixNano())) - interval = uint16(r.Intn(901) + 900) // Generate interval between 15-30 minutes + interval = uint16(r.Intn(241) + 60) // Generate interval between 1-5 minutes } ctx.Logger.Debug(fmt.Sprintf("The interval between proofs is now %d", interval)) From 0b15d5cba9b251303cb98ba4e384b91235924e42 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Tue, 9 May 2023 17:27:50 -0400 Subject: [PATCH 12/28] collecting more strays for the pile --- jprov/strays/stray_manager.go | 17 ++++++++++++++--- jprov/strays/types.go | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/jprov/strays/stray_manager.go b/jprov/strays/stray_manager.go index 44cf78e..a481bf0 100644 --- a/jprov/strays/stray_manager.go +++ b/jprov/strays/stray_manager.go @@ -11,6 +11,7 @@ import ( "github.com/cosmos/cosmos-sdk/client" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/types/bech32" + "github.com/cosmos/cosmos-sdk/types/query" "github.com/cosmos/cosmos-sdk/x/feegrant" storageTypes "github.com/jackalLabs/canine-chain/x/storage/types" "github.com/spf13/cobra" @@ -169,7 +170,16 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command) { m.Context.Logger.Info("Collecting strays from chain...") qClient := storageTypes.NewQueryClient(m.ClientContext) - res, err := qClient.StraysAll(cmd.Context(), &storageTypes.QueryAllStraysRequest{}) + val := m.Rand.Int63n(300) + + page := &query.PageRequest{ + Offset: uint64(val), + Limit: 300, + } + + res, err := qClient.StraysAll(cmd.Context(), &storageTypes.QueryAllStraysRequest{ + Pagination: page, + }) if err != nil { m.Context.Logger.Error(err.Error()) return @@ -182,8 +192,7 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command) { return } - r := rand.New(rand.NewSource(time.Now().UnixNano())) - r.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] }) + m.Rand.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] }) m.Strays = make([]*storageTypes.Strays, 0) @@ -197,6 +206,8 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command) { func (m *StrayManager) Start(cmd *cobra.Command) { // loop through stray system tm, err := cmd.Flags().GetInt64(types.FlagStrayInterval) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + m.Rand = r if err != nil { panic(err) } diff --git a/jprov/strays/types.go b/jprov/strays/types.go index 87e8678..0fcfaeb 100644 --- a/jprov/strays/types.go +++ b/jprov/strays/types.go @@ -1,6 +1,7 @@ package strays import ( + "math/rand" "sync" "github.com/JackalLabs/jackal-provider/jprov/crypto" @@ -20,6 +21,7 @@ type StrayManager struct { Address string Cmd *cobra.Command Ip string + Rand *rand.Rand } func NewStrayManager(cmd *cobra.Command) *StrayManager { From 9399f4a89e16f047703743b13d1db1679fb8f56c Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Tue, 9 May 2023 17:28:54 -0400 Subject: [PATCH 13/28] allowing sometimes for the pile to be reversed --- jprov/strays/stray_manager.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jprov/strays/stray_manager.go b/jprov/strays/stray_manager.go index a481bf0..6a054fe 100644 --- a/jprov/strays/stray_manager.go +++ b/jprov/strays/stray_manager.go @@ -173,8 +173,9 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command) { val := m.Rand.Int63n(300) page := &query.PageRequest{ - Offset: uint64(val), - Limit: 300, + Offset: uint64(val), + Limit: 300, + Reverse: m.Rand.Intn(2) == 0, } res, err := qClient.StraysAll(cmd.Context(), &storageTypes.QueryAllStraysRequest{ From 1c8d0ab23e1082f21c1184e4c74ebbc55c10116c Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Tue, 9 May 2023 18:49:00 -0400 Subject: [PATCH 14/28] removing domain check for different PR --- jprov/api/network/proxy.go | 1 - jprov/server/file_server.go | 14 ++++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/jprov/api/network/proxy.go b/jprov/api/network/proxy.go index cb9ec8b..5665468 100644 --- a/jprov/api/network/proxy.go +++ b/jprov/api/network/proxy.go @@ -12,7 +12,6 @@ import ( ) func GetProxy(cmd *cobra.Command, w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - ok := true var versionRes types.VersionResponse var versionUrl *url.URL diff --git a/jprov/server/file_server.go b/jprov/server/file_server.go index 86b8824..2c5c00a 100644 --- a/jprov/server/file_server.go +++ b/jprov/server/file_server.go @@ -207,12 +207,14 @@ func StartFileServer(cmd *cobra.Command) { providerName = "A Storage Provider" } - fmt.Println("Testing connection...") - connected := testConnection(providers.Providers, me.Providers.Ip) - if !connected { - fmt.Println("Domain not configured correctly, make sure your domain points to your provider.") - return - } + //fmt.Println("Testing connection...") + //connected := testConnection(providers.Providers, me.Providers.Ip) + //if !connected { + // fmt.Println("Domain not configured correctly, make sure your domain points to your provider.") + // return + //} + _ = providers + _ = me manager := strays.NewStrayManager(cmd) // creating and starting the stray management system if !strs { From f6be1b169712e68a8e6ce2b74ab36db759ee1517 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Tue, 9 May 2023 18:51:10 -0400 Subject: [PATCH 15/28] lint --- jprov/server/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/jprov/server/utils.go b/jprov/server/utils.go index 2456b3c..b047879 100644 --- a/jprov/server/utils.go +++ b/jprov/server/utils.go @@ -15,6 +15,7 @@ import ( const ErrNotYours = "not your deal" +//nolint:all func testConnection(providers []storageTypes.Providers, ip string) bool { onlineProviders := 0 respondingProvider := 0 From d9ded6e09167086d63f0a41d15b380a78bb0165c Mon Sep 17 00:00:00 2001 From: BiPhan4 Date: Wed, 10 May 2023 00:59:24 -0400 Subject: [PATCH 16/28] change interval flag description --- jprov/jprovd/provider_commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jprov/jprovd/provider_commands.go b/jprov/jprovd/provider_commands.go index d8e596d..146628c 100644 --- a/jprov/jprovd/provider_commands.go +++ b/jprov/jprovd/provider_commands.go @@ -35,7 +35,7 @@ func StartServerCommand() *cobra.Command { cmd.Flags().Int(types.FlagPort, 3333, "Port to host the server on.") cmd.Flags().String(types.VersionFlag, "", "The value exposed by the version api to allow for custom deployments.") cmd.Flags().Bool(types.HaltStraysFlag, false, "Debug flag to stop picking up strays.") - cmd.Flags().Uint16(types.FlagInterval, 32, "The interval in seconds for which to check proofs. Must be >=1800 if you need a custom interval") + cmd.Flags().Uint16(types.FlagInterval, 32, "The interval in seconds for which to check proofs. Must be >=300 if you need a custom interval") cmd.Flags().Uint(types.FlagThreads, 10, "The amount of stray threads.") cmd.Flags().Int(types.FlagMaxMisses, 16, "The amount of intervals a provider can miss their proofs before removing a file.") cmd.Flags().Int64(types.FlagChunkSize, 10240, "The size of a single file chunk.") From f3442ed6fd3fba407d0f2eb8d9677468f3be8d78 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 11:00:48 -0400 Subject: [PATCH 17/28] no append usage --- jprov/server/http.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/jprov/server/http.go b/jprov/server/http.go index 089d52d..c3a22f3 100644 --- a/jprov/server/http.go +++ b/jprov/server/http.go @@ -83,7 +83,16 @@ func downfil(cmd *cobra.Command, w http.ResponseWriter, ps httprouter.Params, ct return } - var data []byte + chunkSize, err := cmd.Flags().GetInt64(types.FlagChunkSize) + if err != nil { + fmt.Println(err) + return + } + + dataSize := int64(len(files)) * chunkSize + + data := make([]byte, dataSize) + dataCount := 0 for i := 0; i < len(files); i += 1 { f, err := os.ReadFile(filepath.Join(utils.GetStoragePath(clientCtx, ps.ByName("file")), fmt.Sprintf("%d.jkl", i))) @@ -95,13 +104,16 @@ func downfil(cmd *cobra.Command, w http.ResponseWriter, ps httprouter.Params, ct } return } + dataCount += len(f) - data = append(data, f...) + for k, b := range f { + data[i*int(chunkSize)+k] = b + } } - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data))) + w.Header().Set("Content-Length", fmt.Sprintf("%d", dataCount)) - _, err = w.Write(data) + _, err = w.Write(data[0:dataCount]) if err != nil { return } From 1c1c6177eadb753454d07af874ad55619351d6d8 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 11:10:49 -0400 Subject: [PATCH 18/28] api speed up --- jprov/api/client/list.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jprov/api/client/list.go b/jprov/api/client/list.go index 8b7efd1..b62b647 100644 --- a/jprov/api/client/list.go +++ b/jprov/api/client/list.go @@ -17,10 +17,10 @@ import ( ) func ListQueue(cmd *cobra.Command, w http.ResponseWriter, r *http.Request, ps httprouter.Params, q *queue.UploadQueue) { - messages := make([]sdk.Msg, 0) + messages := make([]sdk.Msg, len(q.Queue)) - for _, v := range q.Queue { - messages = append(messages, v.Message) + for k, v := range q.Queue { + messages[k] = v.Message } v := types.QueueResponse{ From 5fd5229fe077edd45288d7410f2b951d92e5854b Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 20:04:55 -0400 Subject: [PATCH 19/28] choose strays more randomly --- jprov/strays/stray_manager.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/jprov/strays/stray_manager.go b/jprov/strays/stray_manager.go index 6a054fe..5c8e059 100644 --- a/jprov/strays/stray_manager.go +++ b/jprov/strays/stray_manager.go @@ -166,14 +166,17 @@ func (m *StrayManager) Init(cmd *cobra.Command, count uint, db *leveldb.DB) { // fmt.Println("Finished Initialization...") } -func (m *StrayManager) CollectStrays(cmd *cobra.Command) { +func (m *StrayManager) CollectStrays(cmd *cobra.Command, lastCount uint64) uint64 { m.Context.Logger.Info("Collecting strays from chain...") qClient := storageTypes.NewQueryClient(m.ClientContext) - val := m.Rand.Int63n(300) + var val uint64 + if lastCount > 300 { + val = uint64(m.Rand.Int63n(int64(lastCount))) + } page := &query.PageRequest{ - Offset: uint64(val), + Offset: val, Limit: 300, Reverse: m.Rand.Intn(2) == 0, } @@ -183,14 +186,14 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command) { }) if err != nil { m.Context.Logger.Error(err.Error()) - return + return 0 } s := res.Strays if len(s) == 0 { // If there are no strays, the network has claimed them all. We will try again later. m.Context.Logger.Info("No strays found.") - return + return 0 } m.Rand.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] }) @@ -203,6 +206,8 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command) { m.Strays = append(m.Strays, &k) } + + return res.Pagination.Total } func (m *StrayManager) Start(cmd *cobra.Command) { // loop through stray system @@ -212,8 +217,10 @@ func (m *StrayManager) Start(cmd *cobra.Command) { // loop through stray system if err != nil { panic(err) } + + var s uint64 for { - m.CollectStrays(cmd) // query strays from the chain + s = m.CollectStrays(cmd, s) // query strays from the chain m.Distribute() // hands strays out to hands for _, hand := range m.hands { // process every stray in parallel go hand.Process(m.Context, m) From 5276736aa4478f731e448e5ee00a4574ae5cbe2f Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 20:58:39 -0400 Subject: [PATCH 20/28] logging --- jprov/strays/stray_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jprov/strays/stray_manager.go b/jprov/strays/stray_manager.go index 5c8e059..c1aace8 100644 --- a/jprov/strays/stray_manager.go +++ b/jprov/strays/stray_manager.go @@ -167,7 +167,7 @@ func (m *StrayManager) Init(cmd *cobra.Command, count uint, db *leveldb.DB) { // } func (m *StrayManager) CollectStrays(cmd *cobra.Command, lastCount uint64) uint64 { - m.Context.Logger.Info("Collecting strays from chain...") + m.Context.Logger.Info(fmt.Sprintf("Collecting strays from chain... ~ %d", lastCount)) qClient := storageTypes.NewQueryClient(m.ClientContext) var val uint64 From 6a0f2706026882112df398410fd32ba34be5b4f9 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 21:03:39 -0400 Subject: [PATCH 21/28] count not fixed --- jprov/strays/stray_manager.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jprov/strays/stray_manager.go b/jprov/strays/stray_manager.go index c1aace8..89a8cd7 100644 --- a/jprov/strays/stray_manager.go +++ b/jprov/strays/stray_manager.go @@ -176,9 +176,10 @@ func (m *StrayManager) CollectStrays(cmd *cobra.Command, lastCount uint64) uint6 } page := &query.PageRequest{ - Offset: val, - Limit: 300, - Reverse: m.Rand.Intn(2) == 0, + Offset: val, + Limit: 300, + Reverse: m.Rand.Intn(2) == 0, + CountTotal: true, } res, err := qClient.StraysAll(cmd.Context(), &storageTypes.QueryAllStraysRequest{ From 781eea0835b2195c694f569e6dbd2ea118388a5f Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 22:00:16 -0400 Subject: [PATCH 22/28] mem leak caused by ReadDir should be fixed, as ReadDir was removed --- jprov/server/http.go | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/jprov/server/http.go b/jprov/server/http.go index 089d52d..d9b582c 100644 --- a/jprov/server/http.go +++ b/jprov/server/http.go @@ -77,31 +77,22 @@ func checkVersion(cmd *cobra.Command, w http.ResponseWriter, ctx *utils.Context) func downfil(cmd *cobra.Command, w http.ResponseWriter, ps httprouter.Params, ctx *utils.Context) { clientCtx := client.GetClientContextFromCmd(cmd) - files, err := os.ReadDir(utils.GetStoragePath(clientCtx, ps.ByName("file"))) - if err != nil { - ctx.Logger.Error(err.Error()) - return - } - var data []byte - for i := 0; i < len(files); i += 1 { - f, err := os.ReadFile(filepath.Join(utils.GetStoragePath(clientCtx, ps.ByName("file")), fmt.Sprintf("%d.jkl", i))) + var i int + for { // loop through every file in the directory and fail once it hits a file that it can't find + path := filepath.Join(utils.GetStoragePath(clientCtx, ps.ByName("file")), fmt.Sprintf("%d.jkl", i)) + f, err := os.ReadFile(path) if err != nil { - ctx.Logger.Info("Error can't open file!") - _, err = w.Write([]byte("cannot find file")) - if err != nil { - ctx.Logger.Error(err.Error()) - } - return + break } - data = append(data, f...) + i++ } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data))) - _, err = w.Write(data) + _, err := w.Write(data) if err != nil { return } From 05e54d3b456d9c071342967d163524502a67e992 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 22:49:02 -0400 Subject: [PATCH 23/28] prealloc data --- jprov/server/http.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/jprov/server/http.go b/jprov/server/http.go index d9b582c..b72e923 100644 --- a/jprov/server/http.go +++ b/jprov/server/http.go @@ -76,8 +76,15 @@ func checkVersion(cmd *cobra.Command, w http.ResponseWriter, ctx *utils.Context) func downfil(cmd *cobra.Command, w http.ResponseWriter, ps httprouter.Params, ctx *utils.Context) { clientCtx := client.GetClientContextFromCmd(cmd) + chunkSize, err := cmd.Flags().GetInt64(types.FlagChunkSize) + if err != nil { + fmt.Println(err) + return + } + + var fileList []*[]byte - var data []byte + var dataLength int var i int for { // loop through every file in the directory and fail once it hits a file that it can't find @@ -86,13 +93,22 @@ func downfil(cmd *cobra.Command, w http.ResponseWriter, ps httprouter.Params, ct if err != nil { break } - data = append(data, f...) + fileList = append(fileList, &f) + dataLength += len(f) i++ } + data := make([]byte, dataLength) + + for i, file := range fileList { + for k, b := range *file { + data[i*int(chunkSize)+k] = b + } + } + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data))) - _, err := w.Write(data) + _, err = w.Write(data) if err != nil { return } From 1405b8d9285050e2f03ad17ad0913e9a6b1f05a4 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 22:57:02 -0400 Subject: [PATCH 24/28] lint --- jprov/server/http.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jprov/server/http.go b/jprov/server/http.go index fe17e39..67338be 100644 --- a/jprov/server/http.go +++ b/jprov/server/http.go @@ -86,7 +86,6 @@ func downfil(cmd *cobra.Command, w http.ResponseWriter, ps httprouter.Params, ct var dataLength int - var i int for { // loop through every file in the directory and fail once it hits a file that it can't find path := filepath.Join(utils.GetStoragePath(clientCtx, ps.ByName("file")), fmt.Sprintf("%d.jkl", i)) @@ -103,14 +102,13 @@ func downfil(cmd *cobra.Command, w http.ResponseWriter, ps httprouter.Params, ct for i, file := range fileList { for k, b := range *file { - data[i*int(chunkSize)+k] = b } } - w.Header().Set("Content-Length", fmt.Sprintf("%d", dataCount)) + w.Header().Set("Content-Length", fmt.Sprintf("%d", dataLength)) - _, err = w.Write(data[0:dataCount]) + _, err = w.Write(data) if err != nil { return } From 0e9170ffa25e705a7f510821cf0f3c2244ea2672 Mon Sep 17 00:00:00 2001 From: BiPhan4 Date: Wed, 10 May 2023 23:20:43 -0400 Subject: [PATCH 25/28] sleep for 1-30 mins before checking if need to proof --- jprov/jprovd/provider_commands.go | 2 +- jprov/server/proofs.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jprov/jprovd/provider_commands.go b/jprov/jprovd/provider_commands.go index 146628c..d8e596d 100644 --- a/jprov/jprovd/provider_commands.go +++ b/jprov/jprovd/provider_commands.go @@ -35,7 +35,7 @@ func StartServerCommand() *cobra.Command { cmd.Flags().Int(types.FlagPort, 3333, "Port to host the server on.") cmd.Flags().String(types.VersionFlag, "", "The value exposed by the version api to allow for custom deployments.") cmd.Flags().Bool(types.HaltStraysFlag, false, "Debug flag to stop picking up strays.") - cmd.Flags().Uint16(types.FlagInterval, 32, "The interval in seconds for which to check proofs. Must be >=300 if you need a custom interval") + cmd.Flags().Uint16(types.FlagInterval, 32, "The interval in seconds for which to check proofs. Must be >=1800 if you need a custom interval") cmd.Flags().Uint(types.FlagThreads, 10, "The amount of stray threads.") cmd.Flags().Int(types.FlagMaxMisses, 16, "The amount of intervals a provider can miss their proofs before removing a file.") cmd.Flags().Int64(types.FlagChunkSize, 10240, "The size of a single file chunk.") diff --git a/jprov/server/proofs.go b/jprov/server/proofs.go index 9850892..c8da796 100644 --- a/jprov/server/proofs.go +++ b/jprov/server/proofs.go @@ -172,10 +172,10 @@ func postProofs(cmd *cobra.Command, db *leveldb.DB, q *queue.UploadQueue, ctx *u for { interval := intervalFromCMD - if interval < 300 { // If the provider picked an interval that's less than 5 minutes, we generate a random interval for them anyways + if interval < 1800 { // If the provider picked an interval that's less than 30 minutes, we generate a random interval for them anyways r := rand.New(rand.NewSource(time.Now().UnixNano())) - interval = uint16(r.Intn(241) + 60) // Generate interval between 1-5 minutes + interval = uint16(r.Intn(1801) + 60) // Generate interval between 1-30 minutes } ctx.Logger.Debug(fmt.Sprintf("The interval between proofs is now %d", interval)) From db562c08b1038ef0f46adf7fe1a5c54e65a56c35 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Wed, 10 May 2023 23:42:45 -0400 Subject: [PATCH 26/28] better string usage --- jprov/utils/network.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jprov/utils/network.go b/jprov/utils/network.go index ba2c3f6..4108481 100644 --- a/jprov/utils/network.go +++ b/jprov/utils/network.go @@ -10,6 +10,8 @@ import ( "os" "path/filepath" "runtime" + "strconv" + "strings" "github.com/tendermint/tendermint/libs/log" "github.com/wealdtech/go-merkletree" @@ -80,7 +82,10 @@ func WriteFileToDisk(cmd *cobra.Command, reader io.Reader, file io.ReaderAt, clo } for i := int64(0); i < size; i += blockSize { - f, err := os.OpenFile(filepath.Join(path, fmt.Sprintf("%d.jkl", i/blockSize)), os.O_WRONLY|os.O_CREATE, 0o666) + var str strings.Builder + str.WriteString(strconv.FormatInt(i/blockSize, 10)) + str.WriteString(".jkl") + f, err := os.OpenFile(filepath.Join(path, str.String()), os.O_WRONLY|os.O_CREATE, 0o666) if err != nil { return fid, "", data, err } From 9fb37ae65ae8013d78f6995aa18d581fbaf89780 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Thu, 11 May 2023 12:30:11 -0400 Subject: [PATCH 27/28] removed a bunch of sprints --- jprov/server/file_server.go | 26 ++++++++++++++++---------- jprov/utils/network.go | 27 ++++++++++++++++++--------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/jprov/server/file_server.go b/jprov/server/file_server.go index 2c5c00a..257d586 100644 --- a/jprov/server/file_server.go +++ b/jprov/server/file_server.go @@ -10,6 +10,7 @@ import ( "mime/multipart" "net/http" "os" + "strings" "sync" "github.com/JackalLabs/jackal-provider/jprov/crypto" @@ -49,14 +50,19 @@ func saveFile(file multipart.File, handler *multipart.FileHeader, sender string, return err } - cidhash := sha256.New() + cidHash := sha256.New() - _, err = io.WriteString(cidhash, fmt.Sprintf("%s%s%s", sender, address, fid)) + var str strings.Builder // building the FID + str.WriteString(sender) + str.WriteString(address) + str.WriteString(fid) + + _, err = io.WriteString(cidHash, str.String()) if err != nil { return err } - cid := cidhash.Sum(nil) - strcid, err := utils.MakeCid(cid) + cid := cidHash.Sum(nil) + strCid, err := utils.MakeCid(cid) if err != nil { return err } @@ -64,15 +70,15 @@ func saveFile(file multipart.File, handler *multipart.FileHeader, sender string, var wg sync.WaitGroup wg.Add(1) - msg, ctrerr := MakeContract(cmd, fid, sender, &wg, q, merkle, fmt.Sprintf("%d", size)) - if ctrerr != nil { - ctx.Logger.Error("CONTRACT ERROR: %v", ctrerr) - return ctrerr + msg, ctrErr := MakeContract(cmd, fid, sender, &wg, q, merkle, fmt.Sprintf("%d", size)) + if ctrErr != nil { + ctx.Logger.Error("CONTRACT ERROR: %v", ctrErr) + return ctrErr } wg.Wait() v := types.UploadResponse{ - CID: strcid, + CID: strCid, FID: fid, } @@ -92,7 +98,7 @@ func saveFile(file multipart.File, handler *multipart.FileHeader, sender string, return err } - err = utils.SaveToDatabase(fid, strcid, db, ctx.Logger) + err = utils.SaveToDatabase(fid, strCid, db, ctx.Logger) if err != nil { return err } diff --git a/jprov/utils/network.go b/jprov/utils/network.go index 4108481..011d321 100644 --- a/jprov/utils/network.go +++ b/jprov/utils/network.go @@ -76,9 +76,9 @@ func WriteFileToDisk(cmd *cobra.Command, reader io.Reader, file io.ReaderAt, clo path := GetStoragePath(clientCtx, fid) // This is path which we want to store the file - direrr := os.MkdirAll(path, os.ModePerm) - if direrr != nil { - return fid, "", data, direrr + dirErr := os.MkdirAll(path, os.ModePerm) + if dirErr != nil { + return fid, "", data, dirErr } for i := int64(0); i < size; i += blockSize { @@ -90,15 +90,20 @@ func WriteFileToDisk(cmd *cobra.Command, reader io.Reader, file io.ReaderAt, clo return fid, "", data, err } - firstx := make([]byte, blockSize) - read, err := file.ReadAt(firstx, i) - logger.Debug(fmt.Sprintf("Bytes read: %d", read)) + firstX := make([]byte, blockSize) + read, err := file.ReadAt(firstX, i) + + var loggerBuilder strings.Builder + loggerBuilder.WriteString("Bytes read:") + loggerBuilder.WriteString(strconv.Itoa(read)) + + logger.Debug(loggerBuilder.String()) if err != nil && err != io.EOF { return fid, "", data, err } - firstx = firstx[:read] - _, writeErr := f.Write(firstx) + firstX = firstX[:read] + _, writeErr := f.Write(firstX) if writeErr != nil { return fid, "", data, err } @@ -107,8 +112,12 @@ func WriteFileToDisk(cmd *cobra.Command, reader io.Reader, file io.ReaderAt, clo return fid, "", data, err } + var hashBuilder strings.Builder + hashBuilder.WriteString(strconv.FormatInt(i/blockSize, 10)) + hashBuilder.WriteString(hex.EncodeToString(firstX)) + hash := sha256.New() - _, err = io.WriteString(hash, fmt.Sprintf("%d%x", i/blockSize, firstx)) + _, err = io.WriteString(hash, hashBuilder.String()) if err != nil { return fid, "", data, err } From 95caf9a4bbfd9d694fd2623a362079a936d86ea6 Mon Sep 17 00:00:00 2001 From: Marston Connell <34043723+TheMarstonConnell@users.noreply.github.com> Date: Thu, 11 May 2023 22:42:07 -0400 Subject: [PATCH 28/28] lowering threads to 3 --- jprov/jprovd/provider_commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jprov/jprovd/provider_commands.go b/jprov/jprovd/provider_commands.go index d8e596d..c833cf2 100644 --- a/jprov/jprovd/provider_commands.go +++ b/jprov/jprovd/provider_commands.go @@ -36,7 +36,7 @@ func StartServerCommand() *cobra.Command { cmd.Flags().String(types.VersionFlag, "", "The value exposed by the version api to allow for custom deployments.") cmd.Flags().Bool(types.HaltStraysFlag, false, "Debug flag to stop picking up strays.") cmd.Flags().Uint16(types.FlagInterval, 32, "The interval in seconds for which to check proofs. Must be >=1800 if you need a custom interval") - cmd.Flags().Uint(types.FlagThreads, 10, "The amount of stray threads.") + cmd.Flags().Uint(types.FlagThreads, 3, "The amount of stray threads.") cmd.Flags().Int(types.FlagMaxMisses, 16, "The amount of intervals a provider can miss their proofs before removing a file.") cmd.Flags().Int64(types.FlagChunkSize, 10240, "The size of a single file chunk.") cmd.Flags().Int64(types.FlagStrayInterval, 20, "The interval in seconds to check for new strays.")