From 5f1f65b42bf54a79cc47689817b1d92b749dc2e6 Mon Sep 17 00:00:00 2001 From: marston Date: Mon, 27 Nov 2023 11:19:43 -0500 Subject: [PATCH 1/2] cdn redirects on any storage provider --- api/file_handler.go | 105 +++++++++++++++++++++++++++++++++++-- api/server.go | 3 +- file_system/file_system.go | 16 ++++++ file_system/keys.go | 1 + 4 files changed, 121 insertions(+), 4 deletions(-) diff --git a/api/file_handler.go b/api/file_handler.go index 0cc180b..aabfcc9 100644 --- a/api/file_handler.go +++ b/api/file_handler.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "net/http" + "net/url" "strconv" "time" @@ -124,7 +125,73 @@ func PostFileHandler(fio *file_system.FileSystem, prover *proofs.Prover, wl *wal } } -func DownloadFileHandler(f *file_system.FileSystem) func(http.ResponseWriter, *http.Request) { +func ForwardDownload(merkle []byte, wallet *wallet.Wallet, w http.ResponseWriter) { + cl := storageTypes.NewQueryClient(wallet.Client.GRPCConn) + fReq := storageTypes.QueryFindFile{Merkle: merkle} + + fileRes, err := cl.FindFile(context.Background(), &fReq) + if err != nil { + v := types.ErrorResponse{ + Error: err.Error(), + } + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(v) + return + } + + ips := fileRes.ProviderIps + var ipStrings []string + err = json.Unmarshal([]byte(ips), ipStrings) + if err != nil { + v := types.ErrorResponse{ + Error: err.Error(), + } + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(v) + return + } + + for _, ipString := range ipStrings { + u, err := url.Parse(ipString) + if err != nil { + continue + } + + merkleString := hex.EncodeToString(merkle) + + hasU := u.JoinPath("has", merkleString) + + hcl := http.DefaultClient + + r, err := http.NewRequest("GET", hasU.String(), nil) + if err != nil { + continue + } + + hRes, err := hcl.Do(r) + if err != nil { + continue + } + + if hRes.StatusCode != 200 { + continue + } + + mU := u.JoinPath("download", merkleString) + http.Redirect(w, r, mU.String(), http.StatusFound) + return + } + + v := types.ErrorResponse{ + Error: "cannot find file on network", + } + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(v) + return + +} + +func DownloadFileHandler(f *file_system.FileSystem, wallet *wallet.Wallet) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, req *http.Request) { vars := mux.Vars(req) @@ -136,19 +203,51 @@ func DownloadFileHandler(f *file_system.FileSystem) func(http.ResponseWriter, *h } w.WriteHeader(http.StatusInternalServerError) _ = json.NewEncoder(w).Encode(v) - + return } file, err := f.GetFileDataByMerkle(merkle) + if err != nil { + ForwardDownload(merkle, wallet, w) + return + } + + _, _ = w.Write(file) + } +} + +func HasFileHandler(f *file_system.FileSystem) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + + merkleString := vars["merkle"] + merkle, err := hex.DecodeString(merkleString) if err != nil { v := types.ErrorResponse{ Error: err.Error(), } w.WriteHeader(http.StatusInternalServerError) _ = json.NewEncoder(w).Encode(v) + return } - _, _ = w.Write(file) + found, err := f.HasFile(merkle) + if err != nil { + v := types.ErrorResponse{ + Error: err.Error(), + } + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(v) + return + } + + if found { + w.WriteHeader(200) + return + } + + w.WriteHeader(http.StatusNotFound) + } } diff --git a/api/server.go b/api/server.go index 234c5e6..d9552e9 100644 --- a/api/server.go +++ b/api/server.go @@ -39,7 +39,8 @@ func (a *API) Serve(f *file_system.FileSystem, p *proofs.Prover, wallet *wallet. r := mux.NewRouter() r.HandleFunc("/", IndexHandler(wallet.AccAddress())) r.HandleFunc("/upload", PostFileHandler(f, p, wallet, chunkSize)) - r.HandleFunc("/download/{fid}", DownloadFileHandler(f)) + r.HandleFunc("/download/{fid}", DownloadFileHandler(f, wallet)) + r.HandleFunc("/has/{fid}", HasFileHandler(f)) r.HandleFunc("/list", ListFilesHandler(f)) r.HandleFunc("/api/data/fids", LegacyListFilesHandler(f)) diff --git a/file_system/file_system.go b/file_system/file_system.go index fa02a07..f368593 100644 --- a/file_system/file_system.go +++ b/file_system/file_system.go @@ -355,3 +355,19 @@ func (f *FileSystem) GetFileDataByMerkle(merkle []byte) ([]byte, error) { return fileData, err } + +func (f *FileSystem) HasFile(merkle []byte) (found bool, err error) { + found = false + err = f.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + prefix := majorChunkMerkleKey(merkle) + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + found = true + return nil + } + + return nil + }) + return found, err +} diff --git a/file_system/keys.go b/file_system/keys.go index 3b1103d..ac1eb78 100644 --- a/file_system/keys.go +++ b/file_system/keys.go @@ -15,6 +15,7 @@ func majorChunkKey(merkle []byte, owner string, start int64) []byte { return []byte(fmt.Sprintf("chunks/%x/%s/%d/", merkle, owner, start)) } +// majorChunkMerkleKey returns a byte array of `chunks/{merkle}` func majorChunkMerkleKey(merkle []byte) []byte { return []byte(fmt.Sprintf("chunks/%x", merkle)) } From a33dbc93b598fc8384fc7725161227670b668c6f Mon Sep 17 00:00:00 2001 From: marston Date: Mon, 27 Nov 2023 11:48:04 -0500 Subject: [PATCH 2/2] lint --- api/file_handler.go | 3 --- file_system/file_system_test.go | 1 - 2 files changed, 4 deletions(-) diff --git a/api/file_handler.go b/api/file_handler.go index aabfcc9..27200cb 100644 --- a/api/file_handler.go +++ b/api/file_handler.go @@ -187,8 +187,6 @@ func ForwardDownload(merkle []byte, wallet *wallet.Wallet, w http.ResponseWriter } w.WriteHeader(http.StatusInternalServerError) _ = json.NewEncoder(w).Encode(v) - return - } func DownloadFileHandler(f *file_system.FileSystem, wallet *wallet.Wallet) func(http.ResponseWriter, *http.Request) { @@ -248,6 +246,5 @@ func HasFileHandler(f *file_system.FileSystem) func(http.ResponseWriter, *http.R } w.WriteHeader(http.StatusNotFound) - } } diff --git a/file_system/file_system_test.go b/file_system/file_system_test.go index 1def316..2a41dd9 100644 --- a/file_system/file_system_test.go +++ b/file_system/file_system_test.go @@ -182,5 +182,4 @@ func TestWriteAndProveFiles(t *testing.T) { require.Equal(t, true, verified) } - }