From ba57b683b0afe00d4976b7c2c0193c7e96ca7a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20=C5=BB=C3=B3=C5=82towski?= Date: Fri, 11 Oct 2024 19:30:15 +0200 Subject: [PATCH] feat(SPV-1095): add merkleroots sync to go client (#273) --- errors.go | 8 ++ examples/Taskfile.yml | 5 + examples/sync_merkleroots/sync_merkleroots.go | 88 ++++++++++++ fixtures/spv_wallet.go | 126 ++++++++++++++++++ fixtures/sync_merkleroots.go | 119 +++++++++++++++++ models/sync_merkleroots.go | 38 ++++++ sync_merkleroots.go | 63 +++++++++ sync_merkleroots_test.go | 102 ++++++++++++++ 8 files changed, 549 insertions(+) create mode 100644 examples/sync_merkleroots/sync_merkleroots.go create mode 100644 fixtures/spv_wallet.go create mode 100644 fixtures/sync_merkleroots.go create mode 100644 models/sync_merkleroots.go create mode 100644 sync_merkleroots.go create mode 100644 sync_merkleroots_test.go diff --git a/errors.go b/errors.go index a3766d81..277ff343 100644 --- a/errors.go +++ b/errors.go @@ -45,6 +45,14 @@ var ErrTotpInvalid = models.SPVError{Message: "totp is invalid", StatusCode: 400 // ErrContactPubKeyInvalid is when contact's PubKey is invalid var ErrContactPubKeyInvalid = models.SPVError{Message: "contact's PubKey is invalid", StatusCode: 400, Code: "error-contact-pubkey-invalid"} +// ErrStaleLastEvaluatedKey is when the last evaluated key returned from sync merkleroots is the same as it was in a previous iteration +// indicating sync issue or a potential loop +var ErrStaleLastEvaluatedKey = models.SPVError{Message: "The last evaluated key has not changed between requests, indicating a possible loop or synchronization issue.", StatusCode: 500, Code: "error-stale-last-evaluated-key"} + +// ErrStaleLastEvaluatedKey is when the last evaluated key returned from sync merkleroots is the same as it was in a previous iteration +// indicating sync issue or a potential loop +var ErrSyncMerkleRootsTimeout = models.SPVError{Message: "SyncMerkleRoots operation timed out", StatusCode: 500, Code: "error-sync-merkleroots-timeout"} + // WrapError wraps an error into SPVError func WrapError(err error) error { if err == nil { diff --git a/examples/Taskfile.yml b/examples/Taskfile.yml index 81d7cbe9..88689de6 100644 --- a/examples/Taskfile.yml +++ b/examples/Taskfile.yml @@ -71,3 +71,8 @@ tasks: cmds: - echo "running webhooks..." - go run ./webhooks/webhooks.go || true + sync_merkleroots: + desc: "running sync_merkleroots.." + cmds: + - echo "running sync_merkleroots..." + - go run ./sync_merkleroots/sync_merkleroots.go diff --git a/examples/sync_merkleroots/sync_merkleroots.go b/examples/sync_merkleroots/sync_merkleroots.go new file mode 100644 index 00000000..8f791ee5 --- /dev/null +++ b/examples/sync_merkleroots/sync_merkleroots.go @@ -0,0 +1,88 @@ +/* +Package main - sync_merkleroots example +*/ +package main + +import ( + "context" + "fmt" + "os" + "time" + + walletclient "github.com/bitcoin-sv/spv-wallet-go-client" + "github.com/bitcoin-sv/spv-wallet-go-client/examples" + "github.com/bitcoin-sv/spv-wallet-go-client/models" +) + +// simulate a storage of merkle roots that exists on a client side that is using SyncMerkleRoots method +type db struct { + MerkleRoots []models.MerkleRoot +} + +func (db *db) SaveMerkleRoots(syncedMerkleRoots []models.MerkleRoot) error { + fmt.Print("\nSaveMerkleRoots called\n") + db.MerkleRoots = append(db.MerkleRoots, syncedMerkleRoots...) + return nil +} + +func (db *db) GetLastMerkleRoot() string { + if len(db.MerkleRoots) == 0 { + return "" + } + return db.MerkleRoots[len(db.MerkleRoots)-1].MerkleRoot +} + +// initalize the storage that exists on a client side +var repository = &db{ + MerkleRoots: []models.MerkleRoot{ + { + MerkleRoot: "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + BlockHeight: 0, + }, + { + MerkleRoot: "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098", + BlockHeight: 1, + }, + { + MerkleRoot: "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5", + BlockHeight: 2, + }, + }, +} + +func getLastFiveOrFewer(merkleroots []models.MerkleRoot) []models.MerkleRoot { + startIndex := len(merkleroots) - 5 + if startIndex < 0 { + startIndex = 0 + } + + return merkleroots[startIndex:] +} + +func main() { + defer examples.HandlePanic() + + server := "http://localhost:3003/api/v1" + + client, err := walletclient.NewWithXPriv(server, examples.ExampleXPriv) + if err != nil { + fmt.Println("Error: ", err) + examples.GetFullErrorMessage(err) + os.Exit(1) + } + ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond) + defer cancel() + + fmt.Printf("\n\n Initial State Length: \n %d\n\n", len(repository.MerkleRoots)) + fmt.Printf("\n\nInitial State Last 5 MerkleRoots (or fewer):\n%+v\n", getLastFiveOrFewer(repository.MerkleRoots)) + + err = client.SyncMerkleRoots(ctx, repository) + if err != nil { + fmt.Println("Error: ", err) + examples.GetFullErrorMessage(err) + os.Exit(1) + } + + fmt.Printf("\n\n After Sync State Length: \n %d\n\n", len(repository.MerkleRoots)) + fmt.Printf("\n\n After Sync State Last 5 MerkleRoots (or fewer):\n%+v\n", getLastFiveOrFewer(repository.MerkleRoots)) +} diff --git a/fixtures/spv_wallet.go b/fixtures/spv_wallet.go new file mode 100644 index 00000000..1162d6f1 --- /dev/null +++ b/fixtures/spv_wallet.go @@ -0,0 +1,126 @@ +package fixtures + +import ( + "slices" + + "github.com/bitcoin-sv/spv-wallet-go-client/models" +) + +const ( + SPVWalletURL = "http://localhost:3003/api/v1" +) + +// MockedSPVWalletData is mocked merkle roots data on spv-wallet side +var MockedSPVWalletData = []models.MerkleRoot{ + { + BlockHeight: 0, + MerkleRoot: "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + }, + { + BlockHeight: 1, + MerkleRoot: "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098", + }, + { + BlockHeight: 2, + MerkleRoot: "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5", + }, + { + BlockHeight: 3, + MerkleRoot: "999e1c837c76a1b7fbb7e57baf87b309960f5ffefbf2a9b95dd890602272f644", + }, + { + BlockHeight: 4, + MerkleRoot: "df2b060fa2e5e9c8ed5eaf6a45c13753ec8c63282b2688322eba40cd98ea067a", + }, + { + BlockHeight: 5, + MerkleRoot: "63522845d294ee9b0188ae5cac91bf389a0c3723f084ca1025e7d9cdfe481ce1", + }, + { + BlockHeight: 6, + MerkleRoot: "20251a76e64e920e58291a30d4b212939aae976baca40e70818ceaa596fb9d37", + }, + { + BlockHeight: 7, + MerkleRoot: "8aa673bc752f2851fd645d6a0a92917e967083007d9c1684f9423b100540673f", + }, + { + BlockHeight: 8, + MerkleRoot: "a6f7f1c0dad0f2eb6b13c4f33de664b1b0e9f22efad5994a6d5b6086d85e85e3", + }, + { + BlockHeight: 9, + MerkleRoot: "0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9", + }, + { + BlockHeight: 10, + MerkleRoot: "d3ad39fa52a89997ac7381c95eeffeaf40b66af7a57e9eba144be0a175a12b11", + }, + { + BlockHeight: 11, + MerkleRoot: "f8325d8f7fa5d658ea143629288d0530d2710dc9193ddc067439de803c37066e", + }, + { + BlockHeight: 12, + MerkleRoot: "3b96bb7e197ef276b85131afd4a09c059cc368133a26ca04ebffb0ab4f75c8b8", + }, + { + BlockHeight: 13, + MerkleRoot: "9962d5c704ec27243364cbe9d384808feeac1c15c35ac790dffd1e929829b271", + }, + { + BlockHeight: 14, + MerkleRoot: "e1afd89295b68bc5247fe0ca2885dd4b8818d7ce430faa615067d7bab8640156", + }, +} + +// LastMockedMerkleRoot returns last merkleroot value from MockedSPVWalletData +func LastMockedMerkleRoot() models.MerkleRoot { + return MockedSPVWalletData[len(MockedSPVWalletData)-1] +} + +// MockedMerkleRootsAPIResponseFn is a mock of SPV-Wallet it will return a paged response of merkle roots since last evaluated merkle root +func MockedMerkleRootsAPIResponseFn(lastMerkleRoot string) models.ExclusiveStartKeyPage[[]models.MerkleRoot] { + if lastMerkleRoot == "" { + return models.ExclusiveStartKeyPage[[]models.MerkleRoot]{ + Content: MockedSPVWalletData, + Page: models.ExclusiveStartKeyPageInfo{ + LastEvaluatedKey: "", + TotalElements: len(MockedSPVWalletData), + Size: len(MockedSPVWalletData), + }, + } + } + + lastMerkleRootIdx := slices.IndexFunc(MockedSPVWalletData, func(mr models.MerkleRoot) bool { + return mr.MerkleRoot == lastMerkleRoot + }) + + // handle case when lastMerkleRoot is already highest in the servers database + if lastMerkleRootIdx == len(MockedSPVWalletData)-1 { + return models.ExclusiveStartKeyPage[[]models.MerkleRoot]{ + Content: []models.MerkleRoot{}, + Page: models.ExclusiveStartKeyPageInfo{ + LastEvaluatedKey: "", + TotalElements: len(MockedSPVWalletData), + Size: 0, + }, + } + } + + content := MockedSPVWalletData[lastMerkleRootIdx+1:] + lastEvaluatedKey := content[len(content)-1].MerkleRoot + + if lastEvaluatedKey == MockedSPVWalletData[len(MockedSPVWalletData)-1].MerkleRoot { + lastEvaluatedKey = "" + } + + return models.ExclusiveStartKeyPage[[]models.MerkleRoot]{ + Content: content, + Page: models.ExclusiveStartKeyPageInfo{ + LastEvaluatedKey: lastEvaluatedKey, + TotalElements: len(MockedSPVWalletData), + Size: len(content), + }, + } +} diff --git a/fixtures/sync_merkleroots.go b/fixtures/sync_merkleroots.go new file mode 100644 index 00000000..bf09c790 --- /dev/null +++ b/fixtures/sync_merkleroots.go @@ -0,0 +1,119 @@ +package fixtures + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "time" + + "github.com/bitcoin-sv/spv-wallet-go-client/models" +) + +// simulate a storage of merkle roots that exists on a client side that is using SyncMerkleRoots method +type DB struct { + MerkleRoots []models.MerkleRoot +} + +func (db *DB) SaveMerkleRoots(syncedMerkleRoots []models.MerkleRoot) error { + db.MerkleRoots = append(db.MerkleRoots, syncedMerkleRoots...) + return nil +} + +func (db *DB) GetLastMerkleRoot() string { + if len(db.MerkleRoots) == 0 { + return "" + } + return db.MerkleRoots[len(db.MerkleRoots)-1].MerkleRoot +} + +// CreateRepository creates a simulated repository a client passes to SyncMerkleRoots() +func CreateRepository(merkleRoots []models.MerkleRoot) *DB { + return &DB{ + MerkleRoots: merkleRoots, + } +} + +func sendJSONResponse(data interface{}, w *http.ResponseWriter) { + (*w).Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(*w).Encode(data); err != nil { + (*w).WriteHeader(http.StatusInternalServerError) + } +} + +func MockMerkleRootsAPIResponseNormal() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/v1/merkleroots" && r.Method == http.MethodGet: + lastEvaluatedKey := r.URL.Query().Get("lastEvaluatedKey") + sendJSONResponse(MockedMerkleRootsAPIResponseFn(lastEvaluatedKey), &w) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + + return server +} + +func MockMerkleRootsAPIResponseDelayed() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/v1/merkleroots" && r.Method == http.MethodGet: + lastEvaluatedKey := r.URL.Query().Get("lastEvaluatedKey") + // it is to limit the result up to 3 merkle roots per request to ensure + // that the sync merkleroots will loop more than once and hit the timeout + all := MockedMerkleRootsAPIResponseFn(lastEvaluatedKey) + if len(all.Content) > 3 { + all.Content = all.Content[:3] + } + + all.Page.Size = len(all.Content) + + if len(all.Content) > 0 { + all.Page.LastEvaluatedKey = all.Content[len(all.Content)-1].MerkleRoot + } else { + all.Page.LastEvaluatedKey = "" + } + + time.Sleep(50 * time.Millisecond) + sendJSONResponse(all, &w) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + + return server +} + +func MockMerkleRootsAPIResponseStale() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/v1/merkleroots" && r.Method == http.MethodGet: + staleLastEvaluatedKeyResponse := models.ExclusiveStartKeyPage[[]models.MerkleRoot]{ + Content: []models.MerkleRoot{ + { + MerkleRoot: "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + BlockHeight: 0, + }, + { + MerkleRoot: "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098", + BlockHeight: 1, + }, + { + MerkleRoot: "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5", + BlockHeight: 2, + }, + }, + Page: models.ExclusiveStartKeyPageInfo{ + LastEvaluatedKey: "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5", + Size: 3, + TotalElements: len(MockedSPVWalletData), + }, + } + sendJSONResponse(staleLastEvaluatedKeyResponse, &w) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + + return server +} diff --git a/models/sync_merkleroots.go b/models/sync_merkleroots.go new file mode 100644 index 00000000..978f86cf --- /dev/null +++ b/models/sync_merkleroots.go @@ -0,0 +1,38 @@ +package models + +// ExclusiveStartKeyPage represents a paginated response for database records using Exclusive Start Key paging +type ExclusiveStartKeyPage[T any] struct { + // List of records for the response + Content T + // Pagination details + Page ExclusiveStartKeyPageInfo +} + +// ExclusiveStartKeyPageInfo represents the pagination information for limiting and sorting database query results +type ExclusiveStartKeyPageInfo struct { + // Field by which to order the results + OrderByField *string `json:"orderByField,omitempty"` // Optional ordering field + // Direction in which to order the results (ASC or DESC) + SortDirection *string `json:"sortDirection,omitempty"` // Optional sort direction + // Total count of elements + TotalElements int `json:"totalElements"` + // Size of the page or returned data + Size int `json:"size"` + // Last evaluated key returned from the database + LastEvaluatedKey string `json:"lastEvaluatedKey"` +} + +// MerkleRoot holds the content of the synced Merkle root response +type MerkleRoot struct { + MerkleRoot string `json:"merkleRoot"` + BlockHeight int `json:"blockHeight"` +} + +// MerkleRootsRepository is an interface responsible for saving synced merkleroots and getting last evaluat key from database +type MerkleRootsRepository interface { + // GetLastMerkleRoot should return the merkle root with the heighest height from your storage or undefined if empty + GetLastMerkleRoot() string + // SaveMerkleRoots should store newly synced merkle roots into your storage; + // NOTE: items are ordered with ascending order by block height + SaveMerkleRoots(syncedMerkleRoots []MerkleRoot) error +} diff --git a/sync_merkleroots.go b/sync_merkleroots.go new file mode 100644 index 00000000..025a39da --- /dev/null +++ b/sync_merkleroots.go @@ -0,0 +1,63 @@ +package walletclient + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/bitcoin-sv/spv-wallet-go-client/models" +) + +// SyncMerkleRoots syncs merkleroots known to spv-wallet with the client database +// If timeout is needed pass context.WithTimeout() as ctx param +func (wc *WalletClient) SyncMerkleRoots(ctx context.Context, repo models.MerkleRootsRepository) error { + lastEvaluatedKey := repo.GetLastMerkleRoot() + requestPath := "merkleroots" + lastEvaluatedKeyQuery := "" + previousLastEvaluatedKey := lastEvaluatedKey + + if lastEvaluatedKey != "" { + lastEvaluatedKeyQuery = fmt.Sprintf("?lastEvaluatedKey=%s", lastEvaluatedKey) + } + + for { + select { + case <-ctx.Done(): + return ErrSyncMerkleRootsTimeout + default: + url := fmt.Sprintf("/%s%s", requestPath, lastEvaluatedKeyQuery) + + var merkleRootsResponse models.ExclusiveStartKeyPage[[]models.MerkleRoot] + + err := wc.doHTTPRequest(ctx, http.MethodGet, url, nil, wc.xPriv, true, &merkleRootsResponse) + + if err != nil { + // In case if the context deadline exceeds its limit during http request, httpClient + // cancels the request wrapping it as spverror, so we need to check if the message + // is the same as context deadline exceeded error + if strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + return ErrSyncMerkleRootsTimeout + } + return WrapError(err) + } + + lastEvaluatedKey = merkleRootsResponse.Page.LastEvaluatedKey + if lastEvaluatedKey != "" && previousLastEvaluatedKey == lastEvaluatedKey { + return ErrStaleLastEvaluatedKey + } + + err = repo.SaveMerkleRoots(merkleRootsResponse.Content) + if err != nil { + return err + } + + previousLastEvaluatedKey = lastEvaluatedKey + if previousLastEvaluatedKey == "" { + return nil + } + + lastEvaluatedKeyQuery = fmt.Sprintf("?lastEvaluatedKey=%s", previousLastEvaluatedKey) + } + } +} diff --git a/sync_merkleroots_test.go b/sync_merkleroots_test.go new file mode 100644 index 00000000..3d9c7157 --- /dev/null +++ b/sync_merkleroots_test.go @@ -0,0 +1,102 @@ +package walletclient + +import ( + "context" + "testing" + "time" + + "github.com/bitcoin-sv/spv-wallet-go-client/fixtures" + "github.com/bitcoin-sv/spv-wallet-go-client/models" + "github.com/stretchr/testify/require" +) + +func TestSyncMerkleRoots(t *testing.T) { + + t.Run("Should properly sync database when empty", func(t *testing.T) { + // setup + server := fixtures.MockMerkleRootsAPIResponseNormal() + defer server.Close() + + // given + repo := fixtures.CreateRepository([]models.MerkleRoot{}) + client, err := NewWithXPriv(server.URL, fixtures.XPrivString) + require.NoError(t, err) + + // when + err = client.SyncMerkleRoots(context.Background(), repo) + + // then + require.NoError(t, err) + require.Len(t, repo.MerkleRoots, len(fixtures.MockedSPVWalletData)) + require.Equal(t, fixtures.LastMockedMerkleRoot(), repo.MerkleRoots[len(repo.MerkleRoots)-1]) + }) + + t.Run("Should properly sync database when partially filled", func(t *testing.T) { + // setup + server := fixtures.MockMerkleRootsAPIResponseNormal() + defer server.Close() + + // given + repo := fixtures.CreateRepository([]models.MerkleRoot{ + { + MerkleRoot: "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + BlockHeight: 0, + }, + { + MerkleRoot: "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098", + BlockHeight: 1, + }, + { + MerkleRoot: "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5", + BlockHeight: 2, + }, + }) + client, err := NewWithXPriv(server.URL, fixtures.XPrivString) + require.NoError(t, err) + + // when + err = client.SyncMerkleRoots(context.Background(), repo) + + // then + require.NoError(t, err) + require.Len(t, repo.MerkleRoots, len(fixtures.MockedSPVWalletData)) + require.Equal(t, fixtures.LastMockedMerkleRoot(), repo.MerkleRoots[len(repo.MerkleRoots)-1]) + }) + + t.Run("Should fail sync merkleroots due to the time out", func(t *testing.T) { + // setup + server := fixtures.MockMerkleRootsAPIResponseDelayed() + defer server.Close() + + // given + repo := fixtures.CreateRepository([]models.MerkleRoot{}) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) + defer cancel() + + client, err := NewWithXPriv(server.URL, fixtures.XPrivString) + require.NoError(t, err) + + // when + err = client.SyncMerkleRoots(ctx, repo) + + // then + require.ErrorIs(t, err, ErrSyncMerkleRootsTimeout) + }) + + t.Run("Should fail sync merkleroots due to last evaluated key being the same in the response", func(t *testing.T) { + // setup + server := fixtures.MockMerkleRootsAPIResponseStale() + defer server.Close() + + // given + repo := fixtures.CreateRepository([]models.MerkleRoot{}) + client, err := NewWithXPriv(server.URL, fixtures.XPrivString) + require.NoError(t, err) + + // when + err = client.SyncMerkleRoots(context.Background(), repo) + + // then + require.ErrorIs(t, err, ErrStaleLastEvaluatedKey) + }) +}