feat(SPV-1095): add merkleroots sync to go client (#273)

errors.go

@@ -45,6 +45,14 @@ var ErrTotpInvalid = models.SPVError{Message: "totp is invalid", StatusCode: 400
 // ErrContactPubKeyInvalid is when contact's PubKey is invalid
 var ErrContactPubKeyInvalid = models.SPVError{Message: "contact's PubKey is invalid", StatusCode: 400, Code: "error-contact-pubkey-invalid"}
 
+// ErrStaleLastEvaluatedKey is when the last evaluated key returned from sync merkleroots is the same as it was in a previous iteration
+// indicating sync issue or a potential loop
+var ErrStaleLastEvaluatedKey = models.SPVError{Message: "The last evaluated key has not changed between requests, indicating a possible loop or synchronization issue.", StatusCode: 500, Code: "error-stale-last-evaluated-key"}
+
+// ErrStaleLastEvaluatedKey is when the last evaluated key returned from sync merkleroots is the same as it was in a previous iteration
+// indicating sync issue or a potential loop
+var ErrSyncMerkleRootsTimeout = models.SPVError{Message: "SyncMerkleRoots operation timed out", StatusCode: 500, Code: "error-sync-merkleroots-timeout"}
+
 // WrapError wraps an error into SPVError
 func WrapError(err error) error {
 	if err == nil {

examples/Taskfile.yml

@@ -71,3 +71,8 @@ tasks:
     cmds:
       - echo "running webhooks..."
       - go run ./webhooks/webhooks.go || true
+  sync_merkleroots:
+    desc: "running sync_merkleroots.."
+    cmds:
+      - echo "running sync_merkleroots..."
+      - go run ./sync_merkleroots/sync_merkleroots.go

examples/sync_merkleroots/sync_merkleroots.go

@@ -0,0 +1,88 @@
+/*
+Package main - sync_merkleroots example
+*/
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	walletclient "github.com/bitcoin-sv/spv-wallet-go-client"
+	"github.com/bitcoin-sv/spv-wallet-go-client/examples"
+	"github.com/bitcoin-sv/spv-wallet-go-client/models"
+)
+
+// simulate a storage of merkle roots that exists on a client side that is using SyncMerkleRoots method
+type db struct {
+	MerkleRoots []models.MerkleRoot
+}
+
+func (db *db) SaveMerkleRoots(syncedMerkleRoots []models.MerkleRoot) error {
+	fmt.Print("\nSaveMerkleRoots called\n")
+	db.MerkleRoots = append(db.MerkleRoots, syncedMerkleRoots...)
+	return nil
+}
+
+func (db *db) GetLastMerkleRoot() string {
+	if len(db.MerkleRoots) == 0 {
+		return ""
+	}
+	return db.MerkleRoots[len(db.MerkleRoots)-1].MerkleRoot
+}
+
+// initalize the storage that exists on a client side
+var repository = &db{
+	MerkleRoots: []models.MerkleRoot{
+		{
+			MerkleRoot:  "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",
+			BlockHeight: 0,
+		},
+		{
+			MerkleRoot:  "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098",
+			BlockHeight: 1,
+		},
+		{
+			MerkleRoot:  "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5",
+			BlockHeight: 2,
+		},
+	},
+}
+
+func getLastFiveOrFewer(merkleroots []models.MerkleRoot) []models.MerkleRoot {
+	startIndex := len(merkleroots) - 5
+	if startIndex < 0 {
+		startIndex = 0
+	}
+
+	return merkleroots[startIndex:]
+}
+
+func main() {
+	defer examples.HandlePanic()
+
+	server := "http://localhost:3003/api/v1"
+
+	client, err := walletclient.NewWithXPriv(server, examples.ExampleXPriv)
+	if err != nil {
+		fmt.Println("Error: ", err)
+		examples.GetFullErrorMessage(err)
+		os.Exit(1)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	fmt.Printf("\n\n Initial State Length: \n %d\n\n", len(repository.MerkleRoots))
+	fmt.Printf("\n\nInitial State Last 5 MerkleRoots (or fewer):\n%+v\n", getLastFiveOrFewer(repository.MerkleRoots))
+
+	err = client.SyncMerkleRoots(ctx, repository)
+	if err != nil {
+		fmt.Println("Error: ", err)
+		examples.GetFullErrorMessage(err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("\n\n After Sync State Length: \n %d\n\n", len(repository.MerkleRoots))
+	fmt.Printf("\n\n After Sync State Last 5 MerkleRoots (or fewer):\n%+v\n", getLastFiveOrFewer(repository.MerkleRoots))
+}

fixtures/spv_wallet.go

@@ -0,0 +1,126 @@
+package fixtures
+
+import (
+	"slices"
+
+	"github.com/bitcoin-sv/spv-wallet-go-client/models"
+)
+
+const (
+	SPVWalletURL = "http://localhost:3003/api/v1"
+)
+
+// MockedSPVWalletData is mocked  merkle roots data on spv-wallet side
+var MockedSPVWalletData = []models.MerkleRoot{
+	{
+		BlockHeight: 0,
+		MerkleRoot:  "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",
+	},
+	{
+		BlockHeight: 1,
+		MerkleRoot:  "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098",
+	},
+	{
+		BlockHeight: 2,
+		MerkleRoot:  "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5",
+	},
+	{
+		BlockHeight: 3,
+		MerkleRoot:  "999e1c837c76a1b7fbb7e57baf87b309960f5ffefbf2a9b95dd890602272f644",
+	},
+	{
+		BlockHeight: 4,
+		MerkleRoot:  "df2b060fa2e5e9c8ed5eaf6a45c13753ec8c63282b2688322eba40cd98ea067a",
+	},
+	{
+		BlockHeight: 5,
+		MerkleRoot:  "63522845d294ee9b0188ae5cac91bf389a0c3723f084ca1025e7d9cdfe481ce1",
+	},
+	{
+		BlockHeight: 6,
+		MerkleRoot:  "20251a76e64e920e58291a30d4b212939aae976baca40e70818ceaa596fb9d37",
+	},
+	{
+		BlockHeight: 7,
+		MerkleRoot:  "8aa673bc752f2851fd645d6a0a92917e967083007d9c1684f9423b100540673f",
+	},
+	{
+		BlockHeight: 8,
+		MerkleRoot:  "a6f7f1c0dad0f2eb6b13c4f33de664b1b0e9f22efad5994a6d5b6086d85e85e3",
+	},
+	{
+		BlockHeight: 9,
+		MerkleRoot:  "0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9",
+	},
+	{
+		BlockHeight: 10,
+		MerkleRoot:  "d3ad39fa52a89997ac7381c95eeffeaf40b66af7a57e9eba144be0a175a12b11",
+	},
+	{
+		BlockHeight: 11,
+		MerkleRoot:  "f8325d8f7fa5d658ea143629288d0530d2710dc9193ddc067439de803c37066e",
+	},
+	{
+		BlockHeight: 12,
+		MerkleRoot:  "3b96bb7e197ef276b85131afd4a09c059cc368133a26ca04ebffb0ab4f75c8b8",
+	},
+	{
+		BlockHeight: 13,
+		MerkleRoot:  "9962d5c704ec27243364cbe9d384808feeac1c15c35ac790dffd1e929829b271",
+	},
+	{
+		BlockHeight: 14,
+		MerkleRoot:  "e1afd89295b68bc5247fe0ca2885dd4b8818d7ce430faa615067d7bab8640156",
+	},
+}
+
+// LastMockedMerkleRoot returns last merkleroot value from MockedSPVWalletData
+func LastMockedMerkleRoot() models.MerkleRoot {
+	return MockedSPVWalletData[len(MockedSPVWalletData)-1]
+}
+
+// MockedMerkleRootsAPIResponseFn is a mock of SPV-Wallet it will return a paged response of merkle roots since last evaluated merkle root
+func MockedMerkleRootsAPIResponseFn(lastMerkleRoot string) models.ExclusiveStartKeyPage[[]models.MerkleRoot] {
+	if lastMerkleRoot == "" {
+		return models.ExclusiveStartKeyPage[[]models.MerkleRoot]{
+			Content: MockedSPVWalletData,
+			Page: models.ExclusiveStartKeyPageInfo{
+				LastEvaluatedKey: "",
+				TotalElements:    len(MockedSPVWalletData),
+				Size:             len(MockedSPVWalletData),
+			},
+		}
+	}
+
+	lastMerkleRootIdx := slices.IndexFunc(MockedSPVWalletData, func(mr models.MerkleRoot) bool {
+		return mr.MerkleRoot == lastMerkleRoot
+	})
+
+	// handle case when lastMerkleRoot is already highest in the servers database
+	if lastMerkleRootIdx == len(MockedSPVWalletData)-1 {
+		return models.ExclusiveStartKeyPage[[]models.MerkleRoot]{
+			Content: []models.MerkleRoot{},
+			Page: models.ExclusiveStartKeyPageInfo{
+				LastEvaluatedKey: "",
+				TotalElements:    len(MockedSPVWalletData),
+				Size:             0,
+			},
+		}
+	}
+
+	content := MockedSPVWalletData[lastMerkleRootIdx+1:]
+	lastEvaluatedKey := content[len(content)-1].MerkleRoot
+
+	if lastEvaluatedKey == MockedSPVWalletData[len(MockedSPVWalletData)-1].MerkleRoot {
+		lastEvaluatedKey = ""
+	}
+
+	return models.ExclusiveStartKeyPage[[]models.MerkleRoot]{
+		Content: content,
+		Page: models.ExclusiveStartKeyPageInfo{
+			LastEvaluatedKey: lastEvaluatedKey,
+			TotalElements:    len(MockedSPVWalletData),
+			Size:             len(content),
+		},
+	}
+}

fixtures/sync_merkleroots.go

@@ -0,0 +1,119 @@
+package fixtures
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	"github.com/bitcoin-sv/spv-wallet-go-client/models"
+)
+
+// simulate a storage of merkle roots that exists on a client side that is using SyncMerkleRoots method
+type DB struct {
+	MerkleRoots []models.MerkleRoot
+}
+
+func (db *DB) SaveMerkleRoots(syncedMerkleRoots []models.MerkleRoot) error {
+	db.MerkleRoots = append(db.MerkleRoots, syncedMerkleRoots...)
+	return nil
+}
+
+func (db *DB) GetLastMerkleRoot() string {
+	if len(db.MerkleRoots) == 0 {
+		return ""
+	}
+	return db.MerkleRoots[len(db.MerkleRoots)-1].MerkleRoot
+}
+
+// CreateRepository creates a simulated repository a client passes to SyncMerkleRoots()
+func CreateRepository(merkleRoots []models.MerkleRoot) *DB {
+	return &DB{
+		MerkleRoots: merkleRoots,
+	}
+}
+
+func sendJSONResponse(data interface{}, w *http.ResponseWriter) {
+	(*w).Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(*w).Encode(data); err != nil {
+		(*w).WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func MockMerkleRootsAPIResponseNormal() *httptest.Server {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.URL.Path == "/v1/merkleroots" && r.Method == http.MethodGet:
+			lastEvaluatedKey := r.URL.Query().Get("lastEvaluatedKey")
+			sendJSONResponse(MockedMerkleRootsAPIResponseFn(lastEvaluatedKey), &w)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+
+	return server
+}
+
+func MockMerkleRootsAPIResponseDelayed() *httptest.Server {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.URL.Path == "/v1/merkleroots" && r.Method == http.MethodGet:
+			lastEvaluatedKey := r.URL.Query().Get("lastEvaluatedKey")
+			// it is to limit the result up to 3 merkle roots per request to ensure
+			// that the sync merkleroots will loop more than once and hit the timeout
+			all := MockedMerkleRootsAPIResponseFn(lastEvaluatedKey)
+			if len(all.Content) > 3 {
+				all.Content = all.Content[:3]
+			}
+
+			all.Page.Size = len(all.Content)
+
+			if len(all.Content) > 0 {
+				all.Page.LastEvaluatedKey = all.Content[len(all.Content)-1].MerkleRoot
+			} else {
+				all.Page.LastEvaluatedKey = ""
+			}
+
+			time.Sleep(50 * time.Millisecond)
+			sendJSONResponse(all, &w)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+
+	return server
+}
+
+func MockMerkleRootsAPIResponseStale() *httptest.Server {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.URL.Path == "/v1/merkleroots" && r.Method == http.MethodGet:
+			staleLastEvaluatedKeyResponse := models.ExclusiveStartKeyPage[[]models.MerkleRoot]{
+				Content: []models.MerkleRoot{
+					{
+						MerkleRoot:  "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",
+						BlockHeight: 0,
+					},
+					{
+						MerkleRoot:  "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098",
+						BlockHeight: 1,
+					},
+					{
+						MerkleRoot:  "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5",
+						BlockHeight: 2,
+					},
+				},
+				Page: models.ExclusiveStartKeyPageInfo{
+					LastEvaluatedKey: "9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5",
+					Size:             3,
+					TotalElements:    len(MockedSPVWalletData),
+				},
+			}
+			sendJSONResponse(staleLastEvaluatedKeyResponse, &w)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+
+	return server
+}

models/sync_merkleroots.go

@@ -0,0 +1,38 @@
+package models
+
+// ExclusiveStartKeyPage represents a paginated response for database records using Exclusive Start Key paging
+type ExclusiveStartKeyPage[T any] struct {
+	// List of records for the response
+	Content T
+	// Pagination details
+	Page ExclusiveStartKeyPageInfo
+}
+
+// ExclusiveStartKeyPageInfo represents the pagination information for limiting and sorting database query results
+type ExclusiveStartKeyPageInfo struct {
+	// Field by which to order the results
+	OrderByField *string `json:"orderByField,omitempty"` // Optional ordering field
+	// Direction in which to order the results (ASC or DESC)
+	SortDirection *string `json:"sortDirection,omitempty"` // Optional sort direction
+	// Total count of elements
+	TotalElements int `json:"totalElements"`
+	// Size of the page or returned data
+	Size int `json:"size"`
+	// Last evaluated key returned from the database
+	LastEvaluatedKey string `json:"lastEvaluatedKey"`
+}
+
+// MerkleRoot holds the content of the synced Merkle root response
+type MerkleRoot struct {
+	MerkleRoot  string `json:"merkleRoot"`
+	BlockHeight int    `json:"blockHeight"`
+}
+
+// MerkleRootsRepository is an interface responsible for saving synced merkleroots and getting last evaluat key from database
+type MerkleRootsRepository interface {
+	// GetLastMerkleRoot should return the merkle root with the heighest height from your storage or undefined if empty
+	GetLastMerkleRoot() string
+	// SaveMerkleRoots should store newly synced merkle roots into your storage;
+	// NOTE: items are ordered with ascending order by block height
+	SaveMerkleRoots(syncedMerkleRoots []MerkleRoot) error
+}