From 7bf720e787975f2b788bf9b22e2b8ad6bb9cfb08 Mon Sep 17 00:00:00 2001 From: Konst Date: Fri, 15 Nov 2024 17:38:27 +0300 Subject: [PATCH 01/12] Updated swagger docs --- cmd/api/docs/swagger.json | 92 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/cmd/api/docs/swagger.json b/cmd/api/docs/swagger.json index d505cc64..4cc85b2d 100644 --- a/cmd/api/docs/swagger.json +++ b/cmd/api/docs/swagger.json @@ -2823,6 +2823,67 @@ } } }, + "/rollup/group": { + "get": { + "description": "Rollup Grouped Statistics", + "produces": [ + "application/json" + ], + "tags": [ + "rollup" + ], + "summary": "Rollup Grouped Statistics", + "operationId": "rollup-grouped-statistics", + "parameters": [ + { + "enum": [ + "sum", + "avg" + ], + "type": "string", + "description": "Aggregate function", + "name": "func", + "in": "query" + }, + { + "enum": [ + "stack", + "type", + "category", + "vm", + "provider" + ], + "type": "string", + "description": "Group column", + "name": "column", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.RollupGroupedStats" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handler.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, "/rollup/slug/{slug}": { "get": { "description": "Get rollup by slug", @@ -5605,6 +5666,12 @@ "type": "string" } }, + "rollup_type": { + "type": "array", + "items": { + "type": "string" + } + }, "status": { "type": "array", "items": { @@ -6305,6 +6372,31 @@ } } }, + "responses.RollupGroupedStats": { + "type": "object", + "properties": { + "blobs_count": { + "type": "integer", + "format": "integer", + "example": 2 + }, + "fee": { + "type": "string", + "format": "string", + "example": "123.456789" + }, + "group": { + "type": "string", + "format": "string", + "example": "group" + }, + "size": { + "type": "integer", + "format": "integer", + "example": 1000 + } + } + }, "responses.RollupStats24h": { "type": "object", "properties": { From acd5c67233425fb068b7f79edd2df7d93ddbe3ad Mon Sep 17 00:00:00 2001 From: Konst Date: Fri, 29 Nov 2024 16:17:07 +0300 Subject: [PATCH 02/12] Added blob proofs API endpoint --- cmd/api/handler/namespace.go | 74 +++++++++++++++++ cmd/api/handler/responses/ods.go | 138 ++++++++++++++++++++++++++++++- cmd/api/init.go | 12 ++- go.mod | 2 +- go.sum | 2 + pkg/node/api.go | 1 + pkg/node/rpc/blob.go | 31 +++++++ pkg/types/blobProofs.go | 29 +++++++ 8 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 pkg/node/rpc/blob.go create mode 100644 pkg/types/blobProofs.go diff --git a/cmd/api/handler/namespace.go b/cmd/api/handler/namespace.go index e377f8b4..76610125 100644 --- a/cmd/api/handler/namespace.go +++ b/cmd/api/handler/namespace.go @@ -7,6 +7,9 @@ import ( "context" "encoding/base64" "encoding/hex" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/da" + "github.com/celestiaorg/go-square/shares" "net/http" "time" @@ -17,6 +20,7 @@ import ( "github.com/celenium-io/celestia-indexer/internal/storage" testsuite "github.com/celenium-io/celestia-indexer/internal/test_suite" "github.com/celenium-io/celestia-indexer/pkg/node" + "github.com/celestiaorg/go-square/square" "github.com/labstack/echo/v4" ) @@ -27,6 +31,7 @@ type NamespaceHandler struct { address storage.IAddress blob node.DalApi state storage.IState + node node.Api indexerName string } @@ -38,6 +43,7 @@ func NewNamespaceHandler( state storage.IState, indexerName string, blob node.DalApi, + node node.Api, ) *NamespaceHandler { return &NamespaceHandler{ namespace: namespace, @@ -47,6 +53,7 @@ func NewNamespaceHandler( blob: blob, state: state, indexerName: indexerName, + node: node, } } @@ -733,3 +740,70 @@ func (handler *NamespaceHandler) Rollups(c echo.Context) error { return returnArray(c, response) } + +// BlobProofs godoc +// +// @Summary Get blob inclusion proofs +// @Description Returns blob inclusion proofs +// @Tags namespace +// @ID get-blob-proof +// @Param request body postBlobRequest true "Request body containing height, commitment and namespace hash" +// @Accept json +// @Produce json +// @Success 200 {object} responses.BlobLog +// @Failure 400 {object} Error +// @Router /blob/proofs [get] +func (handler *NamespaceHandler) BlobProofs(c echo.Context) error { + req, err := bindAndValidate[postBlobRequest](c) + if err != nil { + return badRequestError(c, err) + } + + block, err := handler.node.Block(c.Request().Context(), req.Height) + if err != nil { + return handleError(c, err, handler.namespace) + } + + dataSquare, err := square.Construct( + block.Block.Data.Txs.ToSliceOfBytes(), + appconsts.SquareSizeUpperBound(0), + appconsts.SubtreeRootThreshold(0), + ) + + if err != nil { + return internalServerError(c, err) + } + + eds, err := da.ExtendShares(shares.ToBytes(dataSquare)) + if err != nil { + return internalServerError(c, err) + } + + ods, err := responses.NewODS(eds) + if err != nil { + return internalServerError(c, err) + } + + namespaceOds, err := ods.FindODSByNamespace(req.Hash) + if err != nil { + return internalServerError(c, err) + } + + namespaceShares, err := responses.GetNamespaceShares(eds, namespaceOds.From, namespaceOds.To) + if err != nil { + return internalServerError(c, err) + } + + startBlobIdx, endBlobIdx, err := responses.GetBlobShareIdxs( + namespaceShares, + namespaceOds.From, + eds.Width()/2, + req.Commitment, + ) + + proofs, err := handler.node.BlobProofs(c.Request().Context(), req.Height, startBlobIdx, endBlobIdx) + if err != nil { + return handleError(c, err, handler.namespace) + } + return c.JSON(http.StatusOK, proofs) +} diff --git a/cmd/api/handler/responses/ods.go b/cmd/api/handler/responses/ods.go index 3b7230a6..5fe2bf1c 100644 --- a/cmd/api/handler/responses/ods.go +++ b/cmd/api/handler/responses/ods.go @@ -4,10 +4,17 @@ package responses import ( + "bytes" "encoding/base64" - + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/go-square/namespace" "github.com/celestiaorg/go-square/shares" + incl "github.com/celestiaorg/go-square/v2/inclusion" + "github.com/celestiaorg/go-square/v2/share" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto/merkle" + + _ "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/rsmt2d" ) @@ -24,6 +31,36 @@ type ODSItem struct { Type NamespaceKind `json:"type"` } +type sequence struct { + ns share.Namespace + shareVersion uint8 + startShareIndex int + data []byte + sequenceLen uint32 + signer []byte +} + +const ( + // ShareVersionZero is the first share version format. + ShareVersionZero = uint8(0) + + // ShareVersionOne is the second share version format. + // It requires that a signer is included in the first share in the sequence. + ShareVersionOne = uint8(1) +) + +// SupportedShareVersions is a list of supported share versions. +var SupportedShareVersions = []uint8{ShareVersionZero, ShareVersionOne} + +func (ods *ODS) FindODSByNamespace(namespace string) (*ODSItem, error) { + for _, item := range ods.Items { + if item.Namespace == namespace { + return &item, nil + } + } + return nil, errors.New("item with specified namespace not found") +} + func NewODS(eds *rsmt2d.ExtendedDataSquare) (ODS, error) { ods := ODS{ Width: eds.Width() / 2, @@ -61,6 +98,40 @@ func NewODS(eds *rsmt2d.ExtendedDataSquare) (ODS, error) { return ods, nil } +func GetNamespaceShares(eds *rsmt2d.ExtendedDataSquare, from, to []uint) ([]share.Share, error) { + if len(from) != len(to) { + return nil, errors.New("length of 'from' and 'to' must match") + } + + var resultShares []share.Share + startRow, startCol := from[0], from[1] + endRow, endCol := to[0], to[1] + + if startRow > endRow || (startRow == endRow && startCol > endCol) { + return nil, errors.New("invalid from and to params") + } + currentRow, currentCol := startRow, startCol + + for { + cell := eds.GetCell(currentRow, currentCol) + cellShare, err := share.NewShare(cell) + if err != nil { + return nil, err + } + resultShares = append(resultShares, *cellShare) + if currentRow == endRow && currentCol == endCol { + break + } + currentCol++ + if currentCol == eds.Width()/2 { + currentCol = 0 + currentRow++ + } + } + + return resultShares, nil +} + type NamespaceKind string const ( @@ -88,3 +159,68 @@ func getNamespaceType(ns namespace.Namespace) NamespaceKind { return DefaultNamespace } } + +func GetBlobShareIdxs( + shares []share.Share, + nsStartFromIdx []uint, + edsWidth uint, + b64commitment string, +) (blobStartIdx, blobEndIdx int, err error) { + if len(shares) == 0 { + return 0, 0, errors.New("invalid shares length") + + } + sequences := make([]sequence, 0) + startRow, startCol := nsStartFromIdx[0], nsStartFromIdx[1] + nsStartIdx := int(startRow*edsWidth + startCol) + + for shareIdx, s := range shares { + if !bytes.Contains(SupportedShareVersions, []byte{s.Version()}) { + return 0, 0, errors.New("unsupported share version") + } + + if s.IsPadding() { + continue + } + + if s.IsSequenceStart() { + sequences = append(sequences, sequence{ + ns: s.Namespace(), + shareVersion: s.Version(), + startShareIndex: shareIdx, + data: s.RawData(), + sequenceLen: s.SequenceLen(), + signer: share.GetSigner(s), + }) + } else { + if len(sequences) == 0 { + return 0, 0, errors.New("continuation share without a sequence start share") + } + // FIXME: it doesn't look like we check whether all the shares belong to the same namespace. + prev := &sequences[len(sequences)-1] + prev.data = append(prev.data, s.RawData()...) + } + } + for i, seq := range sequences { + if int(seq.sequenceLen) < len(seq.data) { + seq.data = seq.data[:seq.sequenceLen] + } + + blob, err := share.NewBlob(seq.ns, seq.data, seq.shareVersion, seq.signer) + if err != nil { + return 0, 0, err + } + commitment, err := incl.CreateCommitment(blob, merkle.HashFromByteSlices, appconsts.SubtreeRootThreshold(0)) + if err != nil { + return 0, 0, err + } + if base64.StdEncoding.EncodeToString(commitment) == b64commitment { + if i == (len(sequences))-1 { // last sequence element + return seq.startShareIndex, seq.startShareIndex, err + } + + return nsStartIdx + seq.startShareIndex, nsStartIdx + sequences[i+1].startShareIndex - 1, err + } + } + return 0, 0, err +} diff --git a/cmd/api/init.go b/cmd/api/init.go index c8e54fb1..94a6f501 100644 --- a/cmd/api/init.go +++ b/cmd/api/init.go @@ -358,13 +358,23 @@ func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Sto panic(err) } - namespaceHandlers := handler.NewNamespaceHandler(db.Namespace, db.BlobLogs, db.Rollup, db.Address, db.State, cfg.Indexer.Name, blobReceiver) + namespaceHandlers := handler.NewNamespaceHandler( + db.Namespace, + db.BlobLogs, + db.Rollup, + db.Address, + db.State, + cfg.Indexer.Name, + blobReceiver, + &node, + ) blobGroup := v1.Group("/blob") { blobGroup.GET("", namespaceHandlers.Blobs) blobGroup.POST("", namespaceHandlers.Blob) blobGroup.POST("/metadata", namespaceHandlers.BlobMetadata) + blobGroup.POST("/proofs", namespaceHandlers.BlobProofs) } namespaceGroup := v1.Group("/namespace") diff --git a/go.mod b/go.mod index a1d9fb3b..c46d34ab 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/aws/smithy-go v1.20.2 github.com/celestiaorg/celestia-app/v3 v3.0.0-mocha github.com/celestiaorg/go-square v1.1.1 - github.com/celestiaorg/go-square/v2 v2.0.0 + github.com/celestiaorg/go-square/v2 v2.1.0 github.com/celestiaorg/rsmt2d v0.14.0 github.com/cosmos/cosmos-sdk v0.46.16 github.com/cosmos/ibc-go/v6 v6.2.2 diff --git a/go.sum b/go.sum index 7f8a6d89..a6432479 100644 --- a/go.sum +++ b/go.sum @@ -301,6 +301,8 @@ github.com/celestiaorg/go-square v1.1.1 h1:Cy3p8WVspVcyOqHM8BWFuuYPwMitO1pYGe+Im github.com/celestiaorg/go-square v1.1.1/go.mod h1:1EXMErhDrWJM8B8V9hN7dqJ2kUTClfwdqMOmF9yQUa0= github.com/celestiaorg/go-square/v2 v2.0.0 h1:U5QV8/de5lc7glosfgyHhcxbFwNuwU4+6aYZ2RgjM04= github.com/celestiaorg/go-square/v2 v2.0.0/go.mod h1:y0BolG0tRM7UN1sAQyDDUkT+aMJPwFIjviVvnCB62C0= +github.com/celestiaorg/go-square/v2 v2.1.0 h1:ECIvYEeHIWiIJGDCJxQNtzqm5DmnBly7XGhSpLsl+Lw= +github.com/celestiaorg/go-square/v2 v2.1.0/go.mod h1:n3ztrh8CBjWOD6iWYMo3pPOlQIgzLK9yrnqMPcNo6g8= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.22.2 h1:JmOMtZL9zWAed1hiwb9DDs+ELcKp/ZQZ3rPverge/V8= diff --git a/pkg/node/api.go b/pkg/node/api.go index 14a5b312..00b828ad 100644 --- a/pkg/node/api.go +++ b/pkg/node/api.go @@ -20,6 +20,7 @@ type Api interface { Genesis(ctx context.Context) (types.Genesis, error) BlockData(ctx context.Context, level pkgTypes.Level) (pkgTypes.BlockData, error) BlockDataGet(ctx context.Context, level pkgTypes.Level) (pkgTypes.BlockData, error) + BlobProofs(ctx context.Context, level pkgTypes.Level, startShare, endShare int) (pkgTypes.BlobProof, error) } //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed diff --git a/pkg/node/rpc/blob.go b/pkg/node/rpc/blob.go new file mode 100644 index 00000000..333a0215 --- /dev/null +++ b/pkg/node/rpc/blob.go @@ -0,0 +1,31 @@ +package rpc + +import ( + "context" + "github.com/celenium-io/celestia-indexer/pkg/node/types" + pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types" + "github.com/pkg/errors" + "strconv" +) + +const pathBlockProofs = "prove_shares_v2" + +func (api *API) BlobProofs(ctx context.Context, level pkgTypes.Level, startShare, endShare int) (pkgTypes.BlobProof, error) { + args := make(map[string]string) + if level != 0 { + args["height"] = strconv.FormatInt(int64(level), 10) + args["startShare"] = strconv.FormatInt(int64(startShare), 10) + args["endShare"] = strconv.FormatInt(int64(endShare), 10) + } + + var proof types.Response[pkgTypes.BlobProof] + if err := api.get(ctx, pathBlockProofs, args, &proof); err != nil { + return pkgTypes.BlobProof{}, errors.Wrap(err, "api.get") + } + + if proof.Error != nil { + return pkgTypes.BlobProof{}, errors.Wrapf(types.ErrRequest, "request %d error: %s", proof.Id, proof.Error.Error()) + } + + return proof.Result, nil +} diff --git a/pkg/types/blobProofs.go b/pkg/types/blobProofs.go new file mode 100644 index 00000000..2fa75f69 --- /dev/null +++ b/pkg/types/blobProofs.go @@ -0,0 +1,29 @@ +package types + +// BlobProof - +type BlobProof struct { + ShareProof ShareProofData `json:"share_proof"` +} + +// ShareProofData - +type ShareProofData struct { + ShareProofs []ShareProof `json:"share_proofs"` + RowProof RowProof `json:"row_proof"` +} + +// ShareProof - +type ShareProof struct { + Start int `json:"start"` + End int `json:"end"` + Nodes []string `json:"nodes"` +} + +// RowProof - +type RowProof struct { + Proofs []Proof `json:"proofs"` +} + +// Proof - +type Proof struct { + LeafHash string `json:"leaf_hash"` +} From b6789b207d441664d415d558a6d27454f47229c2 Mon Sep 17 00:00:00 2001 From: Konst Date: Fri, 29 Nov 2024 17:19:19 +0300 Subject: [PATCH 03/12] Added test cases and linter fixes --- cmd/api/handler/namespace.go | 3 +++ cmd/api/handler/namespace_test.go | 12 ++++++++- pkg/node/mock/api.go | 44 ++++++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/cmd/api/handler/namespace.go b/cmd/api/handler/namespace.go index 76610125..9886a186 100644 --- a/cmd/api/handler/namespace.go +++ b/cmd/api/handler/namespace.go @@ -800,6 +800,9 @@ func (handler *NamespaceHandler) BlobProofs(c echo.Context) error { eds.Width()/2, req.Commitment, ) + if err != nil { + return internalServerError(c, err) + } proofs, err := handler.node.BlobProofs(c.Request().Context(), req.Height, startBlobIdx, endBlobIdx) if err != nil { diff --git a/cmd/api/handler/namespace_test.go b/cmd/api/handler/namespace_test.go index 3aa1e497..039a79de 100644 --- a/cmd/api/handler/namespace_test.go +++ b/cmd/api/handler/namespace_test.go @@ -51,6 +51,7 @@ type NamespaceTestSuite struct { address *mock.MockIAddress state *mock.MockIState blobReceiver *nodeMock.MockDalApi + node *nodeMock.MockApi echo *echo.Echo handler *NamespaceHandler ctrl *gomock.Controller @@ -67,7 +68,16 @@ func (s *NamespaceTestSuite) SetupSuite() { s.rollups = mock.NewMockIRollup(s.ctrl) s.state = mock.NewMockIState(s.ctrl) s.blobReceiver = nodeMock.NewMockDalApi(s.ctrl) - s.handler = NewNamespaceHandler(s.namespaces, s.blobLogs, s.rollups, s.address, s.state, testIndexerName, s.blobReceiver) + s.handler = NewNamespaceHandler( + s.namespaces, + s.blobLogs, + s.rollups, + s.address, + s.state, + testIndexerName, + s.blobReceiver, + s.node, + ) } // TearDownSuite - diff --git a/pkg/node/mock/api.go b/pkg/node/mock/api.go index e3c6495c..23440607 100644 --- a/pkg/node/mock/api.go +++ b/pkg/node/mock/api.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2024 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: api.go // @@ -25,6 +22,7 @@ import ( type MockApi struct { ctrl *gomock.Controller recorder *MockApiMockRecorder + isgomock struct{} } // MockApiMockRecorder is the mock recorder for MockApi. @@ -44,6 +42,45 @@ func (m *MockApi) EXPECT() *MockApiMockRecorder { return m.recorder } +// BlobProofs mocks base method. +func (m *MockApi) BlobProofs(ctx context.Context, level types0.Level, startShare, endShare int) (types0.BlobProof, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BlobProofs", ctx, level, startShare, endShare) + ret0, _ := ret[0].(types0.BlobProof) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BlobProofs indicates an expected call of BlobProofs. +func (mr *MockApiMockRecorder) BlobProofs(ctx, level, startShare, endShare any) *MockApiBlobProofsCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlobProofs", reflect.TypeOf((*MockApi)(nil).BlobProofs), ctx, level, startShare, endShare) + return &MockApiBlobProofsCall{Call: call} +} + +// MockApiBlobProofsCall wrap *gomock.Call +type MockApiBlobProofsCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockApiBlobProofsCall) Return(arg0 types0.BlobProof, arg1 error) *MockApiBlobProofsCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockApiBlobProofsCall) Do(f func(context.Context, types0.Level, int, int) (types0.BlobProof, error)) *MockApiBlobProofsCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockApiBlobProofsCall) DoAndReturn(f func(context.Context, types0.Level, int, int) (types0.BlobProof, error)) *MockApiBlobProofsCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // Block mocks base method. func (m *MockApi) Block(ctx context.Context, level types0.Level) (types0.ResultBlock, error) { m.ctrl.T.Helper() @@ -321,6 +358,7 @@ func (c *MockApiStatusCall) DoAndReturn(f func(context.Context) (types.Status, e type MockDalApi struct { ctrl *gomock.Controller recorder *MockDalApiMockRecorder + isgomock struct{} } // MockDalApiMockRecorder is the mock recorder for MockDalApi. From 4d7542e9564751e5e963d7d6738399677ae90df4 Mon Sep 17 00:00:00 2001 From: Konst Date: Mon, 2 Dec 2024 21:09:06 +0300 Subject: [PATCH 04/12] Review fixes --- cmd/api/handler/responses/ods.go | 24 +++++------------------- pkg/node/rpc/blob.go | 22 +++++++++++++++------- pkg/types/blobProofs.go | 3 +++ 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/cmd/api/handler/responses/ods.go b/cmd/api/handler/responses/ods.go index 5fe2bf1c..25989f48 100644 --- a/cmd/api/handler/responses/ods.go +++ b/cmd/api/handler/responses/ods.go @@ -4,7 +4,6 @@ package responses import ( - "bytes" "encoding/base64" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/go-square/namespace" @@ -40,18 +39,6 @@ type sequence struct { signer []byte } -const ( - // ShareVersionZero is the first share version format. - ShareVersionZero = uint8(0) - - // ShareVersionOne is the second share version format. - // It requires that a signer is included in the first share in the sequence. - ShareVersionOne = uint8(1) -) - -// SupportedShareVersions is a list of supported share versions. -var SupportedShareVersions = []uint8{ShareVersionZero, ShareVersionOne} - func (ods *ODS) FindODSByNamespace(namespace string) (*ODSItem, error) { for _, item := range ods.Items { if item.Namespace == namespace { @@ -175,7 +162,7 @@ func GetBlobShareIdxs( nsStartIdx := int(startRow*edsWidth + startCol) for shareIdx, s := range shares { - if !bytes.Contains(SupportedShareVersions, []byte{s.Version()}) { + if !(s.Version() <= 1) { return 0, 0, errors.New("unsupported share version") } @@ -188,7 +175,7 @@ func GetBlobShareIdxs( ns: s.Namespace(), shareVersion: s.Version(), startShareIndex: shareIdx, - data: s.RawData(), + data: s.RawData(), // todo: remove all fields except this sequenceLen: s.SequenceLen(), signer: share.GetSigner(s), }) @@ -196,15 +183,14 @@ func GetBlobShareIdxs( if len(sequences) == 0 { return 0, 0, errors.New("continuation share without a sequence start share") } - // FIXME: it doesn't look like we check whether all the shares belong to the same namespace. prev := &sequences[len(sequences)-1] prev.data = append(prev.data, s.RawData()...) } } for i, seq := range sequences { - if int(seq.sequenceLen) < len(seq.data) { - seq.data = seq.data[:seq.sequenceLen] - } + //if int(seq.sequenceLen) < len(seq.data) { + seq.data = seq.data[:seq.sequenceLen] + //} blob, err := share.NewBlob(seq.ns, seq.data, seq.shareVersion, seq.signer) if err != nil { diff --git a/pkg/node/rpc/blob.go b/pkg/node/rpc/blob.go index 333a0215..19dfdacb 100644 --- a/pkg/node/rpc/blob.go +++ b/pkg/node/rpc/blob.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + package rpc import ( @@ -8,18 +11,23 @@ import ( "strconv" ) -const pathBlockProofs = "prove_shares_v2" +const pathBlobProofs = "prove_shares_v2" func (api *API) BlobProofs(ctx context.Context, level pkgTypes.Level, startShare, endShare int) (pkgTypes.BlobProof, error) { - args := make(map[string]string) - if level != 0 { - args["height"] = strconv.FormatInt(int64(level), 10) - args["startShare"] = strconv.FormatInt(int64(startShare), 10) - args["endShare"] = strconv.FormatInt(int64(endShare), 10) + if startShare < 0 || endShare < 0 { + return pkgTypes.BlobProof{}, errors.New("params 'startShare' and 'endShare' should not be lower than 0") + } + if level <= 0 { + return pkgTypes.BlobProof{}, errors.New("param 'level' should be greater than 0") } + args := make(map[string]string) + args["height"] = strconv.FormatInt(int64(level), 10) + args["startShare"] = strconv.FormatInt(int64(startShare), 10) + args["endShare"] = strconv.FormatInt(int64(endShare), 10) + var proof types.Response[pkgTypes.BlobProof] - if err := api.get(ctx, pathBlockProofs, args, &proof); err != nil { + if err := api.get(ctx, pathBlobProofs, args, &proof); err != nil { return pkgTypes.BlobProof{}, errors.Wrap(err, "api.get") } diff --git a/pkg/types/blobProofs.go b/pkg/types/blobProofs.go index 2fa75f69..120b4d3d 100644 --- a/pkg/types/blobProofs.go +++ b/pkg/types/blobProofs.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + package types // BlobProof - From 063f45b3da00b30d4b6aca36d120371b4f90f712 Mon Sep 17 00:00:00 2001 From: Konst Date: Mon, 2 Dec 2024 21:11:50 +0300 Subject: [PATCH 05/12] Updated routes test --- cmd/api/routes_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/api/routes_test.go b/cmd/api/routes_test.go index fe66325e..52f92bce 100644 --- a/cmd/api/routes_test.go +++ b/cmd/api/routes_test.go @@ -104,6 +104,7 @@ func TestRoutes(t *testing.T) { "/v1/stats/messages_count_24h GET": {}, "/v1/rollup/stats/series GET": {}, "/v1/rollup/group GET": {}, + "/v1/blob/proofs POST": {}, } ctx, cancel := context.WithCancel(context.Background()) From 2edf8aa90b5864c41d4d41ae5d264c5b094bb036 Mon Sep 17 00:00:00 2001 From: Konst Date: Mon, 2 Dec 2024 21:14:15 +0300 Subject: [PATCH 06/12] Removed commented conditional seq.data trimming in GetBlobShareIdxs func --- cmd/api/handler/responses/ods.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/api/handler/responses/ods.go b/cmd/api/handler/responses/ods.go index 25989f48..f44fe02d 100644 --- a/cmd/api/handler/responses/ods.go +++ b/cmd/api/handler/responses/ods.go @@ -188,10 +188,7 @@ func GetBlobShareIdxs( } } for i, seq := range sequences { - //if int(seq.sequenceLen) < len(seq.data) { seq.data = seq.data[:seq.sequenceLen] - //} - blob, err := share.NewBlob(seq.ns, seq.data, seq.shareVersion, seq.signer) if err != nil { return 0, 0, err From a3d31b26c4b079848048165565269a38a581cc45 Mon Sep 17 00:00:00 2001 From: Konst Date: Tue, 3 Dec 2024 18:20:51 +0300 Subject: [PATCH 07/12] Correct proofs --- cmd/api/handler/responses/ods.go | 35 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/cmd/api/handler/responses/ods.go b/cmd/api/handler/responses/ods.go index f44fe02d..d0977665 100644 --- a/cmd/api/handler/responses/ods.go +++ b/cmd/api/handler/responses/ods.go @@ -31,12 +31,13 @@ type ODSItem struct { } type sequence struct { - ns share.Namespace - shareVersion uint8 - startShareIndex int - data []byte - sequenceLen uint32 - signer []byte + ns share.Namespace + shareVersion uint8 + startShareIdx int + endShareIdx int + data []byte + sequenceLen uint32 + signer []byte } func (ods *ODS) FindODSByNamespace(namespace string) (*ODSItem, error) { @@ -172,12 +173,13 @@ func GetBlobShareIdxs( if s.IsSequenceStart() { sequences = append(sequences, sequence{ - ns: s.Namespace(), - shareVersion: s.Version(), - startShareIndex: shareIdx, - data: s.RawData(), // todo: remove all fields except this - sequenceLen: s.SequenceLen(), - signer: share.GetSigner(s), + ns: s.Namespace(), + shareVersion: s.Version(), + startShareIdx: shareIdx, + endShareIdx: shareIdx, + data: s.RawData(), + sequenceLen: s.SequenceLen(), + signer: share.GetSigner(s), }) } else { if len(sequences) == 0 { @@ -185,9 +187,10 @@ func GetBlobShareIdxs( } prev := &sequences[len(sequences)-1] prev.data = append(prev.data, s.RawData()...) + prev.endShareIdx = shareIdx } } - for i, seq := range sequences { + for _, seq := range sequences { seq.data = seq.data[:seq.sequenceLen] blob, err := share.NewBlob(seq.ns, seq.data, seq.shareVersion, seq.signer) if err != nil { @@ -198,11 +201,7 @@ func GetBlobShareIdxs( return 0, 0, err } if base64.StdEncoding.EncodeToString(commitment) == b64commitment { - if i == (len(sequences))-1 { // last sequence element - return seq.startShareIndex, seq.startShareIndex, err - } - - return nsStartIdx + seq.startShareIndex, nsStartIdx + sequences[i+1].startShareIndex - 1, err + return nsStartIdx + seq.startShareIdx, nsStartIdx + seq.endShareIdx + 1, err } } return 0, 0, err From c2c1f581415995d35b90c821b704fcd5d8e6819a Mon Sep 17 00:00:00 2001 From: Konst Date: Tue, 3 Dec 2024 18:30:38 +0300 Subject: [PATCH 08/12] Removed RowProof from API response --- pkg/types/blobProofs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/types/blobProofs.go b/pkg/types/blobProofs.go index 120b4d3d..1a12f0a2 100644 --- a/pkg/types/blobProofs.go +++ b/pkg/types/blobProofs.go @@ -11,7 +11,6 @@ type BlobProof struct { // ShareProofData - type ShareProofData struct { ShareProofs []ShareProof `json:"share_proofs"` - RowProof RowProof `json:"row_proof"` } // ShareProof - From 492c4e4df52082d92d3c8c021b6e1e862915ec61 Mon Sep 17 00:00:00 2001 From: Konst Date: Wed, 4 Dec 2024 19:38:39 +0300 Subject: [PATCH 09/12] Refactor: removed EDS and ODS usage, finding blob share indexes straightly from data square --- cmd/api/handler/namespace.go | 33 +-------- cmd/api/handler/responses/ods.go | 111 +++++++++++-------------------- 2 files changed, 40 insertions(+), 104 deletions(-) diff --git a/cmd/api/handler/namespace.go b/cmd/api/handler/namespace.go index 9886a186..3aac2352 100644 --- a/cmd/api/handler/namespace.go +++ b/cmd/api/handler/namespace.go @@ -8,19 +8,17 @@ import ( "encoding/base64" "encoding/hex" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" - "github.com/celestiaorg/celestia-app/v3/pkg/da" - "github.com/celestiaorg/go-square/shares" "net/http" "time" "github.com/celenium-io/celestia-indexer/pkg/types" + "github.com/celestiaorg/go-square/v2" sdk "github.com/dipdup-net/indexer-sdk/pkg/storage" "github.com/celenium-io/celestia-indexer/cmd/api/handler/responses" "github.com/celenium-io/celestia-indexer/internal/storage" testsuite "github.com/celenium-io/celestia-indexer/internal/test_suite" "github.com/celenium-io/celestia-indexer/pkg/node" - "github.com/celestiaorg/go-square/square" "github.com/labstack/echo/v4" ) @@ -774,37 +772,12 @@ func (handler *NamespaceHandler) BlobProofs(c echo.Context) error { return internalServerError(c, err) } - eds, err := da.ExtendShares(shares.ToBytes(dataSquare)) + startBlobIndex, endBlobIndex, err := responses.GetBlobShareIndexes(dataSquare, req.Hash, req.Commitment) if err != nil { return internalServerError(c, err) } - ods, err := responses.NewODS(eds) - if err != nil { - return internalServerError(c, err) - } - - namespaceOds, err := ods.FindODSByNamespace(req.Hash) - if err != nil { - return internalServerError(c, err) - } - - namespaceShares, err := responses.GetNamespaceShares(eds, namespaceOds.From, namespaceOds.To) - if err != nil { - return internalServerError(c, err) - } - - startBlobIdx, endBlobIdx, err := responses.GetBlobShareIdxs( - namespaceShares, - namespaceOds.From, - eds.Width()/2, - req.Commitment, - ) - if err != nil { - return internalServerError(c, err) - } - - proofs, err := handler.node.BlobProofs(c.Request().Context(), req.Height, startBlobIdx, endBlobIdx) + proofs, err := handler.node.BlobProofs(c.Request().Context(), req.Height, startBlobIndex, endBlobIndex) if err != nil { return handleError(c, err, handler.namespace) } diff --git a/cmd/api/handler/responses/ods.go b/cmd/api/handler/responses/ods.go index d0977665..c72c29c4 100644 --- a/cmd/api/handler/responses/ods.go +++ b/cmd/api/handler/responses/ods.go @@ -4,11 +4,12 @@ package responses import ( + "bytes" "encoding/base64" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/go-square/namespace" "github.com/celestiaorg/go-square/shares" - incl "github.com/celestiaorg/go-square/v2/inclusion" + "github.com/celestiaorg/go-square/v2/inclusion" "github.com/celestiaorg/go-square/v2/share" "github.com/pkg/errors" "github.com/tendermint/tendermint/crypto/merkle" @@ -30,25 +31,6 @@ type ODSItem struct { Type NamespaceKind `json:"type"` } -type sequence struct { - ns share.Namespace - shareVersion uint8 - startShareIdx int - endShareIdx int - data []byte - sequenceLen uint32 - signer []byte -} - -func (ods *ODS) FindODSByNamespace(namespace string) (*ODSItem, error) { - for _, item := range ods.Items { - if item.Namespace == namespace { - return &item, nil - } - } - return nil, errors.New("item with specified namespace not found") -} - func NewODS(eds *rsmt2d.ExtendedDataSquare) (ODS, error) { ods := ODS{ Width: eds.Width() / 2, @@ -86,40 +68,6 @@ func NewODS(eds *rsmt2d.ExtendedDataSquare) (ODS, error) { return ods, nil } -func GetNamespaceShares(eds *rsmt2d.ExtendedDataSquare, from, to []uint) ([]share.Share, error) { - if len(from) != len(to) { - return nil, errors.New("length of 'from' and 'to' must match") - } - - var resultShares []share.Share - startRow, startCol := from[0], from[1] - endRow, endCol := to[0], to[1] - - if startRow > endRow || (startRow == endRow && startCol > endCol) { - return nil, errors.New("invalid from and to params") - } - currentRow, currentCol := startRow, startCol - - for { - cell := eds.GetCell(currentRow, currentCol) - cellShare, err := share.NewShare(cell) - if err != nil { - return nil, err - } - resultShares = append(resultShares, *cellShare) - if currentRow == endRow && currentCol == endCol { - break - } - currentCol++ - if currentCol == eds.Width()/2 { - currentCol = 0 - currentRow++ - } - } - - return resultShares, nil -} - type NamespaceKind string const ( @@ -148,21 +96,32 @@ func getNamespaceType(ns namespace.Namespace) NamespaceKind { } } -func GetBlobShareIdxs( +type sequence struct { + ns share.Namespace + shareVersion uint8 + startShareIdx int + endShareIdx int + data []byte + sequenceLen uint32 + signer []byte +} + +func GetBlobShareIndexes( shares []share.Share, - nsStartFromIdx []uint, - edsWidth uint, - b64commitment string, -) (blobStartIdx, blobEndIdx int, err error) { + base64namespace string, + base64commitment string, +) (blobStartIndex, blobEndIndex int, err error) { if len(shares) == 0 { return 0, 0, errors.New("invalid shares length") } sequences := make([]sequence, 0) - startRow, startCol := nsStartFromIdx[0], nsStartFromIdx[1] - nsStartIdx := int(startRow*edsWidth + startCol) + namespaceBytes, err := base64.StdEncoding.DecodeString(base64namespace) + if err != nil { + return 0, 0, errors.Wrap(err, "decoding base64 namespace") + } - for shareIdx, s := range shares { + for shareIndex, s := range shares { if !(s.Version() <= 1) { return 0, 0, errors.New("unsupported share version") } @@ -171,37 +130,41 @@ func GetBlobShareIdxs( continue } + if !bytes.Equal(s.Namespace().Bytes(), namespaceBytes) { + continue + } + if s.IsSequenceStart() { sequences = append(sequences, sequence{ ns: s.Namespace(), shareVersion: s.Version(), - startShareIdx: shareIdx, - endShareIdx: shareIdx, + startShareIdx: shareIndex, + endShareIdx: shareIndex, data: s.RawData(), sequenceLen: s.SequenceLen(), signer: share.GetSigner(s), }) } else { if len(sequences) == 0 { - return 0, 0, errors.New("continuation share without a sequence start share") + return 0, 0, errors.New("continuation share without a s start share") } prev := &sequences[len(sequences)-1] prev.data = append(prev.data, s.RawData()...) - prev.endShareIdx = shareIdx + prev.endShareIdx = shareIndex } } - for _, seq := range sequences { - seq.data = seq.data[:seq.sequenceLen] - blob, err := share.NewBlob(seq.ns, seq.data, seq.shareVersion, seq.signer) + for _, s := range sequences { + s.data = s.data[:s.sequenceLen] + blob, err := share.NewBlob(s.ns, s.data, s.shareVersion, s.signer) if err != nil { - return 0, 0, err + return 0, 0, errors.Wrap(err, "creating blob") } - commitment, err := incl.CreateCommitment(blob, merkle.HashFromByteSlices, appconsts.SubtreeRootThreshold(0)) + commitment, err := inclusion.CreateCommitment(blob, merkle.HashFromByteSlices, appconsts.SubtreeRootThreshold(0)) if err != nil { - return 0, 0, err + return 0, 0, errors.Wrap(err, "creating commitment") } - if base64.StdEncoding.EncodeToString(commitment) == b64commitment { - return nsStartIdx + seq.startShareIdx, nsStartIdx + seq.endShareIdx + 1, err + if base64.StdEncoding.EncodeToString(commitment) == base64commitment { + return s.startShareIdx, s.endShareIdx + 1, err } } return 0, 0, err From 3bc3c79a42176d52e3ba7983fc4edf6874d5b314 Mon Sep 17 00:00:00 2001 From: Konst Date: Wed, 4 Dec 2024 21:19:24 +0300 Subject: [PATCH 10/12] Refactor: blob proofs querying replaced by NewShareInclusionProofFromEDS from celestia-app --- cmd/api/handler/namespace.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/api/handler/namespace.go b/cmd/api/handler/namespace.go index 3aac2352..d5752f09 100644 --- a/cmd/api/handler/namespace.go +++ b/cmd/api/handler/namespace.go @@ -8,6 +8,8 @@ import ( "encoding/base64" "encoding/hex" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/da" + "github.com/celestiaorg/go-square/v2/share" "net/http" "time" @@ -19,6 +21,7 @@ import ( "github.com/celenium-io/celestia-indexer/internal/storage" testsuite "github.com/celenium-io/celestia-indexer/internal/test_suite" "github.com/celenium-io/celestia-indexer/pkg/node" + "github.com/celestiaorg/celestia-app/v3/pkg/proof" "github.com/labstack/echo/v4" ) @@ -776,10 +779,30 @@ func (handler *NamespaceHandler) BlobProofs(c echo.Context) error { if err != nil { return internalServerError(c, err) } + blobSharesRange := share.Range{ + Start: startBlobIndex, + End: endBlobIndex, + } + + eds, err := da.ExtendShares(share.ToBytes(dataSquare)) + if err != nil { + return internalServerError(c, err) + } + + namespaceBytes, err := base64.StdEncoding.DecodeString(req.Hash) + if err != nil { + return internalServerError(c, err) + } - proofs, err := handler.node.BlobProofs(c.Request().Context(), req.Height, startBlobIndex, endBlobIndex) + namespace, err := share.NewNamespaceFromBytes(namespaceBytes) + if err != nil { + return internalServerError(c, err) + } + + proofs, err := proof.NewShareInclusionProofFromEDS(eds, namespace, blobSharesRange) if err != nil { return handleError(c, err, handler.namespace) } - return c.JSON(http.StatusOK, proofs) + + return c.JSON(http.StatusOK, proofs.ShareProofs) } From 9914c8a35c60c5a3392432ed25cd12d8083d2d1a Mon Sep 17 00:00:00 2001 From: Konst Date: Wed, 4 Dec 2024 22:33:15 +0300 Subject: [PATCH 11/12] Added tests for API BlobProofs method --- cmd/api/handler/namespace_test.go | 119 ++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/cmd/api/handler/namespace_test.go b/cmd/api/handler/namespace_test.go index 039a79de..9a3a0f74 100644 --- a/cmd/api/handler/namespace_test.go +++ b/cmd/api/handler/namespace_test.go @@ -9,6 +9,7 @@ import ( "crypto/rand" "encoding/base64" "encoding/json" + "github.com/celestiaorg/celestia-app/v3/pkg/proof" "net/http" "net/http/httptest" "net/url" @@ -25,6 +26,7 @@ import ( nodeTypes "github.com/celenium-io/celestia-indexer/pkg/node/types" "github.com/labstack/echo/v4" "github.com/stretchr/testify/suite" + tendermintTypes "github.com/tendermint/tendermint/types" "go.uber.org/mock/gomock" ) @@ -40,6 +42,62 @@ var ( } testNamespaceId = "0000000000000000000000000000000000000000fc7443b155920156" testNamespaceBase64 = "AAAAAAAAAAAAAAAAAAAAAAAAAAAA/HRDsVWSAVY=" + testTxBytes = []byte{ + 0xa, 0xcb, 0x2, 0xa, 0xa0, 0x1, 0xa, 0x9d, 0x1, 0xa, 0x20, 0x2f, 0x63, 0x65, 0x6c, 0x65, 0x73, 0x74, 0x69, 0x61, + 0x2e, 0x62, 0x6c, 0x6f, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x73, 0x67, 0x50, 0x61, 0x79, 0x46, 0x6f, 0x72, + 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x79, 0xa, 0x2f, 0x63, 0x65, 0x6c, 0x65, 0x73, 0x74, 0x69, 0x61, 0x31, 0x73, + 0x73, 0x77, 0x7a, 0x77, 0x33, 0x6d, 0x74, 0x66, 0x75, 0x65, 0x38, 0x70, 0x7a, 0x79, 0x32, 0x79, 0x6d, 0x63, + 0x37, 0x6a, 0x61, 0x78, 0x64, 0x30, 0x79, 0x72, 0x79, 0x37, 0x72, 0x6b, 0x76, 0x34, 0x34, 0x6e, 0x32, 0x75, + 0x73, 0x12, 0x1d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x56, 0x49, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1a, 0x2, 0xab, 0x5, 0x22, 0x20, 0x86, 0x8c, 0xd6, 0x1d, + 0x4b, 0x1c, 0x90, 0xc5, 0xdd, 0x6, 0x10, 0x4e, 0xd7, 0x94, 0xcb, 0xae, 0x26, 0xf8, 0xac, 0xeb, 0x69, 0x80, 0xdb, + 0xbf, 0x1f, 0x2b, 0xb6, 0xdb, 0x98, 0x23, 0x14, 0xb2, 0x42, 0x1, 0x0, 0x12, 0x64, 0xa, 0x4e, 0xa, 0x46, 0xa, + 0x1f, 0x2f, 0x63, 0x6f, 0x73, 0x6d, 0x6f, 0x73, 0x2e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x2e, 0x73, 0x65, + 0x63, 0x70, 0x32, 0x35, 0x36, 0x6b, 0x31, 0x2e, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x23, 0xa, 0x21, 0x2, + 0x1e, 0x73, 0x47, 0xc3, 0x17, 0x85, 0x7d, 0xe, 0xa4, 0x47, 0x8f, 0x71, 0xb, 0x3, 0xbe, 0x97, 0x18, 0x84, 0x3, + 0xee, 0x91, 0x88, 0xef, 0xfe, 0x89, 0x28, 0x2c, 0x27, 0x13, 0xdc, 0x84, 0x66, 0x12, 0x4, 0xa, 0x2, 0x8, 0x1, + 0x12, 0x12, 0xa, 0xc, 0xa, 0x4, 0x75, 0x74, 0x69, 0x61, 0x12, 0x4, 0x39, 0x32, 0x32, 0x39, 0x10, 0xf9, 0xd0, + 0x5, 0x1a, 0x40, 0xe7, 0x8d, 0x57, 0x47, 0xeb, 0x49, 0xd0, 0xd4, 0x5b, 0x14, 0x98, 0xa, 0xcd, 0xcb, 0xc3, 0x5c, + 0x89, 0x88, 0xe1, 0x69, 0x4c, 0x2a, 0x64, 0x76, 0xb7, 0x74, 0x56, 0x3a, 0x81, 0x7a, 0x5c, 0x20, 0x51, 0x88, + 0x37, 0x35, 0xc7, 0x22, 0xa8, 0x0, 0x30, 0x47, 0x91, 0xce, 0xf3, 0xba, 0xf1, 0x7f, 0xa, 0xed, 0xe1, 0xb4, 0x38, + 0xce, 0xe8, 0x69, 0xe, 0x74, 0xcf, 0x24, 0x86, 0xbf, 0x7, 0x48, 0x12, 0xcc, 0x5, 0xa, 0x1c, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x56, 0x49, 0x41, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x12, 0xab, 0x5, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x80, 0x1, 0x1, 0x16, 0x96, 0xe5, 0xba, 0x4e, 0x9a, 0x5, 0x5a, + 0xaf, 0xe8, 0x86, 0x47, 0xaf, 0xe3, 0xa1, 0x38, 0x90, 0x65, 0x40, 0x54, 0x8f, 0xba, 0xc9, 0x3a, 0x5a, 0x7f, + 0xdf, 0x97, 0xc6, 0x32, 0x57, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x1, 0x0, 0x2, 0x42, 0x4, 0x0, 0xc, 0x71, 0xe9, 0x17, 0x21, 0xf9, 0x91, 0x85, 0x76, 0xd7, 0x60, 0xf0, + 0x2f, 0x3, 0xca, 0xc4, 0x7c, 0x6f, 0x40, 0x3, 0x31, 0x60, 0x31, 0x84, 0x8e, 0x3c, 0x1d, 0x99, 0xe6, 0xe8, 0x3a, + 0x47, 0x43, 0x29, 0x2, 0x54, 0x39, 0xaa, 0xc0, 0x2d, 0x5a, 0x69, 0x62, 0xcc, 0xce, 0xe5, 0xd4, 0xad, 0xb4, 0x8a, + 0x36, 0xbb, 0xbf, 0x44, 0x3a, 0x53, 0x17, 0x21, 0x48, 0x43, 0x81, 0x12, 0x59, 0x37, 0xf3, 0x0, 0x1a, 0xc5, 0xff, + 0x87, 0x5b, 0x19, 0x28, 0x7b, 0x98, 0x8d, 0x61, 0x7e, 0xc0, 0x5a, 0xcb, 0xbf, 0x5f, 0xe2, 0x45, 0x29, 0xa6, + 0x4b, 0x23, 0x85, 0xa9, 0x6a, 0xad, 0x43, 0xf0, 0x9b, 0xe1, 0xad, 0xa9, 0x2c, 0x70, 0x40, 0x31, 0xdc, 0xc1, + 0x48, 0x1b, 0x29, 0x2, 0x54, 0x11, 0x2f, 0x28, 0xa2, 0x54, 0x20, 0xc1, 0xd9, 0xd7, 0x5, 0x35, 0x8c, 0x13, 0x4c, + 0xc6, 0x1, 0xd9, 0xd1, 0x84, 0xcb, 0x4d, 0xfd, 0xde, 0x7e, 0x1c, 0xac, 0x2b, 0xc3, 0xd4, 0xd3, 0x8b, 0xf9, 0xec, + 0x44, 0xe6, 0x9, 0x64, 0x12, 0x3b, 0xaf, 0xc5, 0x86, 0xf7, 0x77, 0x64, 0x48, 0x8c, 0xd2, 0x4c, 0x6a, 0x77, 0x54, + 0x6e, 0x5a, 0xf, 0xe8, 0xbd, 0xfb, 0x4f, 0xa2, 0x3, 0xcf, 0xaf, 0xfc, 0x36, 0xcc, 0xe4, 0xdd, 0x5b, 0x89, 0x1, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x67, 0x50, 0x70, 0x0, 0x8e, 0x7d, 0xd0, 0x6a, 0xc5, + 0xb7, 0x3b, 0x47, 0x3b, 0xe6, 0xbc, 0x5a, 0x51, 0x3, 0xf, 0x4c, 0x74, 0x37, 0x65, 0x7c, 0xb7, 0xb2, 0x9b, 0xf3, + 0x76, 0xc5, 0x64, 0xb8, 0xd1, 0x67, 0x5a, 0x5e, 0x89, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x67, 0x50, 0x70, 0x1, 0x4b, 0xa8, 0x4e, 0x1f, 0x37, 0xd0, 0x41, 0xbc, 0x6e, 0x55, 0xba, 0x39, 0x68, + 0x26, 0xcc, 0x49, 0x4e, 0x84, 0xd4, 0x81, 0x5b, 0x6d, 0xb5, 0x26, 0x90, 0x42, 0x2e, 0xea, 0x73, 0x86, 0x31, + 0x4f, 0x0, 0xe8, 0xe7, 0x76, 0x26, 0x58, 0x6f, 0x73, 0xb9, 0x55, 0x36, 0x4c, 0x7b, 0x4b, 0xbf, 0xb, 0xb7, 0xf7, + 0x68, 0x5e, 0xbd, 0x40, 0xe8, 0x52, 0xb1, 0x64, 0x63, 0x3a, 0x4a, 0xcb, 0xd3, 0x24, 0x4c, 0x3d, 0xe2, 0x20, + 0x2c, 0xcb, 0x62, 0x6a, 0xd3, 0x87, 0xd7, 0x7, 0x22, 0xe6, 0x4f, 0xbe, 0x44, 0x56, 0x2e, 0x2f, 0x23, 0x1a, 0x29, + 0xc, 0x8, 0x53, 0x2b, 0x8d, 0x6a, 0xba, 0x40, 0x2f, 0xf5, 0x0, 0x96, 0xf9, 0x7a, 0x20, 0x84, 0x4a, 0x4f, 0xba, + 0x58, 0x29, 0x5, 0x1, 0x37, 0xa, 0xa1, 0x44, 0xb7, 0x7d, 0x26, 0x25, 0x3a, 0x38, 0x4d, 0x27, 0x4a, 0xec, 0x57, + 0x22, 0xce, 0x21, 0xf8, 0xf1, 0x79, 0x9, 0x35, 0x88, 0xd0, 0xe8, 0x47, 0xef, 0xa7, 0x3a, 0x10, 0xce, 0x20, 0xe4, + 0x79, 0x9f, 0xb1, 0xe4, 0x66, 0x42, 0xd6, 0x56, 0x17, 0xc7, 0xe5, 0x21, 0x3f, 0xa0, 0x49, 0x89, 0xd9, 0x2d, + 0x89, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x67, 0x50, 0x70, 0x1, 0x87, 0xde, 0xd2, + 0x47, 0xe1, 0x66, 0xf, 0x82, 0x70, 0x71, 0xc7, 0xf1, 0x37, 0x19, 0x34, 0x58, 0x97, 0x51, 0x8, 0x53, 0x84, 0xfc, + 0x9f, 0x44, 0x62, 0xc1, 0xf1, 0x89, 0x7c, 0x5c, 0x3e, 0xef, 0x89, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x86, 0x24, 0x81, 0x93, 0xeb, 0x4d, 0xd2, 0xa8, 0xce, 0x81, 0x5f, 0x87, + 0x6c, 0x12, 0x4d, 0x48, 0x35, 0x95, 0x22, 0xf0, 0x85, 0x4d, 0x95, 0xd8, 0x7, 0x2e, 0xaf, 0xf0, 0xd3, 0x7d, 0x55, + 0xbd, 0x11, 0x3, 0x20, 0x91, 0x1d, 0xd2, 0xad, 0x74, 0x3f, 0xf2, 0x37, 0xd4, 0x11, 0x64, 0x8a, 0xf, 0xe3, 0x2c, + 0x6d, 0x74, 0xee, 0xc0, 0x60, 0x71, 0x6a, 0x2a, 0x74, 0x35, 0x2f, 0x6b, 0x1c, 0x43, 0x5b, 0x5d, 0x67, 0x0, 0xa8, + 0xba, 0x1e, 0x76, 0xb4, 0xd8, 0x56, 0xb, 0xee, 0x8c, 0x3, 0x9c, 0x91, 0xd8, 0x8b, 0x48, 0x60, 0xca, 0xe7, 0x9b, + 0x5, 0xd4, 0xb2, 0xd6, 0x56, 0xf0, 0xd3, 0x8d, 0x78, 0x56, 0xa2, 0xdd, 0x1a, 0x4, 0x42, 0x4c, 0x4f, 0x42} ) // NamespaceTestSuite - @@ -68,6 +126,7 @@ func (s *NamespaceTestSuite) SetupSuite() { s.rollups = mock.NewMockIRollup(s.ctrl) s.state = mock.NewMockIState(s.ctrl) s.blobReceiver = nodeMock.NewMockDalApi(s.ctrl) + s.node = nodeMock.NewMockApi(s.ctrl) s.handler = NewNamespaceHandler( s.namespaces, s.blobLogs, @@ -800,3 +859,63 @@ func (s *NamespaceTestSuite) TestBlobs() { s.Require().EqualValues(testAddress, blob.Signer) s.Require().EqualValues(testNamespace.Hash(), blob.Namespace) } + +func (s *NamespaceTestSuite) TestBlobProofs() { + commitment := "hozWHUsckMXdBhBO15TLrib4rOtpgNu/Hyu225gjFLI=" + namespaceBase64 := "AAAAAAAAAAAAAAAAAAAAAAAAAAAAVklBAAAAAAA=" + blockHeight := pkgTypes.Level(2892352) + txs := [][]byte{testTxBytes} + + proofReq := map[string]any{ + "hash": namespaceBase64, + "height": blockHeight, + "commitment": commitment, + } + stream := new(bytes.Buffer) + err := json.NewEncoder(stream).Encode(proofReq) + s.Require().NoError(err) + + req := httptest.NewRequest(http.MethodPost, "/", stream) + rec := httptest.NewRecorder() + c := s.echo.NewContext(req, rec) + c.SetPath("/blob/proofs") + + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + + s.node.EXPECT(). + Block(gomock.Any(), blockHeight). + Return(pkgTypes.ResultBlock{ + Block: &pkgTypes.Block{ + Data: pkgTypes.Data{ + Txs: tendermintTypes.ToTxs(txs), + }, + }, + }, nil) + + s.Require().NoError(s.handler.BlobProofs(c)) + s.Require().Equal(http.StatusOK, rec.Code) + + var proofs []*proof.NMTProof + err = json.NewDecoder(rec.Body).Decode(&proofs) + s.Require().NoError(err) + s.Require().EqualValues(2, len(proofs)) + s.Require().EqualValues(1, proofs[0].Start) + s.Require().EqualValues(2, proofs[0].End) + s.Require().EqualValues(1, proofs[1].End) + s.Require().EqualValues( + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABJKmGf0yBpvzj9FXhOZbyVHBq1LBKP5eI34r38GvSPZ9", + base64.StdEncoding.EncodeToString(proofs[0].Nodes[0]), + ) + s.Require().EqualValues( + "/////////////////////////////////////////////////////////////////////////////55b1xq+xbf5Olhxqwf4N8vvgExxFkjtgX4j/uCbESK9", + base64.StdEncoding.EncodeToString(proofs[0].Nodes[1]), + ) + s.Require().EqualValues( + "//////////////////////////////////////7//////////////////////////////////////plEqgR/c4IAVkNdYRWOYOAESD4whneKR54Dz5Dfe4p2", + base64.StdEncoding.EncodeToString(proofs[1].Nodes[0]), + ) + s.Require().EqualValues( + "/////////////////////////////////////////////////////////////////////////////9DUGO+QswnUItJQnpHTEz6nj13KfN9iuS9pG2tYF5tI", + base64.StdEncoding.EncodeToString(proofs[1].Nodes[1]), + ) +} From 4baff5c6e7f0759080bebf02913742534daa7775 Mon Sep 17 00:00:00 2001 From: Konst Date: Thu, 5 Dec 2024 14:27:59 +0300 Subject: [PATCH 12/12] Removed node API get BlobProofs method --- pkg/node/api.go | 1 - pkg/node/mock/api.go | 39 --------------------------------------- pkg/node/rpc/blob.go | 39 --------------------------------------- pkg/types/blobProofs.go | 31 ------------------------------- 4 files changed, 110 deletions(-) delete mode 100644 pkg/node/rpc/blob.go delete mode 100644 pkg/types/blobProofs.go diff --git a/pkg/node/api.go b/pkg/node/api.go index 00b828ad..14a5b312 100644 --- a/pkg/node/api.go +++ b/pkg/node/api.go @@ -20,7 +20,6 @@ type Api interface { Genesis(ctx context.Context) (types.Genesis, error) BlockData(ctx context.Context, level pkgTypes.Level) (pkgTypes.BlockData, error) BlockDataGet(ctx context.Context, level pkgTypes.Level) (pkgTypes.BlockData, error) - BlobProofs(ctx context.Context, level pkgTypes.Level, startShare, endShare int) (pkgTypes.BlobProof, error) } //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed diff --git a/pkg/node/mock/api.go b/pkg/node/mock/api.go index 23440607..0a6fc7b4 100644 --- a/pkg/node/mock/api.go +++ b/pkg/node/mock/api.go @@ -42,45 +42,6 @@ func (m *MockApi) EXPECT() *MockApiMockRecorder { return m.recorder } -// BlobProofs mocks base method. -func (m *MockApi) BlobProofs(ctx context.Context, level types0.Level, startShare, endShare int) (types0.BlobProof, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BlobProofs", ctx, level, startShare, endShare) - ret0, _ := ret[0].(types0.BlobProof) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// BlobProofs indicates an expected call of BlobProofs. -func (mr *MockApiMockRecorder) BlobProofs(ctx, level, startShare, endShare any) *MockApiBlobProofsCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlobProofs", reflect.TypeOf((*MockApi)(nil).BlobProofs), ctx, level, startShare, endShare) - return &MockApiBlobProofsCall{Call: call} -} - -// MockApiBlobProofsCall wrap *gomock.Call -type MockApiBlobProofsCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockApiBlobProofsCall) Return(arg0 types0.BlobProof, arg1 error) *MockApiBlobProofsCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockApiBlobProofsCall) Do(f func(context.Context, types0.Level, int, int) (types0.BlobProof, error)) *MockApiBlobProofsCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockApiBlobProofsCall) DoAndReturn(f func(context.Context, types0.Level, int, int) (types0.BlobProof, error)) *MockApiBlobProofsCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Block mocks base method. func (m *MockApi) Block(ctx context.Context, level types0.Level) (types0.ResultBlock, error) { m.ctrl.T.Helper() diff --git a/pkg/node/rpc/blob.go b/pkg/node/rpc/blob.go deleted file mode 100644 index 19dfdacb..00000000 --- a/pkg/node/rpc/blob.go +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-FileCopyrightText: 2024 PK Lab AG -// SPDX-License-Identifier: MIT - -package rpc - -import ( - "context" - "github.com/celenium-io/celestia-indexer/pkg/node/types" - pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types" - "github.com/pkg/errors" - "strconv" -) - -const pathBlobProofs = "prove_shares_v2" - -func (api *API) BlobProofs(ctx context.Context, level pkgTypes.Level, startShare, endShare int) (pkgTypes.BlobProof, error) { - if startShare < 0 || endShare < 0 { - return pkgTypes.BlobProof{}, errors.New("params 'startShare' and 'endShare' should not be lower than 0") - } - if level <= 0 { - return pkgTypes.BlobProof{}, errors.New("param 'level' should be greater than 0") - } - - args := make(map[string]string) - args["height"] = strconv.FormatInt(int64(level), 10) - args["startShare"] = strconv.FormatInt(int64(startShare), 10) - args["endShare"] = strconv.FormatInt(int64(endShare), 10) - - var proof types.Response[pkgTypes.BlobProof] - if err := api.get(ctx, pathBlobProofs, args, &proof); err != nil { - return pkgTypes.BlobProof{}, errors.Wrap(err, "api.get") - } - - if proof.Error != nil { - return pkgTypes.BlobProof{}, errors.Wrapf(types.ErrRequest, "request %d error: %s", proof.Id, proof.Error.Error()) - } - - return proof.Result, nil -} diff --git a/pkg/types/blobProofs.go b/pkg/types/blobProofs.go deleted file mode 100644 index 1a12f0a2..00000000 --- a/pkg/types/blobProofs.go +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-FileCopyrightText: 2024 PK Lab AG -// SPDX-License-Identifier: MIT - -package types - -// BlobProof - -type BlobProof struct { - ShareProof ShareProofData `json:"share_proof"` -} - -// ShareProofData - -type ShareProofData struct { - ShareProofs []ShareProof `json:"share_proofs"` -} - -// ShareProof - -type ShareProof struct { - Start int `json:"start"` - End int `json:"end"` - Nodes []string `json:"nodes"` -} - -// RowProof - -type RowProof struct { - Proofs []Proof `json:"proofs"` -} - -// Proof - -type Proof struct { - LeafHash string `json:"leaf_hash"` -}