From 4669019c915aa4890848681bd35ac21d806487eb Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 16 Oct 2023 15:11:15 +0200 Subject: [PATCH] chore!: move blob protos to app under it's own package (#2659) Ref: https://github.com/celestiaorg/celestia-app/issues/2570 This PR creates a `pkg/blob` where the protos for `Blob` and `BlobTx` is created. The intention is that this becomes it's own go.mod to avoid needing to depend on `celestia-core` and our `cosmos-sdk` fork. Other common structures like namespace should do the same so we can achieve some cannonical representation of the structs between node and core/app The reason for creating the types here instead of in `celestia-core` is so that only `celestia-app` becomes a multi go.mod repo as opposed to either. Additionally, `Blob` and `BlobTx` were never part of the upstream CometBFT. I also made the decision to have a single representation of blobs rather than a proto and go native. The reason being is one of simplification and performance. With large blobs, copying the data across to another type can lead to a noticeable impact. Having a single struct however does have the dowside of potential coversion errors from uint32 to uint8 so data needs to be validated before it can be used (this is also the same as before). This PR also ports over the proofs. I have put this all under a common/v1 proto package. --- app/check_tx.go | 4 +- app/process_proposal.go | 4 +- app/test/check_tx_test.go | 5 +- app/test/integration_test.go | 32 +- app/test/max_total_blob_size_test.go | 12 +- app/test/prepare_proposal_test.go | 9 +- app/test/priority_test.go | 1 - app/test/process_proposal_test.go | 27 +- app/validate_txs.go | 14 +- pkg/appconsts/global_consts.go | 7 +- pkg/blob/blob.go | 96 ++ pkg/blob/blob.pb.go | 740 ++++++++++ pkg/da/data_availability_header.go | 2 +- pkg/namespace/consts.go | 8 +- pkg/proof/proof.go | 3 +- pkg/proof/proof.pb.go | 1223 +++++++++++++++++ pkg/shares/parse.go | 5 +- pkg/shares/parse_sparse_shares.go | 13 +- pkg/shares/parse_sparse_shares_test.go | 14 +- pkg/shares/parse_test.go | 11 +- pkg/shares/share_splitting.go | 3 +- pkg/shares/sparse_shares_test.go | 4 +- pkg/shares/split_sparse_shares.go | 22 +- pkg/shares/split_sparse_shares_test.go | 25 +- pkg/square/builder.go | 19 +- pkg/square/builder_test.go | 7 +- pkg/square/square.go | 19 +- pkg/square/square_test.go | 7 +- pkg/user/signer.go | 13 +- pkg/user/signer_test.go | 2 +- proto/celestia/core/v1/blob/blob.proto | 24 + .../v1}/da/data_availability_header.pb.go | 39 +- .../v1}/da/data_availability_header.proto | 4 +- proto/celestia/core/v1/proof/proof.proto | 48 + proto/tendermint/crypto/proof.proto | 39 + test/txsim/sequence.go | 2 +- test/util/blobfactory/payforblob_factory.go | 55 +- test/util/direct_tx_gen.go | 3 +- test/util/malicious/app_test.go | 2 +- test/util/malicious/out_of_order_builder.go | 8 +- test/util/testfactory/blob.go | 26 +- test/util/testnode/node_interaction_api.go | 5 +- test/util/testnode/read.go | 3 +- x/blob/client/cli/payforblob.go | 8 +- x/blob/types/blob_tx.go | 40 +- x/blob/types/blob_tx_test.go | 61 +- x/blob/types/estimate_gas_test.go | 16 +- x/blob/types/payforblob.go | 25 +- x/blob/types/payforblob_test.go | 42 +- 49 files changed, 2441 insertions(+), 360 deletions(-) create mode 100644 pkg/blob/blob.go create mode 100644 pkg/blob/blob.pb.go create mode 100644 pkg/proof/proof.pb.go create mode 100644 proto/celestia/core/v1/blob/blob.proto rename proto/celestia/{ => core/v1}/da/data_availability_header.pb.go (85%) rename proto/celestia/{ => core/v1}/da/data_availability_header.proto (94%) create mode 100644 proto/celestia/core/v1/proof/proof.proto create mode 100644 proto/tendermint/crypto/proof.proto diff --git a/app/check_tx.go b/app/check_tx.go index 1cb54c52c5..4f5438a7ce 100644 --- a/app/check_tx.go +++ b/app/check_tx.go @@ -3,10 +3,10 @@ package app import ( "fmt" + "github.com/celestiaorg/celestia-app/pkg/blob" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" abci "github.com/tendermint/tendermint/abci/types" - coretypes "github.com/tendermint/tendermint/types" ) // CheckTx implements the ABCI interface and executes a tx in CheckTx mode. This @@ -15,7 +15,7 @@ import ( func (app *App) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx { tx := req.Tx // check if the transaction contains blobs - btx, isBlob := coretypes.UnmarshalBlobTx(tx) + btx, isBlob := blob.UnmarshalBlobTx(tx) if !isBlob { // reject transactions that can't be decoded diff --git a/app/process_proposal.go b/app/process_proposal.go index 5089e995bc..9a09731871 100644 --- a/app/process_proposal.go +++ b/app/process_proposal.go @@ -6,6 +6,7 @@ import ( "time" "github.com/celestiaorg/celestia-app/app/ante" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/pkg/square" @@ -16,7 +17,6 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/log" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - coretypes "github.com/tendermint/tendermint/types" ) const rejectedPropBlockLog = "Rejected proposal block:" @@ -52,7 +52,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp // blobTxs have no PFBs present for idx, rawTx := range req.BlockData.Txs { tx := rawTx - blobTx, isBlobTx := coretypes.UnmarshalBlobTx(rawTx) + blobTx, isBlobTx := blob.UnmarshalBlobTx(rawTx) if isBlobTx { tx = blobTx.Tx } diff --git a/app/test/check_tx_test.go b/app/test/check_tx_test.go index 1d2dc5ed3c..8842790114 100644 --- a/app/test/check_tx_test.go +++ b/app/test/check_tx_test.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/user" testutil "github.com/celestiaorg/celestia-app/test/util" @@ -82,9 +83,9 @@ func TestCheckTx(t *testing.T) { []int{100}, )[0] - dtx, _ := coretypes.UnmarshalBlobTx(btx) + dtx, _ := blob.UnmarshalBlobTx(btx) dtx.Blobs[0].NamespaceId = appns.RandomBlobNamespace().ID - bbtx, err := coretypes.MarshalBlobTx(dtx.Tx, dtx.Blobs[0]) + bbtx, err := blob.MarshalBlobTx(dtx.Tx, dtx.Blobs[0]) require.NoError(t, err) return bbtx }, diff --git a/app/test/integration_test.go b/app/test/integration_test.go index 5c00219038..4a2e054e71 100644 --- a/app/test/integration_test.go +++ b/app/test/integration_test.go @@ -23,6 +23,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/da" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/square" @@ -32,7 +33,6 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" abci "github.com/tendermint/tendermint/abci/types" tmrand "github.com/tendermint/tendermint/libs/rand" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" ) @@ -210,15 +210,13 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob() { t := s.T() ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)) - mustNewBlob := func(ns appns.Namespace, data []byte, shareVersion uint8) *blobtypes.Blob { - b, err := blobtypes.NewBlob(ns, data, shareVersion) - require.NoError(t, err) - return b + mustNewBlob := func(ns appns.Namespace, data []byte, shareVersion uint8) *blob.Blob { + return blob.New(ns, data, shareVersion) } type test struct { name string - blob *blobtypes.Blob + blob *blob.Blob opts []user.TxOption } @@ -265,7 +263,7 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob() { addr := testfactory.GetAddress(s.cctx.Keyring, s.accounts[141]) signer, err := user.SetupSigner(s.cctx.GoContext(), s.cctx.Keyring, s.cctx.GRPCClient, addr, s.ecfg) require.NoError(t, err) - res, err := signer.SubmitPayForBlob(context.TODO(), []*blobtypes.Blob{tc.blob, tc.blob}, tc.opts...) + res, err := signer.SubmitPayForBlob(context.TODO(), []*blob.Blob{tc.blob, tc.blob}, tc.opts...) require.NoError(t, err) require.NotNil(t, res) require.Equal(t, abci.CodeTypeOK, res.Code, res.Logs) @@ -391,34 +389,34 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob_blobSizes() { type testCase struct { name string - blob *tmproto.Blob + blob *blob.Blob // txResponseCode is the expected tx response ABCI code. txResponseCode uint32 } testCases := []testCase{ { name: "1,000 byte blob", - blob: mustNewBlob(t, 1_000), + blob: mustNewBlob(1_000), txResponseCode: abci.CodeTypeOK, }, { name: "10,000 byte blob", - blob: mustNewBlob(t, 10_000), + blob: mustNewBlob(10_000), txResponseCode: abci.CodeTypeOK, }, { name: "100,000 byte blob", - blob: mustNewBlob(t, 100_000), + blob: mustNewBlob(100_000), txResponseCode: abci.CodeTypeOK, }, { name: "1,000,000 byte blob", - blob: mustNewBlob(t, 1_000_000), + blob: mustNewBlob(1_000_000), txResponseCode: abci.CodeTypeOK, }, { name: "10,000,000 byte blob returns err tx too large", - blob: mustNewBlob(t, 10_000_000), + blob: mustNewBlob(10_000_000), txResponseCode: errors.ErrTxTooLarge.ABCICode(), }, } @@ -427,7 +425,7 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob_blobSizes() { s.Run(tc.name, func() { subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), 30*time.Second) defer cancel() - res, err := signer.SubmitPayForBlob(subCtx, []*blobtypes.Blob{tc.blob}, user.SetGasLimit(1_000_000_000)) + res, err := signer.SubmitPayForBlob(subCtx, []*blob.Blob{tc.blob}, user.SetGasLimit(1_000_000_000)) if tc.txResponseCode == abci.CodeTypeOK { require.NoError(t, err) } else { @@ -439,10 +437,8 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob_blobSizes() { } } -func mustNewBlob(t *testing.T, blobSize int) *tmproto.Blob { +func mustNewBlob(blobSize int) *blob.Blob { ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)) data := tmrand.Bytes(blobSize) - result, err := blobtypes.NewBlob(ns1, data, appconsts.ShareVersionZero) - require.NoError(t, err) - return result + return blob.New(ns1, data, appconsts.ShareVersionZero) } diff --git a/app/test/max_total_blob_size_test.go b/app/test/max_total_blob_size_test.go index 87764727f0..8c7875f2f9 100644 --- a/app/test/max_total_blob_size_test.go +++ b/app/test/max_total_blob_size_test.go @@ -8,6 +8,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/square" "github.com/celestiaorg/celestia-app/pkg/user" "github.com/celestiaorg/celestia-app/test/util/testfactory" @@ -17,7 +18,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" abci "github.com/tendermint/tendermint/abci/types" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" ) const ( @@ -70,24 +70,24 @@ func (s *MaxTotalBlobSizeSuite) TestSubmitPayForBlob_blobSizes() { type testCase struct { name string - blob *tmproto.Blob + blob *blob.Blob // want is the expected tx response ABCI code. want uint32 } testCases := []testCase{ { name: "1 byte blob", - blob: mustNewBlob(t, 1), + blob: mustNewBlob(1), want: abci.CodeTypeOK, }, { name: "1 mebibyte blob", - blob: mustNewBlob(t, mebibyte), + blob: mustNewBlob(mebibyte), want: abci.CodeTypeOK, }, { name: "2 mebibyte blob", - blob: mustNewBlob(t, 2*mebibyte), + blob: mustNewBlob(2 * mebibyte), want: blobtypes.ErrTotalBlobSizeTooLarge.ABCICode(), }, } @@ -97,7 +97,7 @@ func (s *MaxTotalBlobSizeSuite) TestSubmitPayForBlob_blobSizes() { for _, tc := range testCases { s.Run(tc.name, func() { - blobTx, err := signer.CreatePayForBlob([]*tmproto.Blob{tc.blob}, user.SetGasLimit(1e9)) + blobTx, err := signer.CreatePayForBlob([]*blob.Blob{tc.blob}, user.SetGasLimit(1e9)) require.NoError(t, err) subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), 30*time.Second) defer cancel() diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go index b28002913a..67b7134535 100644 --- a/app/test/prepare_proposal_test.go +++ b/app/test/prepare_proposal_test.go @@ -16,6 +16,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" testutil "github.com/celestiaorg/celestia-app/test/util" "github.com/celestiaorg/celestia-app/test/util/blobfactory" @@ -36,12 +37,8 @@ func TestPrepareProposalPutsPFBsAtEnd(t *testing.T) { testutil.ChainID, accnts[:numBlobTxs], infos[:numBlobTxs], - testfactory.Repeat([]*tmproto.Blob{ - { - NamespaceId: appns.RandomBlobNamespace().ID, - Data: []byte{1}, - ShareVersion: uint32(appconsts.DefaultShareVersion), - }, + testfactory.Repeat([]*blob.Blob{ + blob.New(appns.RandomBlobNamespace(), []byte{1}, appconsts.DefaultShareVersion), }, numBlobTxs), ) diff --git a/app/test/priority_test.go b/app/test/priority_test.go index c7cffd07ef..256c13eb0d 100644 --- a/app/test/priority_test.go +++ b/app/test/priority_test.go @@ -78,7 +78,6 @@ func (s *PriorityTestSuite) TestPriorityByGasPrice() { gasPrice := s.rand.Float64() btx, err := signer.CreatePayForBlob( blobfactory.ManyBlobs( - t, s.rand, []namespace.Namespace{namespace.RandomBlobNamespace()}, []int{100}), diff --git a/app/test/process_proposal_test.go b/app/test/process_proposal_test.go index ae805021b2..26ebeacc70 100644 --- a/app/test/process_proposal_test.go +++ b/app/test/process_proposal_test.go @@ -16,6 +16,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/da" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" @@ -112,8 +113,8 @@ func TestProcessProposal(t *testing.T) { name: "modified a blobTx", input: validData(), mutator: func(d *tmproto.Data) { - blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0]) - blobTx.Blobs[0] = &tmproto.Blob{ + blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0]) + blobTx.Blobs[0] = &blob.Blob{ NamespaceId: ns1.ID, Data: data, NamespaceVersion: uint32(ns1.Version), @@ -128,8 +129,8 @@ func TestProcessProposal(t *testing.T) { name: "invalid namespace TailPadding", input: validData(), mutator: func(d *tmproto.Data) { - blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0]) - blobTx.Blobs[0] = &tmproto.Blob{ + blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0]) + blobTx.Blobs[0] = &blob.Blob{ NamespaceId: appns.TailPaddingNamespace.ID, Data: data, NamespaceVersion: uint32(appns.TailPaddingNamespace.Version), @@ -144,8 +145,8 @@ func TestProcessProposal(t *testing.T) { name: "invalid namespace TxNamespace", input: validData(), mutator: func(d *tmproto.Data) { - blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0]) - blobTx.Blobs[0] = &tmproto.Blob{ + blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0]) + blobTx.Blobs[0] = &blob.Blob{ NamespaceId: appns.TxNamespace.ID, Data: data, NamespaceVersion: uint32(appns.TxNamespace.Version), @@ -160,8 +161,8 @@ func TestProcessProposal(t *testing.T) { name: "invalid namespace ParityShares", input: validData(), mutator: func(d *tmproto.Data) { - blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0]) - blobTx.Blobs[0] = &tmproto.Blob{ + blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0]) + blobTx.Blobs[0] = &blob.Blob{ NamespaceId: appns.ParitySharesNamespace.ID, Data: data, NamespaceVersion: uint32(appns.ParitySharesNamespace.Version), @@ -176,8 +177,8 @@ func TestProcessProposal(t *testing.T) { name: "invalid blob namespace", input: validData(), mutator: func(d *tmproto.Data) { - blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0]) - blobTx.Blobs[0] = &tmproto.Blob{ + blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0]) + blobTx.Blobs[0] = &blob.Blob{ NamespaceId: invalidNamespace.ID, Data: data, ShareVersion: uint32(appconsts.ShareVersionZero), @@ -192,7 +193,7 @@ func TestProcessProposal(t *testing.T) { name: "pfb namespace version does not match blob", input: validData(), mutator: func(d *tmproto.Data) { - blobTx, _ := coretypes.UnmarshalBlobTx(blobTxs[0]) + blobTx, _ := blob.UnmarshalBlobTx(blobTxs[0]) blobTx.Blobs[0].NamespaceVersion = appns.NamespaceVersionMax blobTxBytes, _ := blobTx.Marshal() d.Txs[0] = blobTxBytes @@ -205,8 +206,8 @@ func TestProcessProposal(t *testing.T) { input: validData(), mutator: func(d *tmproto.Data) { index := 4 - tx, blob := blobfactory.IndexWrappedTxWithInvalidNamespace(t, tmrand.NewRand(), signer, uint32(index)) - blobTx, err := coretypes.MarshalBlobTx(tx, &blob) + tx, b := blobfactory.IndexWrappedTxWithInvalidNamespace(t, tmrand.NewRand(), signer, uint32(index)) + blobTx, err := blob.MarshalBlobTx(tx, &b) require.NoError(t, err) // Replace the data with new contents diff --git a/app/validate_txs.go b/app/validate_txs.go index 9a74fb0bbb..594f85de97 100644 --- a/app/validate_txs.go +++ b/app/validate_txs.go @@ -1,21 +1,21 @@ package app import ( + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" tmbytes "github.com/tendermint/tendermint/libs/bytes" "github.com/tendermint/tendermint/libs/log" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" ) // separateTxs decodes raw tendermint txs into normal and blob txs. -func separateTxs(_ client.TxConfig, rawTxs [][]byte) ([][]byte, []tmproto.BlobTx) { +func separateTxs(_ client.TxConfig, rawTxs [][]byte) ([][]byte, []blob.BlobTx) { normalTxs := make([][]byte, 0, len(rawTxs)) - blobTxs := make([]tmproto.BlobTx, 0, len(rawTxs)) + blobTxs := make([]blob.BlobTx, 0, len(rawTxs)) for _, rawTx := range rawTxs { - bTx, isBlob := coretypes.UnmarshalBlobTx(rawTx) + bTx, isBlob := blob.UnmarshalBlobTx(rawTx) if isBlob { blobTxs = append(blobTxs, bTx) } else { @@ -69,7 +69,7 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler // filterBlobTxs applies the provided antehandler to each transaction // and removes transactions that return an error. Panics are caught by the checkTxValidity // function used to apply the ante handler. -func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []tmproto.BlobTx) ([]tmproto.BlobTx, sdk.Context) { +func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []blob.BlobTx) ([]blob.BlobTx, sdk.Context) { n := 0 for _, tx := range txs { sdkTx, err := dec(tx.Tx) @@ -105,11 +105,11 @@ func msgTypes(sdkTx sdk.Tx) []string { return msgNames } -func encodeBlobTxs(blobTxs []tmproto.BlobTx) [][]byte { +func encodeBlobTxs(blobTxs []blob.BlobTx) [][]byte { txs := make([][]byte, len(blobTxs)) var err error for i, tx := range blobTxs { - txs[i], err = coretypes.MarshalBlobTx(tx.Tx, tx.Blobs...) + txs[i], err = blob.MarshalBlobTx(tx.Tx, tx.Blobs...) if err != nil { panic(err) } diff --git a/pkg/appconsts/global_consts.go b/pkg/appconsts/global_consts.go index fd7f4f7199..19c83eb2cf 100644 --- a/pkg/appconsts/global_consts.go +++ b/pkg/appconsts/global_consts.go @@ -3,6 +3,7 @@ package appconsts import ( "math" + ns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/rsmt2d" "github.com/tendermint/tendermint/pkg/consts" ) @@ -13,16 +14,16 @@ import ( // They can not change throughout the lifetime of a network. const ( // NamespaceVersionSize is the size of a namespace version in bytes. - NamespaceVersionSize = 1 + NamespaceVersionSize = ns.NamespaceVersionSize // NamespaceVersionMaxValue is the maximum value a namespace version can be. // This const must be updated if NamespaceVersionSize is changed. NamespaceVersionMaxValue = math.MaxUint8 // NamespaceIDSize is the size of a namespace ID in bytes. - NamespaceIDSize = 28 + NamespaceIDSize = ns.NamespaceIDSize // NamespaceSize is the size of a namespace (version + ID) in bytes. - NamespaceSize = NamespaceVersionSize + NamespaceIDSize + NamespaceSize = ns.NamespaceSize // ShareSize is the size of a share in bytes. ShareSize = 512 diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go new file mode 100644 index 0000000000..40d3a16bf7 --- /dev/null +++ b/pkg/blob/blob.go @@ -0,0 +1,96 @@ +package blob + +import ( + "bytes" + "errors" + fmt "fmt" + math "math" + "sort" + + "github.com/celestiaorg/celestia-app/pkg/namespace" +) + +// ProtoBlobTxTypeID is included in each encoded BlobTx to help prevent +// decoding binaries that are not actually BlobTxs. +const ProtoBlobTxTypeID = "BLOB" + +// NewBlob creates a new coretypes.Blob from the provided data after performing +// basic stateless checks over it. +func New(ns namespace.Namespace, blob []byte, shareVersion uint8) *Blob { + return &Blob{ + NamespaceId: ns.ID, + Data: blob, + ShareVersion: uint32(shareVersion), + NamespaceVersion: uint32(ns.Version), + } +} + +// Namespace returns the namespace of the blob +func (b Blob) Namespace() namespace.Namespace { + return namespace.Namespace{ + Version: uint8(b.NamespaceVersion), + ID: b.NamespaceId, + } +} + +// Validate runs a stateless validity check on the form of the struct. +func (b *Blob) Validate() error { + if b == nil { + return errors.New("nil blob") + } + if len(b.NamespaceId) != namespace.NamespaceIDSize { + return fmt.Errorf("namespace id must be %d bytes", namespace.NamespaceIDSize) + } + if b.ShareVersion > math.MaxUint8 { + return errors.New("share version can not be greater than MaxShareVersion") + } + if b.NamespaceVersion > namespace.NamespaceVersionMax { + return errors.New("namespace version can not be greater than MaxNamespaceVersion") + } + if len(b.Data) == 0 { + return errors.New("blob data can not be empty") + } + return nil +} + +// UnmarshalBlobTx attempts to unmarshal a transaction into blob transaction. If an +// error is thrown, false is returned. +func UnmarshalBlobTx(tx []byte) (bTx BlobTx, isBlob bool) { + err := bTx.Unmarshal(tx) + if err != nil { + return BlobTx{}, false + } + // perform some quick basic checks to prevent false positives + if bTx.TypeId != ProtoBlobTxTypeID { + return bTx, false + } + if len(bTx.Blobs) == 0 { + return bTx, false + } + for _, b := range bTx.Blobs { + if len(b.NamespaceId) != namespace.NamespaceIDSize { + return bTx, false + } + } + return bTx, true +} + +// MarshalBlobTx creates a BlobTx using a normal transaction and some number of +// blobs. +// +// NOTE: Any checks on the blobs or the transaction must be performed in the +// application +func MarshalBlobTx(tx []byte, blobs ...*Blob) ([]byte, error) { + bTx := BlobTx{ + Tx: tx, + Blobs: blobs, + TypeId: ProtoBlobTxTypeID, + } + return bTx.Marshal() +} + +func Sort(blobs []*Blob) { + sort.SliceStable(blobs, func(i, j int) bool { + return bytes.Compare(blobs[i].Namespace().Bytes(), blobs[j].Namespace().Bytes()) < 0 + }) +} diff --git a/pkg/blob/blob.pb.go b/pkg/blob/blob.pb.go new file mode 100644 index 0000000000..3bd6447adb --- /dev/null +++ b/pkg/blob/blob.pb.go @@ -0,0 +1,740 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: celestia/core/v1/blob/blob.proto + +package blob + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// Blob (named after binary large object) is a chunk of data submitted by a user +// to be published to the Celestia blockchain. The data of a Blob is published +// to a namespace and is encoded into shares based on the format specified by +// share_version. +type Blob struct { + NamespaceId []byte `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + ShareVersion uint32 `protobuf:"varint,3,opt,name=share_version,json=shareVersion,proto3" json:"share_version,omitempty"` + NamespaceVersion uint32 `protobuf:"varint,4,opt,name=namespace_version,json=namespaceVersion,proto3" json:"namespace_version,omitempty"` +} + +func (m *Blob) Reset() { *m = Blob{} } +func (m *Blob) String() string { return proto.CompactTextString(m) } +func (*Blob) ProtoMessage() {} +func (*Blob) Descriptor() ([]byte, []int) { + return fileDescriptor_c6da8a12c2dbf976, []int{0} +} +func (m *Blob) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Blob) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Blob.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Blob) XXX_Merge(src proto.Message) { + xxx_messageInfo_Blob.Merge(m, src) +} +func (m *Blob) XXX_Size() int { + return m.Size() +} +func (m *Blob) XXX_DiscardUnknown() { + xxx_messageInfo_Blob.DiscardUnknown(m) +} + +var xxx_messageInfo_Blob proto.InternalMessageInfo + +func (m *Blob) GetNamespaceId() []byte { + if m != nil { + return m.NamespaceId + } + return nil +} + +func (m *Blob) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *Blob) GetShareVersion() uint32 { + if m != nil { + return m.ShareVersion + } + return 0 +} + +func (m *Blob) GetNamespaceVersion() uint32 { + if m != nil { + return m.NamespaceVersion + } + return 0 +} + +// BlobTx wraps an encoded sdk.Tx with a second field to contain blobs of data. +// The raw bytes of the blobs are not signed over, instead we verify each blob +// using the relevant MsgPayForBlobs that is signed over in the encoded sdk.Tx. +type BlobTx struct { + Tx []byte `protobuf:"bytes,1,opt,name=tx,proto3" json:"tx,omitempty"` + Blobs []*Blob `protobuf:"bytes,2,rep,name=blobs,proto3" json:"blobs,omitempty"` + TypeId string `protobuf:"bytes,3,opt,name=type_id,json=typeId,proto3" json:"type_id,omitempty"` +} + +func (m *BlobTx) Reset() { *m = BlobTx{} } +func (m *BlobTx) String() string { return proto.CompactTextString(m) } +func (*BlobTx) ProtoMessage() {} +func (*BlobTx) Descriptor() ([]byte, []int) { + return fileDescriptor_c6da8a12c2dbf976, []int{1} +} +func (m *BlobTx) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *BlobTx) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_BlobTx.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *BlobTx) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlobTx.Merge(m, src) +} +func (m *BlobTx) XXX_Size() int { + return m.Size() +} +func (m *BlobTx) XXX_DiscardUnknown() { + xxx_messageInfo_BlobTx.DiscardUnknown(m) +} + +var xxx_messageInfo_BlobTx proto.InternalMessageInfo + +func (m *BlobTx) GetTx() []byte { + if m != nil { + return m.Tx + } + return nil +} + +func (m *BlobTx) GetBlobs() []*Blob { + if m != nil { + return m.Blobs + } + return nil +} + +func (m *BlobTx) GetTypeId() string { + if m != nil { + return m.TypeId + } + return "" +} + +func init() { + proto.RegisterType((*Blob)(nil), "celestia.core.v1.blob.Blob") + proto.RegisterType((*BlobTx)(nil), "celestia.core.v1.blob.BlobTx") +} + +func init() { proto.RegisterFile("celestia/core/v1/blob/blob.proto", fileDescriptor_c6da8a12c2dbf976) } + +var fileDescriptor_c6da8a12c2dbf976 = []byte{ + // 289 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x48, 0x4e, 0xcd, 0x49, + 0x2d, 0x2e, 0xc9, 0x4c, 0xd4, 0x4f, 0xce, 0x2f, 0x4a, 0xd5, 0x2f, 0x33, 0xd4, 0x4f, 0xca, 0xc9, + 0x4f, 0x02, 0x13, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xa2, 0x30, 0x15, 0x7a, 0x20, 0x15, + 0x7a, 0x65, 0x86, 0x7a, 0x20, 0x49, 0xa5, 0x7e, 0x46, 0x2e, 0x16, 0xa7, 0x9c, 0xfc, 0x24, 0x21, + 0x45, 0x2e, 0x9e, 0xbc, 0xc4, 0xdc, 0xd4, 0xe2, 0x82, 0xc4, 0xe4, 0xd4, 0xf8, 0xcc, 0x14, 0x09, + 0x46, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x6e, 0xb8, 0x98, 0x67, 0x8a, 0x90, 0x10, 0x17, 0x4b, 0x4a, + 0x62, 0x49, 0xa2, 0x04, 0x13, 0x58, 0x0a, 0xcc, 0x16, 0x52, 0xe6, 0xe2, 0x2d, 0xce, 0x48, 0x2c, + 0x4a, 0x8d, 0x2f, 0x4b, 0x2d, 0x2a, 0xce, 0xcc, 0xcf, 0x93, 0x60, 0x56, 0x60, 0xd4, 0xe0, 0x0d, + 0xe2, 0x01, 0x0b, 0x86, 0x41, 0xc4, 0x84, 0xb4, 0xb9, 0x04, 0x11, 0x66, 0xc3, 0x14, 0xb2, 0x80, + 0x15, 0x0a, 0xc0, 0x25, 0xa0, 0x8a, 0x95, 0x52, 0xb8, 0xd8, 0x40, 0x0e, 0x0a, 0xa9, 0x10, 0xe2, + 0xe3, 0x62, 0x2a, 0xa9, 0x80, 0x3a, 0x84, 0xa9, 0xa4, 0x42, 0xc8, 0x90, 0x8b, 0x15, 0xe4, 0xe6, + 0x62, 0x09, 0x26, 0x05, 0x66, 0x0d, 0x6e, 0x23, 0x69, 0x3d, 0xac, 0x5e, 0xd2, 0x03, 0xe9, 0x0e, + 0x82, 0xa8, 0x14, 0x12, 0xe7, 0x62, 0x2f, 0xa9, 0x2c, 0x00, 0x7b, 0x08, 0xe4, 0x30, 0xce, 0x20, + 0x36, 0x10, 0xd7, 0x33, 0xc5, 0xc9, 0xed, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, + 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, 0x8f, 0xe5, 0x18, + 0xa2, 0x74, 0xd2, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, 0xf5, 0x61, 0x16, 0xe4, + 0x17, 0xa5, 0xc3, 0xd9, 0xba, 0x89, 0x05, 0x05, 0xfa, 0x05, 0xd9, 0xe9, 0xe0, 0xc0, 0x4d, 0x62, + 0x03, 0x87, 0xae, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x06, 0x11, 0x3d, 0x47, 0x81, 0x01, 0x00, + 0x00, +} + +func (m *Blob) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Blob) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Blob) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NamespaceVersion != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.NamespaceVersion)) + i-- + dAtA[i] = 0x20 + } + if m.ShareVersion != 0 { + i = encodeVarintBlob(dAtA, i, uint64(m.ShareVersion)) + i-- + dAtA[i] = 0x18 + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = encodeVarintBlob(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0x12 + } + if len(m.NamespaceId) > 0 { + i -= len(m.NamespaceId) + copy(dAtA[i:], m.NamespaceId) + i = encodeVarintBlob(dAtA, i, uint64(len(m.NamespaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *BlobTx) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlobTx) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlobTx) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.TypeId) > 0 { + i -= len(m.TypeId) + copy(dAtA[i:], m.TypeId) + i = encodeVarintBlob(dAtA, i, uint64(len(m.TypeId))) + i-- + dAtA[i] = 0x1a + } + if len(m.Blobs) > 0 { + for iNdEx := len(m.Blobs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Blobs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBlob(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.Tx) > 0 { + i -= len(m.Tx) + copy(dAtA[i:], m.Tx) + i = encodeVarintBlob(dAtA, i, uint64(len(m.Tx))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintBlob(dAtA []byte, offset int, v uint64) int { + offset -= sovBlob(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Blob) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.NamespaceId) + if l > 0 { + n += 1 + l + sovBlob(uint64(l)) + } + l = len(m.Data) + if l > 0 { + n += 1 + l + sovBlob(uint64(l)) + } + if m.ShareVersion != 0 { + n += 1 + sovBlob(uint64(m.ShareVersion)) + } + if m.NamespaceVersion != 0 { + n += 1 + sovBlob(uint64(m.NamespaceVersion)) + } + return n +} + +func (m *BlobTx) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Tx) + if l > 0 { + n += 1 + l + sovBlob(uint64(l)) + } + if len(m.Blobs) > 0 { + for _, e := range m.Blobs { + l = e.Size() + n += 1 + l + sovBlob(uint64(l)) + } + } + l = len(m.TypeId) + if l > 0 { + n += 1 + l + sovBlob(uint64(l)) + } + return n +} + +func sovBlob(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozBlob(x uint64) (n int) { + return sovBlob(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Blob) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Blob: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Blob: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NamespaceId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NamespaceId = append(m.NamespaceId[:0], dAtA[iNdEx:postIndex]...) + if m.NamespaceId == nil { + m.NamespaceId = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ShareVersion", wireType) + } + m.ShareVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ShareVersion |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NamespaceVersion", wireType) + } + m.NamespaceVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NamespaceVersion |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipBlob(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthBlob + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlobTx) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlobTx: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlobTx: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tx", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Tx = append(m.Tx[:0], dAtA[iNdEx:postIndex]...) + if m.Tx == nil { + m.Tx = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Blobs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Blobs = append(m.Blobs, &Blob{}) + if err := m.Blobs[len(m.Blobs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TypeId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlob + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthBlob + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthBlob + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TypeId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBlob(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthBlob + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipBlob(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBlob + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBlob + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBlob + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthBlob + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupBlob + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthBlob + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthBlob = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowBlob = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupBlob = fmt.Errorf("proto: unexpected end of group") +) diff --git a/pkg/da/data_availability_header.go b/pkg/da/data_availability_header.go index 6d6a26d832..bff5b3cf27 100644 --- a/pkg/da/data_availability_header.go +++ b/pkg/da/data_availability_header.go @@ -13,7 +13,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/pkg/square" "github.com/celestiaorg/celestia-app/pkg/wrapper" - daproto "github.com/celestiaorg/celestia-app/proto/celestia/da" + daproto "github.com/celestiaorg/celestia-app/proto/celestia/core/v1/da" ) var ( diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index 9c930cc572..6da6e911c2 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -3,19 +3,17 @@ package namespace import ( "bytes" "math" - - "github.com/celestiaorg/celestia-app/pkg/appconsts" ) const ( // NamespaveVersionSize is the size of a namespace version in bytes. - NamespaceVersionSize = appconsts.NamespaceVersionSize + NamespaceVersionSize = 1 // NamespaceIDSize is the size of a namespace ID in bytes. - NamespaceIDSize = appconsts.NamespaceIDSize + NamespaceIDSize = 28 // NamespaceSize is the size of a namespace (version + ID) in bytes. - NamespaceSize = appconsts.NamespaceSize + NamespaceSize = NamespaceVersionSize + NamespaceIDSize // NamespaceVersionZero is the first namespace version. NamespaceVersionZero = uint8(0) diff --git a/pkg/proof/proof.go b/pkg/proof/proof.go index fd73da227e..97ed33de52 100644 --- a/pkg/proof/proof.go +++ b/pkg/proof/proof.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/da" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" @@ -44,7 +45,7 @@ func NewTxInclusionProof(txs [][]byte, txIndex, appVersion uint64) (types.ShareP } func getTxNamespace(tx []byte) (ns appns.Namespace) { - _, isBlobTx := types.UnmarshalBlobTx(tx) + _, isBlobTx := blob.UnmarshalBlobTx(tx) if isBlobTx { return appns.PayForBlobNamespace } diff --git a/pkg/proof/proof.pb.go b/pkg/proof/proof.pb.go new file mode 100644 index 0000000000..bff3ece7da --- /dev/null +++ b/pkg/proof/proof.pb.go @@ -0,0 +1,1223 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: celestia/core/v1/proof/proof.proto + +package proof + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + crypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// ShareProof is an NMT proof that a set of shares exist in a set of rows and a +// Merkle proof that those rows exist in a Merkle tree with a given data root. +type ShareProof struct { + Data [][]byte `protobuf:"bytes,1,rep,name=data,proto3" json:"data,omitempty"` + ShareProofs []*NMTProof `protobuf:"bytes,2,rep,name=share_proofs,json=shareProofs,proto3" json:"share_proofs,omitempty"` + NamespaceId []byte `protobuf:"bytes,3,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + RowProof *RowProof `protobuf:"bytes,4,opt,name=row_proof,json=rowProof,proto3" json:"row_proof,omitempty"` + NamespaceVersion uint32 `protobuf:"varint,5,opt,name=namespace_version,json=namespaceVersion,proto3" json:"namespace_version,omitempty"` +} + +func (m *ShareProof) Reset() { *m = ShareProof{} } +func (m *ShareProof) String() string { return proto.CompactTextString(m) } +func (*ShareProof) ProtoMessage() {} +func (*ShareProof) Descriptor() ([]byte, []int) { + return fileDescriptor_e53d87d8fb5ec353, []int{0} +} +func (m *ShareProof) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShareProof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShareProof.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShareProof) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShareProof.Merge(m, src) +} +func (m *ShareProof) XXX_Size() int { + return m.Size() +} +func (m *ShareProof) XXX_DiscardUnknown() { + xxx_messageInfo_ShareProof.DiscardUnknown(m) +} + +var xxx_messageInfo_ShareProof proto.InternalMessageInfo + +func (m *ShareProof) GetData() [][]byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *ShareProof) GetShareProofs() []*NMTProof { + if m != nil { + return m.ShareProofs + } + return nil +} + +func (m *ShareProof) GetNamespaceId() []byte { + if m != nil { + return m.NamespaceId + } + return nil +} + +func (m *ShareProof) GetRowProof() *RowProof { + if m != nil { + return m.RowProof + } + return nil +} + +func (m *ShareProof) GetNamespaceVersion() uint32 { + if m != nil { + return m.NamespaceVersion + } + return 0 +} + +// RowProof is a Merkle proof that a set of rows exist in a Merkle tree with a +// given data root. +type RowProof struct { + RowRoots [][]byte `protobuf:"bytes,1,rep,name=row_roots,json=rowRoots,proto3" json:"row_roots,omitempty"` + Proofs []*crypto.Proof `protobuf:"bytes,2,rep,name=proofs,proto3" json:"proofs,omitempty"` + Root []byte `protobuf:"bytes,3,opt,name=root,proto3" json:"root,omitempty"` + StartRow uint32 `protobuf:"varint,4,opt,name=start_row,json=startRow,proto3" json:"start_row,omitempty"` + EndRow uint32 `protobuf:"varint,5,opt,name=end_row,json=endRow,proto3" json:"end_row,omitempty"` +} + +func (m *RowProof) Reset() { *m = RowProof{} } +func (m *RowProof) String() string { return proto.CompactTextString(m) } +func (*RowProof) ProtoMessage() {} +func (*RowProof) Descriptor() ([]byte, []int) { + return fileDescriptor_e53d87d8fb5ec353, []int{1} +} +func (m *RowProof) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RowProof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RowProof.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RowProof) XXX_Merge(src proto.Message) { + xxx_messageInfo_RowProof.Merge(m, src) +} +func (m *RowProof) XXX_Size() int { + return m.Size() +} +func (m *RowProof) XXX_DiscardUnknown() { + xxx_messageInfo_RowProof.DiscardUnknown(m) +} + +var xxx_messageInfo_RowProof proto.InternalMessageInfo + +func (m *RowProof) GetRowRoots() [][]byte { + if m != nil { + return m.RowRoots + } + return nil +} + +func (m *RowProof) GetProofs() []*crypto.Proof { + if m != nil { + return m.Proofs + } + return nil +} + +func (m *RowProof) GetRoot() []byte { + if m != nil { + return m.Root + } + return nil +} + +func (m *RowProof) GetStartRow() uint32 { + if m != nil { + return m.StartRow + } + return 0 +} + +func (m *RowProof) GetEndRow() uint32 { + if m != nil { + return m.EndRow + } + return 0 +} + +// NMTProof is a proof of a namespace.ID in an NMT. +// In case this proof proves the absence of a namespace.ID +// in a tree it also contains the leaf hashes of the range +// where that namespace would be. +type NMTProof struct { + // Start index of this proof. + Start int32 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` + // End index of this proof. + End int32 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` + // Nodes that together with the corresponding leaf values can be used to + // recompute the root and verify this proof. Nodes should consist of the max + // and min namespaces along with the actual hash, resulting in each being 48 + // bytes each + Nodes [][]byte `protobuf:"bytes,3,rep,name=nodes,proto3" json:"nodes,omitempty"` + // leafHash are nil if the namespace is present in the NMT. In case the + // namespace to be proved is in the min/max range of the tree but absent, this + // will contain the leaf hash necessary to verify the proof of absence. Leaf + // hashes should consist of the namespace along with the actual hash, + // resulting 40 bytes total. + LeafHash []byte `protobuf:"bytes,4,opt,name=leaf_hash,json=leafHash,proto3" json:"leaf_hash,omitempty"` +} + +func (m *NMTProof) Reset() { *m = NMTProof{} } +func (m *NMTProof) String() string { return proto.CompactTextString(m) } +func (*NMTProof) ProtoMessage() {} +func (*NMTProof) Descriptor() ([]byte, []int) { + return fileDescriptor_e53d87d8fb5ec353, []int{2} +} +func (m *NMTProof) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NMTProof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NMTProof.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NMTProof) XXX_Merge(src proto.Message) { + xxx_messageInfo_NMTProof.Merge(m, src) +} +func (m *NMTProof) XXX_Size() int { + return m.Size() +} +func (m *NMTProof) XXX_DiscardUnknown() { + xxx_messageInfo_NMTProof.DiscardUnknown(m) +} + +var xxx_messageInfo_NMTProof proto.InternalMessageInfo + +func (m *NMTProof) GetStart() int32 { + if m != nil { + return m.Start + } + return 0 +} + +func (m *NMTProof) GetEnd() int32 { + if m != nil { + return m.End + } + return 0 +} + +func (m *NMTProof) GetNodes() [][]byte { + if m != nil { + return m.Nodes + } + return nil +} + +func (m *NMTProof) GetLeafHash() []byte { + if m != nil { + return m.LeafHash + } + return nil +} + +func init() { + proto.RegisterType((*ShareProof)(nil), "celestia.core.v1.proof.ShareProof") + proto.RegisterType((*RowProof)(nil), "celestia.core.v1.proof.RowProof") + proto.RegisterType((*NMTProof)(nil), "celestia.core.v1.proof.NMTProof") +} + +func init() { + proto.RegisterFile("celestia/core/v1/proof/proof.proto", fileDescriptor_e53d87d8fb5ec353) +} + +var fileDescriptor_e53d87d8fb5ec353 = []byte{ + // 429 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0x41, 0x6e, 0xd4, 0x30, + 0x14, 0x1d, 0x37, 0x9d, 0x21, 0x78, 0x52, 0xa9, 0x58, 0x08, 0x2c, 0x10, 0x51, 0x98, 0x55, 0x24, + 0x54, 0x87, 0xc2, 0x9a, 0x0d, 0x2c, 0x80, 0x05, 0x08, 0x19, 0xc4, 0x82, 0xcd, 0xc8, 0x8d, 0xdd, + 0x49, 0x44, 0xc7, 0x3f, 0xb2, 0xcd, 0x44, 0xdc, 0x82, 0x3b, 0x70, 0x19, 0x96, 0x5d, 0xb2, 0x44, + 0x33, 0x57, 0xe0, 0x00, 0xc8, 0xf6, 0x24, 0x05, 0x09, 0x75, 0x13, 0x7d, 0xbf, 0xff, 0xfe, 0x7b, + 0x7e, 0x3f, 0xc6, 0x8b, 0x5a, 0x5d, 0x28, 0xeb, 0x5a, 0x51, 0xd5, 0x60, 0x54, 0xb5, 0x39, 0xad, + 0x3a, 0x03, 0x70, 0x1e, 0xbf, 0xac, 0x33, 0xe0, 0x80, 0xdc, 0x19, 0x38, 0xcc, 0x73, 0xd8, 0xe6, + 0x94, 0x85, 0xee, 0xbd, 0x07, 0x4e, 0x69, 0xa9, 0xcc, 0xba, 0xd5, 0xae, 0xaa, 0xcd, 0xd7, 0xce, + 0xc1, 0xdf, 0x63, 0x8b, 0xdf, 0x08, 0xe3, 0xf7, 0x8d, 0x30, 0xea, 0x9d, 0x07, 0x09, 0xc1, 0x87, + 0x52, 0x38, 0x41, 0x51, 0x91, 0x94, 0x19, 0x0f, 0x35, 0x79, 0x81, 0x33, 0xeb, 0x19, 0xcb, 0x30, + 0x67, 0xe9, 0x41, 0x91, 0x94, 0xf3, 0x27, 0x05, 0xfb, 0xbf, 0x21, 0x7b, 0xfb, 0xe6, 0x43, 0xd0, + 0xe2, 0x73, 0x3b, 0xea, 0x5a, 0xf2, 0x10, 0x67, 0x5a, 0xac, 0x95, 0xed, 0x44, 0xad, 0x96, 0xad, + 0xa4, 0x49, 0x81, 0xca, 0x8c, 0xcf, 0x47, 0xec, 0xb5, 0x24, 0xcf, 0xf0, 0x4d, 0x03, 0x7d, 0x74, + 0xa1, 0x87, 0x05, 0xba, 0xce, 0x84, 0x43, 0x1f, 0x4d, 0x52, 0xb3, 0xaf, 0xc8, 0x23, 0x7c, 0xeb, + 0xca, 0x61, 0xa3, 0x8c, 0x6d, 0x41, 0xd3, 0x69, 0x81, 0xca, 0x23, 0x7e, 0x3c, 0x36, 0x3e, 0x46, + 0x7c, 0xf1, 0x1d, 0xe1, 0x74, 0xd0, 0x20, 0xf7, 0xa3, 0xb1, 0x01, 0x70, 0x76, 0x9f, 0xdc, 0xcb, + 0x72, 0x7f, 0x26, 0x8f, 0xf1, 0xec, 0x9f, 0xdc, 0x94, 0x5d, 0x2d, 0x94, 0xc5, 0x85, 0xb2, 0x78, + 0x95, 0x3d, 0xcf, 0xef, 0xd0, 0x4b, 0xed, 0x23, 0x86, 0xda, 0x5b, 0x58, 0x27, 0x8c, 0x5b, 0x1a, + 0xe8, 0x43, 0xb6, 0x23, 0x9e, 0x06, 0x80, 0x43, 0x4f, 0xee, 0xe2, 0x1b, 0x4a, 0xcb, 0xd0, 0x8a, + 0xf7, 0x9d, 0x29, 0x2d, 0x39, 0xf4, 0x0b, 0x85, 0xd3, 0x61, 0x9b, 0xe4, 0x36, 0x9e, 0x86, 0x01, + 0x8a, 0x0a, 0x54, 0x4e, 0x79, 0x3c, 0x90, 0x63, 0x9c, 0x28, 0x2d, 0xe9, 0x41, 0xc0, 0x7c, 0xe9, + 0x79, 0x1a, 0xa4, 0xb2, 0x34, 0x09, 0x41, 0xe2, 0xc1, 0xfb, 0x5f, 0x28, 0x71, 0xbe, 0x6c, 0x84, + 0x6d, 0x82, 0x7f, 0xc6, 0x53, 0x0f, 0xbc, 0x12, 0xb6, 0x79, 0xfe, 0xf2, 0xc7, 0x36, 0x47, 0x97, + 0xdb, 0x1c, 0xfd, 0xda, 0xe6, 0xe8, 0xdb, 0x2e, 0x9f, 0x5c, 0xee, 0xf2, 0xc9, 0xcf, 0x5d, 0x3e, + 0xf9, 0x74, 0xb2, 0x6a, 0x5d, 0xf3, 0xe5, 0x8c, 0xd5, 0xb0, 0xae, 0x86, 0x3f, 0x01, 0x66, 0x35, + 0xd6, 0x27, 0xa2, 0xeb, 0xaa, 0xee, 0xf3, 0x2a, 0x3e, 0xa9, 0xb3, 0x59, 0x78, 0x53, 0x4f, 0xff, + 0x04, 0x00, 0x00, 0xff, 0xff, 0x6d, 0xf1, 0x1c, 0x57, 0xb0, 0x02, 0x00, 0x00, +} + +func (m *ShareProof) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShareProof) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShareProof) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.NamespaceVersion != 0 { + i = encodeVarintProof(dAtA, i, uint64(m.NamespaceVersion)) + i-- + dAtA[i] = 0x28 + } + if m.RowProof != nil { + { + size, err := m.RowProof.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintProof(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + if len(m.NamespaceId) > 0 { + i -= len(m.NamespaceId) + copy(dAtA[i:], m.NamespaceId) + i = encodeVarintProof(dAtA, i, uint64(len(m.NamespaceId))) + i-- + dAtA[i] = 0x1a + } + if len(m.ShareProofs) > 0 { + for iNdEx := len(m.ShareProofs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.ShareProofs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintProof(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.Data) > 0 { + for iNdEx := len(m.Data) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Data[iNdEx]) + copy(dAtA[i:], m.Data[iNdEx]) + i = encodeVarintProof(dAtA, i, uint64(len(m.Data[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *RowProof) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RowProof) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RowProof) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.EndRow != 0 { + i = encodeVarintProof(dAtA, i, uint64(m.EndRow)) + i-- + dAtA[i] = 0x28 + } + if m.StartRow != 0 { + i = encodeVarintProof(dAtA, i, uint64(m.StartRow)) + i-- + dAtA[i] = 0x20 + } + if len(m.Root) > 0 { + i -= len(m.Root) + copy(dAtA[i:], m.Root) + i = encodeVarintProof(dAtA, i, uint64(len(m.Root))) + i-- + dAtA[i] = 0x1a + } + if len(m.Proofs) > 0 { + for iNdEx := len(m.Proofs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Proofs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintProof(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.RowRoots) > 0 { + for iNdEx := len(m.RowRoots) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RowRoots[iNdEx]) + copy(dAtA[i:], m.RowRoots[iNdEx]) + i = encodeVarintProof(dAtA, i, uint64(len(m.RowRoots[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *NMTProof) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NMTProof) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NMTProof) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.LeafHash) > 0 { + i -= len(m.LeafHash) + copy(dAtA[i:], m.LeafHash) + i = encodeVarintProof(dAtA, i, uint64(len(m.LeafHash))) + i-- + dAtA[i] = 0x22 + } + if len(m.Nodes) > 0 { + for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Nodes[iNdEx]) + copy(dAtA[i:], m.Nodes[iNdEx]) + i = encodeVarintProof(dAtA, i, uint64(len(m.Nodes[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if m.End != 0 { + i = encodeVarintProof(dAtA, i, uint64(m.End)) + i-- + dAtA[i] = 0x10 + } + if m.Start != 0 { + i = encodeVarintProof(dAtA, i, uint64(m.Start)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintProof(dAtA []byte, offset int, v uint64) int { + offset -= sovProof(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *ShareProof) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Data) > 0 { + for _, b := range m.Data { + l = len(b) + n += 1 + l + sovProof(uint64(l)) + } + } + if len(m.ShareProofs) > 0 { + for _, e := range m.ShareProofs { + l = e.Size() + n += 1 + l + sovProof(uint64(l)) + } + } + l = len(m.NamespaceId) + if l > 0 { + n += 1 + l + sovProof(uint64(l)) + } + if m.RowProof != nil { + l = m.RowProof.Size() + n += 1 + l + sovProof(uint64(l)) + } + if m.NamespaceVersion != 0 { + n += 1 + sovProof(uint64(m.NamespaceVersion)) + } + return n +} + +func (m *RowProof) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.RowRoots) > 0 { + for _, b := range m.RowRoots { + l = len(b) + n += 1 + l + sovProof(uint64(l)) + } + } + if len(m.Proofs) > 0 { + for _, e := range m.Proofs { + l = e.Size() + n += 1 + l + sovProof(uint64(l)) + } + } + l = len(m.Root) + if l > 0 { + n += 1 + l + sovProof(uint64(l)) + } + if m.StartRow != 0 { + n += 1 + sovProof(uint64(m.StartRow)) + } + if m.EndRow != 0 { + n += 1 + sovProof(uint64(m.EndRow)) + } + return n +} + +func (m *NMTProof) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Start != 0 { + n += 1 + sovProof(uint64(m.Start)) + } + if m.End != 0 { + n += 1 + sovProof(uint64(m.End)) + } + if len(m.Nodes) > 0 { + for _, b := range m.Nodes { + l = len(b) + n += 1 + l + sovProof(uint64(l)) + } + } + l = len(m.LeafHash) + if l > 0 { + n += 1 + l + sovProof(uint64(l)) + } + return n +} + +func sovProof(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozProof(x uint64) (n int) { + return sovProof(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *ShareProof) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShareProof: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShareProof: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data, make([]byte, postIndex-iNdEx)) + copy(m.Data[len(m.Data)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ShareProofs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ShareProofs = append(m.ShareProofs, &NMTProof{}) + if err := m.ShareProofs[len(m.ShareProofs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NamespaceId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NamespaceId = append(m.NamespaceId[:0], dAtA[iNdEx:postIndex]...) + if m.NamespaceId == nil { + m.NamespaceId = []byte{} + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowProof", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RowProof == nil { + m.RowProof = &RowProof{} + } + if err := m.RowProof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NamespaceVersion", wireType) + } + m.NamespaceVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NamespaceVersion |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipProof(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthProof + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RowProof) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RowProof: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RowProof: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowRoots", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RowRoots = append(m.RowRoots, make([]byte, postIndex-iNdEx)) + copy(m.RowRoots[len(m.RowRoots)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Proofs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Proofs = append(m.Proofs, &crypto.Proof{}) + if err := m.Proofs[len(m.Proofs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Root", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Root = append(m.Root[:0], dAtA[iNdEx:postIndex]...) + if m.Root == nil { + m.Root = []byte{} + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StartRow", wireType) + } + m.StartRow = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StartRow |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EndRow", wireType) + } + m.EndRow = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EndRow |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipProof(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthProof + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NMTProof) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NMTProof: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NMTProof: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) + } + m.Start = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Start |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field End", wireType) + } + m.End = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.End |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Nodes = append(m.Nodes, make([]byte, postIndex-iNdEx)) + copy(m.Nodes[len(m.Nodes)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LeafHash", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProof + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthProof + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthProof + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.LeafHash = append(m.LeafHash[:0], dAtA[iNdEx:postIndex]...) + if m.LeafHash == nil { + m.LeafHash = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipProof(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthProof + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipProof(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowProof + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowProof + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowProof + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthProof + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupProof + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthProof + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthProof = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowProof = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupProof = fmt.Errorf("proto: unexpected end of group") +) diff --git a/pkg/shares/parse.go b/pkg/shares/parse.go index 6181150bb0..fb74c82413 100644 --- a/pkg/shares/parse.go +++ b/pkg/shares/parse.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" coretypes "github.com/tendermint/tendermint/types" ) @@ -26,10 +27,10 @@ func ParseTxs(shares []Share) (coretypes.Txs, error) { } // ParseBlobs collects all blobs from the shares provided -func ParseBlobs(shares []Share) ([]coretypes.Blob, error) { +func ParseBlobs(shares []Share) ([]*blob.Blob, error) { blobList, err := parseSparseShares(shares, appconsts.SupportedShareVersions) if err != nil { - return []coretypes.Blob{}, err + return []*blob.Blob{}, err } return blobList, nil diff --git a/pkg/shares/parse_sparse_shares.go b/pkg/shares/parse_sparse_shares.go index 22e899a3e5..818ad8c65e 100644 --- a/pkg/shares/parse_sparse_shares.go +++ b/pkg/shares/parse_sparse_shares.go @@ -4,18 +4,18 @@ import ( "bytes" "fmt" - coretypes "github.com/tendermint/tendermint/types" + "github.com/celestiaorg/celestia-app/pkg/blob" ) type sequence struct { - blob coretypes.Blob + blob *blob.Blob sequenceLen uint32 } // parseSparseShares iterates through rawShares and parses out individual // blobs. It returns an error if a rawShare contains a share version that // isn't present in supportedShareVersions. -func parseSparseShares(shares []Share, supportedShareVersions []uint8) (blobs []coretypes.Blob, err error) { +func parseSparseShares(shares []Share, supportedShareVersions []uint8) (blobs []*blob.Blob, err error) { if len(shares) == 0 { return nil, nil } @@ -56,12 +56,7 @@ func parseSparseShares(shares []Share, supportedShareVersions []uint8) (blobs [] if err != nil { return nil, err } - blob := coretypes.Blob{ - NamespaceID: ns.ID, - Data: data, - ShareVersion: version, - NamespaceVersion: ns.Version, - } + blob := blob.New(ns, data, version) sequences = append(sequences, sequence{ blob: blob, sequenceLen: sequenceLen, diff --git a/pkg/shares/parse_sparse_shares_test.go b/pkg/shares/parse_sparse_shares_test.go index 0c1d9cf151..f718f82762 100644 --- a/pkg/shares/parse_sparse_shares_test.go +++ b/pkg/shares/parse_sparse_shares_test.go @@ -6,11 +6,11 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/test/util/testfactory" "github.com/celestiaorg/nmt/namespace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - coretypes "github.com/tendermint/tendermint/types" ) func Test_parseSparseShares(t *testing.T) { @@ -53,12 +53,12 @@ func Test_parseSparseShares(t *testing.T) { for _, tc := range tests { // run the tests with identically sized blobs t.Run(fmt.Sprintf("%s identically sized ", tc.name), func(t *testing.T) { - blobs := make([]coretypes.Blob, tc.blobCount) + blobs := make([]*blob.Blob, tc.blobCount) for i := 0; i < tc.blobCount; i++ { blobs[i] = testfactory.GenerateRandomBlob(tc.blobSize) } - blobs = testfactory.SortBlobs(blobs) + blob.Sort(blobs) shares, err := SplitBlobs(blobs...) require.NoError(t, err) @@ -69,7 +69,7 @@ func Test_parseSparseShares(t *testing.T) { // check that the namespaces and data are the same for i := 0; i < len(blobs); i++ { - assert.Equal(t, blobs[i].NamespaceID, parsedBlobs[i].NamespaceID, "parsed blob namespace does not match") + assert.Equal(t, blobs[i].NamespaceId, parsedBlobs[i].NamespaceId, "parsed blob namespace does not match") assert.Equal(t, blobs[i].Data, parsedBlobs[i].Data, "parsed blob data does not match") } }) @@ -86,7 +86,7 @@ func Test_parseSparseShares(t *testing.T) { // check that the namespaces and data are the same for i := 0; i < len(blobs); i++ { - assert.Equal(t, blobs[i].NamespaceID, parsedBlobs[i].NamespaceID) + assert.Equal(t, blobs[i].NamespaceId, parsedBlobs[i].NamespaceId) assert.Equal(t, blobs[i].Data, parsedBlobs[i].Data) } }) @@ -130,11 +130,11 @@ func Test_parseSparseSharesWithNamespacedPadding(t *testing.T) { sss := NewSparseShareSplitter() randomSmallBlob := testfactory.GenerateRandomBlob(appconsts.ContinuationSparseShareContentSize / 2) randomLargeBlob := testfactory.GenerateRandomBlob(appconsts.ContinuationSparseShareContentSize * 4) - blobs := []coretypes.Blob{ + blobs := []*blob.Blob{ randomSmallBlob, randomLargeBlob, } - blobs = testfactory.SortBlobs(blobs) + blob.Sort(blobs) err := sss.Write(blobs[0]) require.NoError(t, err) diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 9e26f2fc75..45dd9a6092 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -244,12 +245,6 @@ func generateRandomTxs(count, size int) types.Txs { return txs } -func generateRandomBlobWithNamespace(namespace appns.Namespace, size int) types.Blob { - blob := types.Blob{ - NamespaceVersion: namespace.Version, - NamespaceID: namespace.ID, - Data: tmrand.Bytes(size), - ShareVersion: appconsts.ShareVersionZero, - } - return blob +func generateRandomBlobWithNamespace(namespace appns.Namespace, size int) *blob.Blob { + return blob.New(namespace, tmrand.Bytes(size), appconsts.ShareVersionZero) } diff --git a/pkg/shares/share_splitting.go b/pkg/shares/share_splitting.go index 01c3f9f955..70aba928cf 100644 --- a/pkg/shares/share_splitting.go +++ b/pkg/shares/share_splitting.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" coretypes "github.com/tendermint/tendermint/types" "golang.org/x/exp/maps" @@ -73,7 +74,7 @@ func SplitTxs(txs coretypes.Txs) (txShares []Share, pfbShares []Share, shareRang } // SplitBlobs splits the provided blobs into shares. -func SplitBlobs(blobs ...coretypes.Blob) ([]Share, error) { +func SplitBlobs(blobs ...*blob.Blob) ([]Share, error) { writer := NewSparseShareSplitter() for _, blob := range blobs { if err := writer.Write(blob); err != nil { diff --git a/pkg/shares/sparse_shares_test.go b/pkg/shares/sparse_shares_test.go index 30d7e35989..8da8a82129 100644 --- a/pkg/shares/sparse_shares_test.go +++ b/pkg/shares/sparse_shares_test.go @@ -4,10 +4,10 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/test/util/testfactory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - coretypes "github.com/tendermint/tendermint/types" ) func TestSparseShareContainsInfoByte(t *testing.T) { @@ -53,7 +53,7 @@ func TestSparseShareContainsInfoByte(t *testing.T) { func TestSparseShareSplitterCount(t *testing.T) { type testCase struct { name string - blob coretypes.Blob + blob *blob.Blob expected int } testCases := []testCase{ diff --git a/pkg/shares/split_sparse_shares.go b/pkg/shares/split_sparse_shares.go index f28c76f4ca..2e48eaf0ab 100644 --- a/pkg/shares/split_sparse_shares.go +++ b/pkg/shares/split_sparse_shares.go @@ -5,8 +5,7 @@ import ( "fmt" "github.com/celestiaorg/celestia-app/pkg/appconsts" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" - coretypes "github.com/tendermint/tendermint/types" + "github.com/celestiaorg/celestia-app/pkg/blob" "golang.org/x/exp/slices" ) @@ -23,19 +22,20 @@ func NewSparseShareSplitter() *SparseShareSplitter { // Write writes the provided blob to this sparse share splitter. It returns an // error or nil if no error is encountered. -func (sss *SparseShareSplitter) Write(blob coretypes.Blob) error { - if !slices.Contains(appconsts.SupportedShareVersions, blob.ShareVersion) { +func (sss *SparseShareSplitter) Write(blob *blob.Blob) error { + if err := blob.Validate(); err != nil { + return err + } + + if !slices.Contains(appconsts.SupportedShareVersions, uint8(blob.ShareVersion)) { return fmt.Errorf("unsupported share version: %d", blob.ShareVersion) } rawData := blob.Data - blobNamespace, err := appns.New(blob.NamespaceVersion, blob.NamespaceID) - if err != nil { - return err - } + blobNamespace := blob.Namespace() - // First share - b, err := NewBuilder(blobNamespace, blob.ShareVersion, true) + // First share (note by validating the blob we can safely cast the share version to uint8) + b, err := NewBuilder(blobNamespace, uint8(blob.ShareVersion), true) if err != nil { return err } @@ -57,7 +57,7 @@ func (sss *SparseShareSplitter) Write(blob coretypes.Blob) error { } sss.shares = append(sss.shares, *share) - b, err = NewBuilder(blobNamespace, blob.ShareVersion, false) + b, err = NewBuilder(blobNamespace, uint8(blob.ShareVersion), false) if err != nil { return err } diff --git a/pkg/shares/split_sparse_shares_test.go b/pkg/shares/split_sparse_shares_test.go index ca3c55853c..90509218f0 100644 --- a/pkg/shares/split_sparse_shares_test.go +++ b/pkg/shares/split_sparse_shares_test.go @@ -5,9 +5,9 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/stretchr/testify/assert" - coretypes "github.com/tendermint/tendermint/types" ) // TestSparseShareSplitter tests that the spare share splitter can split blobs @@ -16,18 +16,8 @@ func TestSparseShareSplitter(t *testing.T) { ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)) ns2 := appns.MustNewV0(bytes.Repeat([]byte{2}, appns.NamespaceVersionZeroIDSize)) - blob1 := coretypes.Blob{ - NamespaceVersion: ns1.Version, - NamespaceID: ns1.ID, - ShareVersion: 0, - Data: []byte("data1"), - } - blob2 := coretypes.Blob{ - NamespaceVersion: ns2.Version, - NamespaceID: ns2.ID, - ShareVersion: 0, - Data: []byte("data2"), - } + blob1 := blob.New(ns1, []byte("data1"), appconsts.ShareVersionZero) + blob2 := blob.New(ns2, []byte("data2"), appconsts.ShareVersionZero) sss := NewSparseShareSplitter() err := sss.Write(blob1) @@ -66,11 +56,6 @@ func TestWriteNamespacePaddingShares(t *testing.T) { assert.Equal(t, info.Version(), appconsts.ShareVersionZero) } -func newBlob(ns appns.Namespace, shareVersion uint8) coretypes.Blob { - return coretypes.Blob{ - NamespaceVersion: ns.Version, - NamespaceID: ns.ID, - ShareVersion: shareVersion, - Data: []byte("data"), - } +func newBlob(ns appns.Namespace, shareVersion uint8) *blob.Blob { + return blob.New(ns, []byte("data"), shareVersion) } diff --git a/pkg/square/builder.go b/pkg/square/builder.go index 781a7cd971..02c912bb1b 100644 --- a/pkg/square/builder.go +++ b/pkg/square/builder.go @@ -7,12 +7,11 @@ import ( "sort" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/tendermint/tendermint/pkg/consts" coretypes "github.com/tendermint/tendermint/proto/tendermint/types" - core "github.com/tendermint/tendermint/types" ) type Builder struct { @@ -54,7 +53,7 @@ func NewBuilder(maxSquareSize int, appVersion uint64, txs ...[]byte) (*Builder, } seenFirstBlobTx := false for idx, tx := range txs { - blobTx, isBlobTx := core.UnmarshalBlobTx(tx) + blobTx, isBlobTx := blob.UnmarshalBlobTx(tx) if isBlobTx { seenFirstBlobTx = true if !builder.AppendBlobTx(blobTx) { @@ -88,7 +87,7 @@ func (b *Builder) AppendTx(tx []byte) bool { // AppendBlobTx attempts to allocate the blob transaction to the square. It returns false if there is not // enough space in the square to fit the transaction. -func (b *Builder) AppendBlobTx(blobTx coretypes.BlobTx) bool { +func (b *Builder) AppendBlobTx(blobTx blob.BlobTx) bool { iw := &coretypes.IndexWrapper{ Tx: blobTx.Tx, TypeId: consts.ProtoIndexWrapperTypeID, @@ -100,11 +99,7 @@ func (b *Builder) AppendBlobTx(blobTx coretypes.BlobTx) bool { // create a new blob element for each blob and track the worst-case share count blobElements := make([]*Element, len(blobTx.Blobs)) maxBlobShareCount := 0 - for idx, blobProto := range blobTx.Blobs { - blob, err := types.BlobFromProto(blobProto) - if err != nil { - return false - } + for idx, blob := range blobTx.Blobs { blobElements[idx] = newElement(blob, len(b.Pfbs), idx, b.subtreeRootThreshold) maxBlobShareCount += blobElements[idx].maxShareOffset() } @@ -136,7 +131,7 @@ func (b *Builder) Export() (Square, error) { // of blobs within a namespace because b.Blobs are already ordered by tx // priority. sort.SliceStable(b.Blobs, func(i, j int) bool { - return bytes.Compare(b.Blobs[i].Blob.Namespace(), b.Blobs[j].Blob.Namespace()) < 0 + return bytes.Compare(b.Blobs[i].Blob.Namespace().Bytes(), b.Blobs[j].Blob.Namespace().Bytes()) < 0 }) // write all the regular transactions into compact shares @@ -368,14 +363,14 @@ func (b *Builder) IsEmpty() bool { } type Element struct { - Blob core.Blob + Blob *blob.Blob PfbIndex int BlobIndex int NumShares int MaxPadding int } -func newElement(blob core.Blob, pfbIndex, blobIndex, subtreeRootThreshold int) *Element { +func newElement(blob *blob.Blob, pfbIndex, blobIndex, subtreeRootThreshold int) *Element { numShares := shares.SparseSharesNeeded(uint32(len(blob.Data))) return &Element{ Blob: blob, diff --git a/pkg/square/builder_test.go b/pkg/square/builder_test.go index 704ba007be..b83faefa77 100644 --- a/pkg/square/builder_test.go +++ b/pkg/square/builder_test.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" ns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/pkg/square" @@ -191,7 +192,7 @@ func TestBuilderRejectsBlobTransactions(t *testing.T) { require.NoError(t, err) txs := generateBlobTxsWithNamespaces(t, ns1.Repeat(len(tc.blobSize)), [][]int{tc.blobSize}) require.Len(t, txs, 1) - blobTx, isBlobTx := coretypes.UnmarshalBlobTx(txs[0]) + blobTx, isBlobTx := blob.UnmarshalBlobTx(txs[0]) require.True(t, isBlobTx) require.Equal(t, tc.added, builder.AppendBlobTx(blobTx)) }) @@ -227,7 +228,7 @@ func TestBuilderFindTxShareRange(t *testing.T) { var lastEnd int for idx, tx := range blockTxs { - blobTx, isBlobTx := coretypes.UnmarshalBlobTx(tx) + blobTx, isBlobTx := blob.UnmarshalBlobTx(tx) if isBlobTx { tx = blobTx.Tx } @@ -420,7 +421,7 @@ func TestSquareBlobPostions(t *testing.T) { builder, err := square.NewBuilder(tt.squareSize, appconsts.LatestVersion) require.NoError(t, err) for _, tx := range tt.blobTxs { - blobTx, isBlobTx := coretypes.UnmarshalBlobTx(tx) + blobTx, isBlobTx := blob.UnmarshalBlobTx(tx) require.True(t, isBlobTx) _ = builder.AppendBlobTx(blobTx) } diff --git a/pkg/square/square.go b/pkg/square/square.go index 0f1ffe73b4..ccdba287dd 100644 --- a/pkg/square/square.go +++ b/pkg/square/square.go @@ -6,11 +6,11 @@ import ( "math" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" - blob "github.com/celestiaorg/celestia-app/x/blob/types" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/cosmos/cosmos-sdk/types" - coreproto "github.com/tendermint/tendermint/proto/tendermint/types" core "github.com/tendermint/tendermint/types" ) @@ -27,7 +27,7 @@ func Build(txs [][]byte, appVersion uint64, maxSquareSize int) (Square, [][]byte normalTxs := make([][]byte, 0, len(txs)) blobTxs := make([][]byte, 0, len(txs)) for _, tx := range txs { - blobTx, isBlobTx := core.UnmarshalBlobTx(tx) + blobTx, isBlobTx := blob.UnmarshalBlobTx(tx) if isBlobTx { if builder.AppendBlobTx(blobTx) { blobTxs = append(blobTxs, tx) @@ -122,7 +122,7 @@ func Deconstruct(s Square, decoder types.TxDecoder) (core.Txs, error) { if len(pfbMsgs) != 1 { return nil, fmt.Errorf("expected PFB to have 1 message, but got %d", len(pfbMsgs)) } - pfb, isPfb := pfbMsgs[0].(*blob.MsgPayForBlobs) + pfb, isPfb := pfbMsgs[0].(*blobtypes.MsgPayForBlobs) if !isPfb { return nil, fmt.Errorf("expected PFB message, but got %T", pfbMsgs[0]) } @@ -130,7 +130,7 @@ func Deconstruct(s Square, decoder types.TxDecoder) (core.Txs, error) { return nil, fmt.Errorf("expected PFB to have %d blob sizes, but got %d", len(wpfb.ShareIndexes), len(pfb.BlobSizes)) } - blobs := make([]*coreproto.Blob, len(wpfb.ShareIndexes)) + blobs := make([]*blob.Blob, len(wpfb.ShareIndexes)) for j, shareIndex := range wpfb.ShareIndexes { end := int(shareIndex) + shares.SparseSharesNeeded(pfb.BlobSizes[j]) parsedBlobs, err := shares.ParseBlobs(s[shareIndex:end]) @@ -141,15 +141,10 @@ func Deconstruct(s Square, decoder types.TxDecoder) (core.Txs, error) { return nil, fmt.Errorf("expected to parse a single blob, but got %d", len(blobs)) } - blobs[j] = &coreproto.Blob{ - NamespaceId: parsedBlobs[0].NamespaceID, - Data: parsedBlobs[0].Data, - ShareVersion: uint32(parsedBlobs[0].ShareVersion), - NamespaceVersion: uint32(parsedBlobs[0].NamespaceVersion), - } + blobs[j] = parsedBlobs[0] } - tx, err := core.MarshalBlobTx(wpfb.Tx, blobs...) + tx, err := blob.MarshalBlobTx(wpfb.Tx, blobs...) if err != nil { return nil, err } diff --git a/pkg/square/square_test.go b/pkg/square/square_test.go index 5548c22a72..798955dc3f 100644 --- a/pkg/square/square_test.go +++ b/pkg/square/square_test.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/inclusion" ns "github.com/celestiaorg/celestia-app/pkg/namespace" @@ -17,7 +18,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/square" "github.com/celestiaorg/celestia-app/test/util/blobfactory" "github.com/celestiaorg/celestia-app/test/util/testnode" - blob "github.com/celestiaorg/celestia-app/x/blob/types" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/rsmt2d" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -147,7 +148,7 @@ func TestSquareBlobShareRange_Flaky(t *testing.T) { require.NoError(t, err) for pfbIdx, tx := range txs { - blobTx, isBlobTx := coretypes.UnmarshalBlobTx(tx) + blobTx, isBlobTx := blob.UnmarshalBlobTx(tx) require.True(t, isBlobTx) for blobIdx := range blobTx.Blobs { shareRange, err := square.BlobShareRange(txs, pfbIdx, blobIdx, appconsts.LatestVersion) @@ -244,7 +245,7 @@ func TestSquareShareCommitments(t *testing.T) { tx, err := decoder(wpfb.Tx) require.NoError(t, err) - pfb, ok := tx.GetMsgs()[0].(*blob.MsgPayForBlobs) + pfb, ok := tx.GetMsgs()[0].(*blobtypes.MsgPayForBlobs) require.True(t, ok) for blobIndex, shareIndex := range wpfb.ShareIndexes { diff --git a/pkg/user/signer.go b/pkg/user/signer.go index 3f0f39f6e4..a6cbd7a41e 100644 --- a/pkg/user/signer.go +++ b/pkg/user/signer.go @@ -9,7 +9,8 @@ import ( "time" "github.com/celestiaorg/celestia-app/app/encoding" - blob "github.com/celestiaorg/celestia-app/x/blob/types" + "github.com/celestiaorg/celestia-app/pkg/blob" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/grpc/tmservice" "github.com/cosmos/cosmos-sdk/crypto/keyring" @@ -19,8 +20,6 @@ import ( "github.com/cosmos/cosmos-sdk/types/tx/signing" authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - tmtypes "github.com/tendermint/tendermint/types" "google.golang.org/grpc" ) @@ -143,7 +142,7 @@ func (s *Signer) SubmitTx(ctx context.Context, msgs []sdktypes.Msg, opts ...TxOp // SubmitPayForBlob forms a transaction from the provided blobs, signs it, and submits it to the chain. // TxOptions may be provided to set the fee and gas limit. -func (s *Signer) SubmitPayForBlob(ctx context.Context, blobs []*tmproto.Blob, opts ...TxOption) (*sdktypes.TxResponse, error) { +func (s *Signer) SubmitPayForBlob(ctx context.Context, blobs []*blob.Blob, opts ...TxOption) (*sdktypes.TxResponse, error) { txBytes, err := s.CreatePayForBlob(blobs, opts...) if err != nil { return nil, err @@ -175,8 +174,8 @@ func (s *Signer) CreateTx(msgs []sdktypes.Msg, opts ...TxOption) ([]byte, error) return s.enc.TxEncoder()(txBuilder.GetTx()) } -func (s *Signer) CreatePayForBlob(blobs []*tmproto.Blob, opts ...TxOption) ([]byte, error) { - msg, err := blob.NewMsgPayForBlobs(s.address.String(), blobs...) +func (s *Signer) CreatePayForBlob(blobs []*blob.Blob, opts ...TxOption) ([]byte, error) { + msg, err := blobtypes.NewMsgPayForBlobs(s.address.String(), blobs...) if err != nil { return nil, err } @@ -186,7 +185,7 @@ func (s *Signer) CreatePayForBlob(blobs []*tmproto.Blob, opts ...TxOption) ([]by return nil, err } - return tmtypes.MarshalBlobTx(txBytes, blobs...) + return blob.MarshalBlobTx(txBytes, blobs...) } // BroadcastTx submits the provided transaction bytes to the chain and returns the response. diff --git a/pkg/user/signer_test.go b/pkg/user/signer_test.go index 4b062d690c..219aa01cee 100644 --- a/pkg/user/signer_test.go +++ b/pkg/user/signer_test.go @@ -47,7 +47,7 @@ func (s *SignerTestSuite) SetupSuite() { func (s *SignerTestSuite) TestSubmitPayForBlob() { t := s.T() - blobs := blobfactory.ManyRandBlobs(t, rand.NewRand(), 1e3, 1e4) + blobs := blobfactory.ManyRandBlobs(rand.NewRand(), 1e3, 1e4) fee := user.SetFee(1e6) gas := user.SetGasLimit(1e6) subCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/proto/celestia/core/v1/blob/blob.proto b/proto/celestia/core/v1/blob/blob.proto new file mode 100644 index 0000000000..41e0620ab7 --- /dev/null +++ b/proto/celestia/core/v1/blob/blob.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; +package celestia.core.v1.blob; + +option go_package = "github.com/celestiaorg/celestia-app/pkg/blob"; + +// Blob (named after binary large object) is a chunk of data submitted by a user +// to be published to the Celestia blockchain. The data of a Blob is published +// to a namespace and is encoded into shares based on the format specified by +// share_version. +message Blob { + bytes namespace_id = 1; + bytes data = 2; + uint32 share_version = 3; + uint32 namespace_version = 4; +} + +// BlobTx wraps an encoded sdk.Tx with a second field to contain blobs of data. +// The raw bytes of the blobs are not signed over, instead we verify each blob +// using the relevant MsgPayForBlobs that is signed over in the encoded sdk.Tx. +message BlobTx { + bytes tx = 1; + repeated Blob blobs = 2; + string type_id = 3; +} \ No newline at end of file diff --git a/proto/celestia/da/data_availability_header.pb.go b/proto/celestia/core/v1/da/data_availability_header.pb.go similarity index 85% rename from proto/celestia/da/data_availability_header.pb.go rename to proto/celestia/core/v1/da/data_availability_header.pb.go index e60760c643..124043cbdf 100644 --- a/proto/celestia/da/data_availability_header.pb.go +++ b/proto/celestia/core/v1/da/data_availability_header.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: celestia/da/data_availability_header.proto +// source: celestia/core/v1/da/data_availability_header.proto package da @@ -43,7 +43,7 @@ func (m *DataAvailabilityHeader) Reset() { *m = DataAvailabilityHeader{} func (m *DataAvailabilityHeader) String() string { return proto.CompactTextString(m) } func (*DataAvailabilityHeader) ProtoMessage() {} func (*DataAvailabilityHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_93b487fd8444a5fd, []int{0} + return fileDescriptor_e4c1f9dadbff5429, []int{0} } func (m *DataAvailabilityHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -87,28 +87,29 @@ func (m *DataAvailabilityHeader) GetColumnRoots() [][]byte { } func init() { - proto.RegisterType((*DataAvailabilityHeader)(nil), "celestia.da.DataAvailabilityHeader") + proto.RegisterType((*DataAvailabilityHeader)(nil), "celestia.core.v1.da.DataAvailabilityHeader") } func init() { - proto.RegisterFile("celestia/da/data_availability_header.proto", fileDescriptor_93b487fd8444a5fd) + proto.RegisterFile("celestia/core/v1/da/data_availability_header.proto", fileDescriptor_e4c1f9dadbff5429) } -var fileDescriptor_93b487fd8444a5fd = []byte{ - // 196 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x4a, 0x4e, 0xcd, 0x49, - 0x2d, 0x2e, 0xc9, 0x4c, 0xd4, 0x4f, 0x01, 0xa1, 0x92, 0xc4, 0xf8, 0xc4, 0xb2, 0xc4, 0xcc, 0x9c, - 0xc4, 0xa4, 0xcc, 0x9c, 0xcc, 0x92, 0xca, 0xf8, 0x8c, 0xd4, 0xc4, 0x94, 0xd4, 0x22, 0xbd, 0x82, - 0xa2, 0xfc, 0x92, 0x7c, 0x21, 0x6e, 0x98, 0x5a, 0xbd, 0x94, 0x44, 0xa5, 0x08, 0x2e, 0x31, 0x97, - 0xc4, 0x92, 0x44, 0x47, 0x24, 0xd5, 0x1e, 0x60, 0xc5, 0x42, 0xd2, 0x5c, 0x9c, 0x45, 0xf9, 0xe5, - 0xf1, 0x45, 0xf9, 0xf9, 0x25, 0xc5, 0x12, 0x8c, 0x0a, 0xcc, 0x1a, 0x3c, 0x41, 0x1c, 0x45, 0xf9, - 0xe5, 0x41, 0x20, 0xbe, 0x90, 0x22, 0x17, 0x4f, 0x72, 0x7e, 0x4e, 0x69, 0x6e, 0x1e, 0x54, 0x9e, - 0x09, 0x2c, 0xcf, 0x0d, 0x11, 0x03, 0x2b, 0x71, 0xf2, 0x3f, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, - 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, - 0x63, 0x39, 0x86, 0x28, 0xd3, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0x7d, - 0x98, 0x5b, 0xf2, 0x8b, 0xd2, 0xe1, 0x6c, 0xdd, 0xc4, 0x82, 0x02, 0x7d, 0xb0, 0x5b, 0xf5, 0x91, - 0xbc, 0x95, 0xc4, 0x06, 0x16, 0x32, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x4d, 0xa3, 0x97, 0x36, - 0xec, 0x00, 0x00, 0x00, +var fileDescriptor_e4c1f9dadbff5429 = []byte{ + // 213 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x32, 0x4a, 0x4e, 0xcd, 0x49, + 0x2d, 0x2e, 0xc9, 0x4c, 0xd4, 0x4f, 0xce, 0x2f, 0x4a, 0xd5, 0x2f, 0x33, 0xd4, 0x4f, 0x49, 0xd4, + 0x4f, 0x49, 0x2c, 0x49, 0x8c, 0x4f, 0x2c, 0x4b, 0xcc, 0xcc, 0x49, 0x4c, 0xca, 0xcc, 0xc9, 0x2c, + 0xa9, 0x8c, 0xcf, 0x48, 0x4d, 0x4c, 0x49, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, + 0x86, 0xe9, 0xd1, 0x03, 0xe9, 0xd1, 0x2b, 0x33, 0xd4, 0x4b, 0x49, 0x54, 0x8a, 0xe0, 0x12, 0x73, + 0x49, 0x2c, 0x49, 0x74, 0x44, 0xd2, 0xe5, 0x01, 0xd6, 0x24, 0x24, 0xcd, 0xc5, 0x59, 0x94, 0x5f, + 0x1e, 0x5f, 0x94, 0x9f, 0x5f, 0x52, 0x2c, 0xc1, 0xa8, 0xc0, 0xac, 0xc1, 0x13, 0xc4, 0x51, 0x94, + 0x5f, 0x1e, 0x04, 0xe2, 0x0b, 0x29, 0x72, 0xf1, 0x24, 0xe7, 0xe7, 0x94, 0xe6, 0xe6, 0x41, 0xe5, + 0x99, 0xc0, 0xf2, 0xdc, 0x10, 0x31, 0xb0, 0x12, 0xa7, 0xf0, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, + 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, + 0x3c, 0x96, 0x63, 0x88, 0xb2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, + 0x87, 0xb9, 0x29, 0xbf, 0x28, 0x1d, 0xce, 0xd6, 0x4d, 0x2c, 0x28, 0xd0, 0x07, 0xbb, 0x59, 0x1f, + 0x8b, 0x37, 0x93, 0xd8, 0xc0, 0x52, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x04, 0x22, 0xee, + 0xe2, 0x04, 0x01, 0x00, 0x00, } func (m *DataAvailabilityHeader) Marshal() (dAtA []byte, err error) { diff --git a/proto/celestia/da/data_availability_header.proto b/proto/celestia/core/v1/da/data_availability_header.proto similarity index 94% rename from proto/celestia/da/data_availability_header.proto rename to proto/celestia/core/v1/da/data_availability_header.proto index 87f9346916..aef7c6bbbd 100644 --- a/proto/celestia/da/data_availability_header.proto +++ b/proto/celestia/core/v1/da/data_availability_header.proto @@ -1,7 +1,7 @@ syntax = "proto3"; -package celestia.da; +package celestia.core.v1.da; -option go_package = "github.com/celestiaorg/celestia-app/proto/celestia/da"; +option go_package = "github.com/celestiaorg/celestia-app/proto/celestia/core/v1/da"; // DataAvailabilityHeader contains the row and column roots of the erasure // coded version of the data in Block.Data. diff --git a/proto/celestia/core/v1/proof/proof.proto b/proto/celestia/core/v1/proof/proof.proto new file mode 100644 index 0000000000..f39857a640 --- /dev/null +++ b/proto/celestia/core/v1/proof/proof.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; +package celestia.core.v1.proof; + +import "tendermint/crypto/proof.proto"; + +option go_package = "github.com/celestiaorg/celestia-app/pkg/proof"; + +// ShareProof is an NMT proof that a set of shares exist in a set of rows and a +// Merkle proof that those rows exist in a Merkle tree with a given data root. +message ShareProof { + repeated bytes data = 1; + repeated NMTProof share_proofs = 2; + bytes namespace_id = 3; + RowProof row_proof = 4; + uint32 namespace_version = 5; +} + +// RowProof is a Merkle proof that a set of rows exist in a Merkle tree with a +// given data root. +message RowProof { + repeated bytes row_roots = 1; + repeated tendermint.crypto.Proof proofs = 2; + bytes root = 3; + uint32 start_row = 4; + uint32 end_row = 5; +} + +// NMTProof is a proof of a namespace.ID in an NMT. +// In case this proof proves the absence of a namespace.ID +// in a tree it also contains the leaf hashes of the range +// where that namespace would be. +message NMTProof { + // Start index of this proof. + int32 start = 1; + // End index of this proof. + int32 end = 2; + // Nodes that together with the corresponding leaf values can be used to + // recompute the root and verify this proof. Nodes should consist of the max + // and min namespaces along with the actual hash, resulting in each being 48 + // bytes each + repeated bytes nodes = 3; + // leafHash are nil if the namespace is present in the NMT. In case the + // namespace to be proved is in the min/max range of the tree but absent, this + // will contain the leaf hash necessary to verify the proof of absence. Leaf + // hashes should consist of the namespace along with the actual hash, + // resulting 40 bytes total. + bytes leaf_hash = 4; +} \ No newline at end of file diff --git a/proto/tendermint/crypto/proof.proto b/proto/tendermint/crypto/proof.proto new file mode 100644 index 0000000000..58657a7852 --- /dev/null +++ b/proto/tendermint/crypto/proof.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; +package tendermint.crypto; + +option go_package = "github.com/tendermint/tendermint/proto/tendermint/crypto"; + +import "gogoproto/gogo.proto"; + +message Proof { + int64 total = 1; + int64 index = 2; + bytes leaf_hash = 3; + repeated bytes aunts = 4; +} + +message ValueOp { + // Encoded in ProofOp.Key. + bytes key = 1; + + // To encode in ProofOp.Data + Proof proof = 2; +} + +message DominoOp { + string key = 1; + string input = 2; + string output = 3; +} + +// ProofOp defines an operation used for calculating Merkle root +// The data could be arbitrary format, providing nessecary data +// for example neighbouring node hash +message ProofOp { + string type = 1; + bytes key = 2; + bytes data = 3; +} + +// ProofOps is Merkle proof defined by the list of ProofOps +message ProofOps { repeated ProofOp ops = 1 [ (gogoproto.nullable) = false ]; } diff --git a/test/txsim/sequence.go b/test/txsim/sequence.go index fc26f383fd..d0d8088a62 100644 --- a/test/txsim/sequence.go +++ b/test/txsim/sequence.go @@ -5,7 +5,7 @@ import ( "errors" "math/rand" - blob "github.com/celestiaorg/celestia-app/x/blob/types" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/cosmos/cosmos-sdk/types" "github.com/gogo/protobuf/grpc" ) diff --git a/test/util/blobfactory/payforblob_factory.go b/test/util/blobfactory/payforblob_factory.go index 7590c6a05a..40211d87bb 100644 --- a/test/util/blobfactory/payforblob_factory.go +++ b/test/util/blobfactory/payforblob_factory.go @@ -7,6 +7,7 @@ import ( "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/user" "github.com/celestiaorg/celestia-app/test/util/testfactory" @@ -16,7 +17,6 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" "google.golang.org/grpc" ) @@ -28,8 +28,8 @@ var ( TestMaxBlobCount = 5 ) -func RandMsgPayForBlobsWithSigner(rand *tmrand.Rand, signer string, size, blobCount int) (*blobtypes.MsgPayForBlobs, []*tmproto.Blob) { - blobs := make([]*tmproto.Blob, blobCount) +func RandMsgPayForBlobsWithSigner(rand *tmrand.Rand, signer string, size, blobCount int) (*blobtypes.MsgPayForBlobs, []*blob.Blob) { + blobs := make([]*blob.Blob, blobCount) for i := 0; i < blobCount; i++ { blob, err := blobtypes.NewBlob(appns.RandomBlobNamespaceWithPRG(rand), tmrand.Bytes(size), appconsts.ShareVersionZero) if err != nil { @@ -45,19 +45,15 @@ func RandMsgPayForBlobsWithSigner(rand *tmrand.Rand, signer string, size, blobCo return msg, blobs } -func RandBlobsWithNamespace(namespaces []appns.Namespace, sizes []int) []*tmproto.Blob { - blobs := make([]*tmproto.Blob, len(namespaces)) +func RandBlobsWithNamespace(namespaces []appns.Namespace, sizes []int) []*blob.Blob { + blobs := make([]*blob.Blob, len(namespaces)) for i, ns := range namespaces { - blob, err := blobtypes.NewBlob(ns, tmrand.Bytes(sizes[i]), appconsts.ShareVersionZero) - if err != nil { - panic(err) - } - blobs[i] = blob + blobs[i] = blob.New(ns, tmrand.Bytes(sizes[i]), appconsts.ShareVersionZero) } return blobs } -func RandMsgPayForBlobsWithNamespaceAndSigner(signer string, ns appns.Namespace, size int) (*blobtypes.MsgPayForBlobs, *tmproto.Blob) { +func RandMsgPayForBlobsWithNamespaceAndSigner(signer string, ns appns.Namespace, size int) (*blobtypes.MsgPayForBlobs, *blob.Blob) { blob, err := blobtypes.NewBlob(ns, tmrand.Bytes(size), appconsts.ShareVersionZero) if err != nil { panic(err) @@ -72,11 +68,8 @@ func RandMsgPayForBlobsWithNamespaceAndSigner(signer string, ns appns.Namespace, return msg, blob } -func RandMsgPayForBlobs(rand *tmrand.Rand, size int) (*blobtypes.MsgPayForBlobs, *tmproto.Blob) { - blob, err := blobtypes.NewBlob(appns.RandomBlobNamespaceWithPRG(rand), tmrand.Bytes(size), appconsts.ShareVersionZero) - if err != nil { - panic(err) - } +func RandMsgPayForBlobs(rand *tmrand.Rand, size int) (*blobtypes.MsgPayForBlobs, *blob.Blob) { + blob := blob.New(appns.RandomBlobNamespaceWithPRG(rand), tmrand.Bytes(size), appconsts.ShareVersionZero) msg, err := blobtypes.NewMsgPayForBlobs( testfactory.TestAccAddr, blob, @@ -185,8 +178,8 @@ func RandBlobTxs(signer *user.Signer, rand *tmrand.Rand, count, blobsPerTx, size return txs } -func ManyRandBlobs(t *testing.T, rand *tmrand.Rand, sizes ...int) []*tmproto.Blob { - return ManyBlobs(t, rand, appns.RandomBlobNamespaces(rand, len(sizes)), sizes) +func ManyRandBlobs(rand *tmrand.Rand, sizes ...int) []*blob.Blob { + return ManyBlobs(rand, appns.RandomBlobNamespaces(rand, len(sizes)), sizes) } func Repeat[T any](s T, count int) []T { @@ -197,18 +190,16 @@ func Repeat[T any](s T, count int) []T { return ss } -func ManyBlobs(t *testing.T, rand *tmrand.Rand, namespaces []appns.Namespace, sizes []int) []*tmproto.Blob { - blobs := make([]*tmproto.Blob, len(namespaces)) +func ManyBlobs(rand *tmrand.Rand, namespaces []appns.Namespace, sizes []int) []*blob.Blob { + blobs := make([]*blob.Blob, len(namespaces)) for i, ns := range namespaces { - blob, err := blobtypes.NewBlob(ns, rand.Bytes(sizes[i]), appconsts.ShareVersionZero) - require.NoError(t, err) - blobs[i] = blob + blobs[i] = blob.New(ns, rand.Bytes(sizes[i]), appconsts.ShareVersionZero) } return blobs } -func NestedBlobs(t *testing.T, namespaces []appns.Namespace, sizes [][]int) [][]*tmproto.Blob { - blobs := make([][]*tmproto.Blob, len(sizes)) +func NestedBlobs(t *testing.T, namespaces []appns.Namespace, sizes [][]int) [][]*blob.Blob { + blobs := make([][]*blob.Blob, len(sizes)) counter := 0 for i, set := range sizes { for _, size := range set { @@ -228,7 +219,7 @@ func ManyMultiBlobTx( chainid string, accounts []string, accInfos []AccountInfo, - blobs [][]*tmproto.Blob, + blobs [][]*blob.Blob, ) [][]byte { t.Helper() txs := make([][]byte, len(accounts)) @@ -250,10 +241,10 @@ func IndexWrappedTxWithInvalidNamespace( rand *tmrand.Rand, signer *user.Signer, index uint32, -) (coretypes.Tx, tmproto.Blob) { +) (coretypes.Tx, blob.Blob) { t.Helper() addr := signer.Address() - blob := ManyRandBlobs(t, rand, 100)[0] + blob := ManyRandBlobs(rand, 100)[0] msg, err := blobtypes.NewMsgPayForBlobs(addr.String(), blob) require.NoError(t, err) msg.Namespaces[0] = bytes.Repeat([]byte{1}, 33) // invalid namespace @@ -276,8 +267,8 @@ func RandBlobTxsWithNamespacesAndSigner( txs := make([]coretypes.Tx, len(namespaces)) for i := 0; i < len(namespaces); i++ { // TODO: this can be refactored as the signer only needs the blobs and can construct the PFB itself - _, blob := RandMsgPayForBlobsWithNamespaceAndSigner(addr.String(), namespaces[i], sizes[i]) - cTx, err := signer.CreatePayForBlob([]*tmproto.Blob{blob}, DefaultTxOpts()...) + _, b := RandMsgPayForBlobsWithNamespaceAndSigner(addr.String(), namespaces[i], sizes[i]) + cTx, err := signer.CreatePayForBlob([]*blob.Blob{b}, DefaultTxOpts()...) if err != nil { panic(err) } @@ -295,7 +286,7 @@ func ComplexBlobTxWithOtherMsgs(t *testing.T, rand *tmrand.Rand, signer *user.Si rawTx, err := signer.CreateTx(msgs, DefaultTxOpts()...) require.NoError(t, err) - btx, err := coretypes.MarshalBlobTx(rawTx, blobs...) + btx, err := blob.MarshalBlobTx(rawTx, blobs...) require.NoError(t, err) return btx } @@ -332,7 +323,7 @@ func RandMultiBlobTxsSameSigner(t *testing.T, rand *tmrand.Rand, signer *user.Si for i := 0; i < pfbCount; i++ { blobsPerPfb := GenerateRandomBlobCount(rand) blobSizes := GenerateRandomBlobSizes(blobsPerPfb, rand) - blobs := ManyRandBlobs(t, rand, blobSizes...) + blobs := ManyRandBlobs(rand, blobSizes...) pfbTxs[i], err = signer.CreatePayForBlob(blobs) require.NoError(t, err) } diff --git a/test/util/direct_tx_gen.go b/test/util/direct_tx_gen.go index b2f80cf6f3..b00615e263 100644 --- a/test/util/direct_tx_gen.go +++ b/test/util/direct_tx_gen.go @@ -7,6 +7,7 @@ import ( tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/user" "github.com/celestiaorg/celestia-app/test/util/blobfactory" "github.com/celestiaorg/celestia-app/test/util/testfactory" @@ -141,7 +142,7 @@ func RandBlobTxsWithManualSequence( tx, err = cfg.TxEncoder()(builder.GetTx()) require.NoError(t, err) } - cTx, err := coretypes.MarshalBlobTx(tx, blobs...) + cTx, err := blob.MarshalBlobTx(tx, blobs...) if err != nil { panic(err) } diff --git a/test/util/malicious/app_test.go b/test/util/malicious/app_test.go index df4bd2bad4..269d43b07a 100644 --- a/test/util/malicious/app_test.go +++ b/test/util/malicious/app_test.go @@ -80,7 +80,7 @@ func TestMaliciousTestNode(t *testing.T) { // malicious square builder. signer, err := testnode.NewSignerFromContext(cctx, accounts[0]) require.NoError(t, err) - blobs := blobfactory.ManyRandBlobs(t, tmrand.NewRand(), 10_000, 10_000, 10_000, 10_000, 10_000, 10_000, 10_000) + blobs := blobfactory.ManyRandBlobs(tmrand.NewRand(), 10_000, 10_000, 10_000, 10_000, 10_000, 10_000, 10_000) txres, err := signer.SubmitPayForBlob(cctx.GoContext(), blobs, blobfactory.DefaultTxOpts()...) require.NoError(t, err) require.Equal(t, abci.CodeTypeOK, txres.Code) diff --git a/test/util/malicious/out_of_order_builder.go b/test/util/malicious/out_of_order_builder.go index 7fc1c42244..0fb22988f0 100644 --- a/test/util/malicious/out_of_order_builder.go +++ b/test/util/malicious/out_of_order_builder.go @@ -6,10 +6,10 @@ import ( "sort" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/pkg/square" - core "github.com/tendermint/tendermint/types" ) type ExportFn func(builder *square.Builder) (square.Square, error) @@ -27,7 +27,7 @@ func Build(txs [][]byte, appVersion uint64, maxSquareSize int, efn ExportFn) (sq normalTxs := make([][]byte, 0, len(txs)) blobTxs := make([][]byte, 0, len(txs)) for _, tx := range txs { - blobTx, isBlobTx := core.UnmarshalBlobTx(tx) + blobTx, isBlobTx := blob.UnmarshalBlobTx(tx) if isBlobTx { if builder.AppendBlobTx(blobTx) { blobTxs = append(blobTxs, tx) @@ -72,14 +72,14 @@ func OutOfOrderExport(b *square.Builder) (square.Square, error) { // sort the blobs in order of namespace. We use slice stable here to respect the // order of multiple blobs within a namespace as per the priority of the PFB sort.SliceStable(b.Blobs, func(i, j int) bool { - return bytes.Compare(b.Blobs[i].Blob.Namespace(), b.Blobs[j].Blob.Namespace()) < 0 + return bytes.Compare(b.Blobs[i].Blob.Namespace().Bytes(), b.Blobs[j].Blob.Namespace().Bytes()) < 0 }) if len(b.Blobs) > 1 { // iterate through each blob and find the first two that have different // namespaces and swap them. for i := 0; i < len(b.Blobs)-1; i++ { - if !bytes.Equal(b.Blobs[i].Blob.Namespace(), b.Blobs[i+1].Blob.Namespace()) { + if !bytes.Equal(b.Blobs[i].Blob.Namespace().Bytes(), b.Blobs[i+1].Blob.Namespace().Bytes()) { b.Blobs[i], b.Blobs[i+1] = b.Blobs[i+1], b.Blobs[i] break } diff --git a/test/util/testfactory/blob.go b/test/util/testfactory/blob.go index a4addccb83..63396c40f4 100644 --- a/test/util/testfactory/blob.go +++ b/test/util/testfactory/blob.go @@ -3,16 +3,16 @@ package testfactory import ( "bytes" "encoding/binary" - "sort" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/types" ) -func GenerateRandomlySizedBlobs(count, maxBlobSize int) []types.Blob { - blobs := make([]types.Blob, count) +func GenerateRandomlySizedBlobs(count, maxBlobSize int) []*blob.Blob { + blobs := make([]*blob.Blob, count) for i := 0; i < count; i++ { blobs[i] = GenerateRandomBlob(tmrand.Intn(maxBlobSize)) if len(blobs[i].Data) == 0 { @@ -25,7 +25,7 @@ func GenerateRandomlySizedBlobs(count, maxBlobSize int) []types.Blob { blobs = nil } - blobs = SortBlobs(blobs) + blob.Sort(blobs) return blobs } @@ -49,19 +49,14 @@ func GenerateBlobsWithNamespace(count int, blobSize int, ns appns.Namespace) []t return blobs } -func GenerateRandomBlob(dataSize int) types.Blob { - blob := types.Blob{ - NamespaceVersion: appns.NamespaceVersionZero, - NamespaceID: append(appns.NamespaceVersionZeroPrefix, bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)...), - Data: tmrand.Bytes(dataSize), - ShareVersion: appconsts.ShareVersionZero, - } - return blob +func GenerateRandomBlob(dataSize int) *blob.Blob { + ns := appns.MustNewV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) + return blob.New(ns, tmrand.Bytes(dataSize), appconsts.ShareVersionZero) } // GenerateRandomBlobOfShareCount returns a blob that spans the given // number of shares -func GenerateRandomBlobOfShareCount(count int) types.Blob { +func GenerateRandomBlobOfShareCount(count int) *blob.Blob { size := rawBlobSize(appconsts.FirstSparseShareContentSize * count) return GenerateRandomBlob(size) } @@ -78,8 +73,3 @@ func DelimLen(size uint64) int { lenBuf := make([]byte, binary.MaxVarintLen64) return binary.PutUvarint(lenBuf, size) } - -func SortBlobs(blobs []types.Blob) []types.Blob { - sort.SliceStable(blobs, func(i, j int) bool { return bytes.Compare(blobs[i].Namespace(), blobs[j].Namespace()) < 0 }) - return blobs -} diff --git a/test/util/testnode/node_interaction_api.go b/test/util/testnode/node_interaction_api.go index 99a86078b4..cf8e2e3bbb 100644 --- a/test/util/testnode/node_interaction_api.go +++ b/test/util/testnode/node_interaction_api.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/pkg/user" @@ -236,7 +237,7 @@ func (c *Context) PostData(account, broadcastMode string, ns appns.Namespace, bl return nil, err } - blob, err := types.NewBlob(ns, blobData, appconsts.ShareVersionZero) + b, err := types.NewBlob(ns, blobData, appconsts.ShareVersionZero) if err != nil { return nil, err } @@ -244,7 +245,7 @@ func (c *Context) PostData(account, broadcastMode string, ns appns.Namespace, bl gas := types.DefaultEstimateGas([]uint32{uint32(len(blobData))}) opts := blobfactory.FeeTxOpts(gas) - blobTx, err := signer.CreatePayForBlob([]*types.Blob{blob}, opts...) + blobTx, err := signer.CreatePayForBlob([]*blob.Blob{b}, opts...) if err != nil { return nil, err } diff --git a/test/util/testnode/read.go b/test/util/testnode/read.go index 4fffd965d4..af95a90904 100644 --- a/test/util/testnode/read.go +++ b/test/util/testnode/read.go @@ -6,6 +6,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" + "github.com/celestiaorg/celestia-app/pkg/blob" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/tendermint/tendermint/rpc/client/http" "github.com/tendermint/tendermint/types" @@ -59,7 +60,7 @@ func DecodeBlockData(data types.Data) ([]sdk.Msg, error) { decoder := encCfg.TxConfig.TxDecoder() msgs := make([]sdk.Msg, 0) for _, txBytes := range data.Txs { - blobTx, isBlobTx := types.UnmarshalBlobTx(txBytes) + blobTx, isBlobTx := blob.UnmarshalBlobTx(txBytes) if isBlobTx { txBytes = blobTx.Tx } diff --git a/x/blob/client/cli/payforblob.go b/x/blob/client/cli/payforblob.go index 15ce7031de..52ebff4cf6 100644 --- a/x/blob/client/cli/payforblob.go +++ b/x/blob/client/cli/payforblob.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/cosmos/cosmos-sdk/client" @@ -17,7 +18,6 @@ import ( "github.com/cosmos/cosmos-sdk/client/input" sdktx "github.com/cosmos/cosmos-sdk/client/tx" sdk "github.com/cosmos/cosmos-sdk/types" - coretypes "github.com/tendermint/tendermint/types" ) const ( @@ -108,7 +108,7 @@ func getNamespace(namespaceID []byte, namespaceVersion uint8) (appns.Namespace, // broadcastPFB creates the new PFB message type that will later be broadcast to tendermint nodes // this private func is used in CmdPayForBlob -func broadcastPFB(cmd *cobra.Command, blob *types.Blob) error { +func broadcastPFB(cmd *cobra.Command, b *blob.Blob) error { clientCtx, err := client.GetClientTxContext(cmd) if err != nil { return err @@ -116,7 +116,7 @@ func broadcastPFB(cmd *cobra.Command, blob *types.Blob) error { // TODO: allow the user to override the share version via a new flag // See https://github.com/celestiaorg/celestia-app/issues/1041 - pfbMsg, err := types.NewMsgPayForBlobs(clientCtx.FromAddress.String(), blob) + pfbMsg, err := types.NewMsgPayForBlobs(clientCtx.FromAddress.String(), b) if err != nil { return err } @@ -131,7 +131,7 @@ func broadcastPFB(cmd *cobra.Command, blob *types.Blob) error { return err } - blobTx, err := coretypes.MarshalBlobTx(txBytes, blob) + blobTx, err := blob.MarshalBlobTx(txBytes, b) if err != nil { return err } diff --git a/x/blob/types/blob_tx.go b/x/blob/types/blob_tx.go index 125c282315..1ebf9b15fc 100644 --- a/x/blob/types/blob_tx.go +++ b/x/blob/types/blob_tx.go @@ -2,36 +2,29 @@ package types import ( "bytes" - "fmt" - math "math" - - "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" shares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/cosmos/cosmos-sdk/client" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - core "github.com/tendermint/tendermint/types" ) -// Blob wraps the tendermint type so that users can simply import this one. -type Blob = tmproto.Blob - // NewBlob creates a new coretypes.Blob from the provided data after performing // basic stateless checks over it. -func NewBlob(ns appns.Namespace, blob []byte, shareVersion uint8) (*Blob, error) { +func NewBlob(ns appns.Namespace, data []byte, shareVersion uint8) (*blob.Blob, error) { err := ValidateBlobNamespace(ns) if err != nil { return nil, err } - if len(blob) == 0 { + if len(data) == 0 { return nil, ErrZeroBlobSize } - return &tmproto.Blob{ + return &blob.Blob{ NamespaceId: ns.ID, - Data: blob, + Data: data, ShareVersion: uint32(shareVersion), NamespaceVersion: uint32(ns.Version), }, nil @@ -39,7 +32,7 @@ func NewBlob(ns appns.Namespace, blob []byte, shareVersion uint8) (*Blob, error) // ValidateBlobTx performs stateless checks on the BlobTx to ensure that the // blobs attached to the transaction are valid. -func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx tmproto.BlobTx) error { +func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx blob.BlobTx) error { sdkTx, err := txcfg.TxDecoder()(bTx.Tx) if err != nil { return err @@ -116,27 +109,6 @@ func BlobTxSharesUsed(btx tmproto.BlobTx) int { return sharesUsed } -func BlobFromProto(p *tmproto.Blob) (core.Blob, error) { - if p == nil { - return core.Blob{}, fmt.Errorf("nil blob") - } - - if p.ShareVersion > math.MaxUint8 { - return core.Blob{}, fmt.Errorf("invalid share version %d", p.ShareVersion) - } - - if p.NamespaceVersion > appconsts.NamespaceVersionMaxValue { - return core.Blob{}, fmt.Errorf("invalid namespace version %d", p.NamespaceVersion) - } - - return core.Blob{ - NamespaceID: p.NamespaceId, - Data: p.Data, - ShareVersion: uint8(p.ShareVersion), - NamespaceVersion: uint8(p.NamespaceVersion), - }, nil -} - func equalSlices[T comparable](a, b []T) bool { if len(a) != len(b) { return false diff --git a/x/blob/types/blob_tx_test.go b/x/blob/types/blob_tx_test.go index e2118b55de..de43cbf53f 100644 --- a/x/blob/types/blob_tx_test.go +++ b/x/blob/types/blob_tx_test.go @@ -7,6 +7,7 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/test/util/blobfactory" "github.com/celestiaorg/celestia-app/test/util/testnode" @@ -16,8 +17,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - coretypes "github.com/tendermint/tendermint/types" ) func TestNewBlob(t *testing.T) { @@ -42,7 +41,7 @@ func TestValidateBlobTx(t *testing.T) { type test struct { name string - getTx func() tmproto.BlobTx + getTx func() blob.BlobTx expectedErr error } @@ -58,18 +57,18 @@ func TestValidateBlobTx(t *testing.T) { tests := []test{ { name: "normal transaction", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { rawBtx := validRawBtx() - btx, _ := coretypes.UnmarshalBlobTx(rawBtx) + btx, _ := blob.UnmarshalBlobTx(rawBtx) return btx }, expectedErr: nil, }, { name: "invalid transaction, mismatched namespace", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { rawBtx := validRawBtx() - btx, _ := coretypes.UnmarshalBlobTx(rawBtx) + btx, _ := blob.UnmarshalBlobTx(rawBtx) btx.Blobs[0].NamespaceId = namespace.RandomBlobNamespace().ID return btx }, @@ -77,22 +76,22 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "invalid transaction, no pfb", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { sendTx := blobfactory.GenerateManyRawSendTxs(signer, 1) - blob, err := types.NewBlob(namespace.RandomBlobNamespace(), tmrand.Bytes(100), appconsts.ShareVersionZero) + b, err := types.NewBlob(namespace.RandomBlobNamespace(), tmrand.Bytes(100), appconsts.ShareVersionZero) require.NoError(t, err) - return tmproto.BlobTx{ + return blob.BlobTx{ Tx: sendTx[0], - Blobs: []*tmproto.Blob{blob}, + Blobs: []*blob.Blob{b}, } }, expectedErr: types.ErrNoPFB, }, { name: "mismatched number of pfbs and blobs", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { rawBtx := validRawBtx() - btx, _ := coretypes.UnmarshalBlobTx(rawBtx) + btx, _ := blob.UnmarshalBlobTx(rawBtx) blob, err := types.NewBlob(namespace.RandomBlobNamespace(), tmrand.Bytes(100), appconsts.ShareVersionZero) require.NoError(t, err) btx.Blobs = append(btx.Blobs, blob) @@ -102,17 +101,17 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "invalid share commitment", - getTx: func() tmproto.BlobTx { - blob, err := types.NewBlob(namespace.RandomBlobNamespace(), tmrand.Bytes(100), appconsts.ShareVersionZero) + getTx: func() blob.BlobTx { + b, err := types.NewBlob(namespace.RandomBlobNamespace(), tmrand.Bytes(100), appconsts.ShareVersionZero) require.NoError(t, err) msg, err := types.NewMsgPayForBlobs( addr.String(), - blob, + b, ) require.NoError(t, err) badCommit, err := types.CreateCommitment( - &types.Blob{ + &blob.Blob{ NamespaceVersion: uint32(namespace.RandomBlobNamespace().Version), NamespaceId: namespace.RandomBlobNamespace().ID, Data: tmrand.Bytes(99), @@ -125,9 +124,9 @@ func TestValidateBlobTx(t *testing.T) { rawTx, err := signer.CreateTx([]sdk.Msg{msg}) require.NoError(t, err) - btx := tmproto.BlobTx{ + btx := blob.BlobTx{ Tx: rawTx, - Blobs: []*tmproto.Blob{blob}, + Blobs: []*blob.Blob{b}, } return btx }, @@ -135,7 +134,7 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "complex transaction with one send and one pfb", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { signerAddr := signer.Address() sendMsg := banktypes.NewMsgSend(signerAddr, signerAddr, sdk.NewCoins(sdk.NewCoin(app.BondDenom, sdk.NewInt(10)))) @@ -145,7 +144,7 @@ func TestValidateBlobTx(t *testing.T) { signer, sendMsg, ) - btx, isBlob := coretypes.UnmarshalBlobTx(tx) + btx, isBlob := blob.UnmarshalBlobTx(tx) require.True(t, isBlob) return btx }, @@ -153,9 +152,9 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "only send tx", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { sendtx := blobfactory.GenerateManyRawSendTxs(signer, 1)[0] - return tmproto.BlobTx{ + return blob.BlobTx{ Tx: sendtx, } }, @@ -163,14 +162,14 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "normal transaction with two blobs w/ different namespaces", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { rawBtx, err := signer.CreatePayForBlob( blobfactory.RandBlobsWithNamespace( []namespace.Namespace{namespace.RandomBlobNamespace(), namespace.RandomBlobNamespace()}, []int{100, 100}), ) require.NoError(t, err) - btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + btx, isBlobTx := blob.UnmarshalBlobTx(rawBtx) require.True(t, isBlobTx) return btx }, @@ -178,14 +177,14 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "normal transaction with two large blobs w/ different namespaces", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { rawBtx, err := signer.CreatePayForBlob( blobfactory.RandBlobsWithNamespace( []namespace.Namespace{namespace.RandomBlobNamespace(), namespace.RandomBlobNamespace()}, []int{100000, 1000000}), ) require.NoError(t, err) - btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + btx, isBlobTx := blob.UnmarshalBlobTx(rawBtx) require.True(t, isBlobTx) return btx }, @@ -193,7 +192,7 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "normal transaction with two blobs w/ same namespace", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { ns := namespace.RandomBlobNamespace() rawBtx, err := signer.CreatePayForBlob( blobfactory.RandBlobsWithNamespace( @@ -201,7 +200,7 @@ func TestValidateBlobTx(t *testing.T) { []int{100, 100}), ) require.NoError(t, err) - btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + btx, isBlobTx := blob.UnmarshalBlobTx(rawBtx) require.True(t, isBlobTx) return btx }, @@ -209,7 +208,7 @@ func TestValidateBlobTx(t *testing.T) { }, { name: "normal transaction with one hundred blobs of the same namespace", - getTx: func() tmproto.BlobTx { + getTx: func() blob.BlobTx { count := 100 ns := namespace.RandomBlobNamespace() sizes := make([]int, count) @@ -224,7 +223,7 @@ func TestValidateBlobTx(t *testing.T) { sizes, )) require.NoError(t, err) - btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + btx, isBlobTx := blob.UnmarshalBlobTx(rawBtx) require.True(t, isBlobTx) return btx }, diff --git a/x/blob/types/estimate_gas_test.go b/x/blob/types/estimate_gas_test.go index fea244756d..b0ed06b034 100644 --- a/x/blob/types/estimate_gas_test.go +++ b/x/blob/types/estimate_gas_test.go @@ -7,17 +7,17 @@ import ( "cosmossdk.io/math" "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" + "github.com/celestiaorg/celestia-app/pkg/blob" "github.com/celestiaorg/celestia-app/pkg/user" testutil "github.com/celestiaorg/celestia-app/test/util" "github.com/celestiaorg/celestia-app/test/util/blobfactory" "github.com/celestiaorg/celestia-app/test/util/testfactory" "github.com/stretchr/testify/require" - blob "github.com/celestiaorg/celestia-app/x/blob/types" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" sdk "github.com/cosmos/cosmos-sdk/types" abci "github.com/tendermint/tendermint/abci/types" tmrand "github.com/tendermint/tendermint/libs/rand" - "github.com/tendermint/tendermint/types" ) func TestPFBGasEstimation(t *testing.T) { @@ -42,12 +42,12 @@ func TestPFBGasEstimation(t *testing.T) { addr := testfactory.GetAddress(kr, accnts[0]) signer, err := user.NewSigner(kr, nil, addr, encCfg.TxConfig, testutil.ChainID, 1, 0) require.NoError(t, err) - blobs := blobfactory.ManyRandBlobs(t, rand, tc.blobSizes...) - gas := blob.DefaultEstimateGas(toUint32(tc.blobSizes)) + blobs := blobfactory.ManyRandBlobs(rand, tc.blobSizes...) + gas := blobtypes.DefaultEstimateGas(toUint32(tc.blobSizes)) fee := sdk.NewCoins(sdk.NewCoin(app.BondDenom, math.NewInt(int64(gas)))) tx, err := signer.CreatePayForBlob(blobs, user.SetGasLimit(gas), user.SetFeeAmount(fee)) require.NoError(t, err) - blobTx, ok := types.UnmarshalBlobTx(tx) + blobTx, ok := blob.UnmarshalBlobTx(tx) require.True(t, ok) resp := testApp.DeliverTx(abci.RequestDeliverTx{ Tx: blobTx.Tx, @@ -87,12 +87,12 @@ func FuzzPFBGasEstimation(f *testing.F) { addr := testfactory.GetAddress(kr, accnts[0]) signer, err := user.NewSigner(kr, nil, addr, encCfg.TxConfig, testutil.ChainID, 1, 0) require.NoError(t, err) - blobs := blobfactory.ManyRandBlobs(t, rand, blobSizes...) - gas := blob.DefaultEstimateGas(toUint32(blobSizes)) + blobs := blobfactory.ManyRandBlobs(rand, blobSizes...) + gas := blobtypes.DefaultEstimateGas(toUint32(blobSizes)) fee := sdk.NewCoins(sdk.NewCoin(app.BondDenom, math.NewInt(int64(gas)))) tx, err := signer.CreatePayForBlob(blobs, user.SetGasLimit(gas), user.SetFeeAmount(fee)) require.NoError(t, err) - blobTx, ok := types.UnmarshalBlobTx(tx) + blobTx, ok := blob.UnmarshalBlobTx(tx) require.True(t, ok) resp := testApp.DeliverTx(abci.RequestDeliverTx{ Tx: blobTx.Tx, diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 07d2f78699..44727da70b 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -7,6 +7,7 @@ import ( "cosmossdk.io/errors" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" appshares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/nmt" @@ -14,8 +15,6 @@ import ( "github.com/cosmos/cosmos-sdk/x/auth/migrations/legacytx" auth "github.com/cosmos/cosmos-sdk/x/auth/types" "github.com/tendermint/tendermint/crypto/merkle" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - coretypes "github.com/tendermint/tendermint/types" "golang.org/x/exp/slices" ) @@ -47,7 +46,7 @@ const ( // See: https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/building-modules/messages-and-queries.md#legacy-amino-legacymsgs var _ legacytx.LegacyMsg = &MsgPayForBlobs{} -func NewMsgPayForBlobs(signer string, blobs ...*Blob) (*MsgPayForBlobs, error) { +func NewMsgPayForBlobs(signer string, blobs ...*blob.Blob) (*MsgPayForBlobs, error) { err := ValidateBlobs(blobs...) if err != nil { return nil, err @@ -215,19 +214,13 @@ func (msg *MsgPayForBlobs) GetSigners() []sdk.AccAddress { // // [data square layout rationale]: ../../specs/src/specs/data_square_layout.md // [blob share commitment rules]: ../../specs/src/specs/data_square_layout.md#blob-share-commitment-rules -func CreateCommitment(blob *Blob) ([]byte, error) { - coreblob := coretypes.Blob{ - NamespaceID: blob.NamespaceId, - Data: blob.Data, - ShareVersion: uint8(blob.ShareVersion), - NamespaceVersion: uint8(blob.NamespaceVersion), - } - namespace, err := appns.New(uint8(blob.NamespaceVersion), blob.NamespaceId) - if err != nil { +func CreateCommitment(blob *blob.Blob) ([]byte, error) { + if err := blob.Validate(); err != nil { return nil, err } + namespace := blob.Namespace() - shares, err := appshares.SplitBlobs(coreblob) + shares, err := appshares.SplitBlobs(blob) if err != nil { return nil, err } @@ -279,7 +272,7 @@ func CreateCommitment(blob *Blob) ([]byte, error) { return merkle.HashFromByteSlices(subTreeRoots), nil } -func CreateCommitments(blobs []*Blob) ([][]byte, error) { +func CreateCommitments(blobs []*blob.Blob) ([][]byte, error) { commitments := make([][]byte, len(blobs)) for i, blob := range blobs { commitment, err := CreateCommitment(blob) @@ -292,7 +285,7 @@ func CreateCommitments(blobs []*Blob) ([][]byte, error) { } // ValidateBlobs performs basic checks over the components of one or more PFBs. -func ValidateBlobs(blobs ...*Blob) error { +func ValidateBlobs(blobs ...*blob.Blob) error { if len(blobs) == 0 { return ErrNoBlobs } @@ -324,7 +317,7 @@ func ValidateBlobs(blobs ...*Blob) error { // ExtractBlobComponents separates and returns the components of a slice of // blobs. -func ExtractBlobComponents(pblobs []*tmproto.Blob) (namespaceVersions []uint32, namespaceIds [][]byte, sizes []uint32, shareVersions []uint32) { +func ExtractBlobComponents(pblobs []*blob.Blob) (namespaceVersions []uint32, namespaceIds [][]byte, sizes []uint32, shareVersions []uint32) { namespaceVersions = make([]uint32, len(pblobs)) namespaceIds = make([][]byte, len(pblobs)) sizes = make([]uint32, len(pblobs)) diff --git a/x/blob/types/payforblob_test.go b/x/blob/types/payforblob_test.go index 4d648ad2d2..a367ead573 100644 --- a/x/blob/types/payforblob_test.go +++ b/x/blob/types/payforblob_test.go @@ -6,6 +6,7 @@ import ( sdkerrors "cosmossdk.io/errors" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" appns "github.com/celestiaorg/celestia-app/pkg/namespace" shares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/test/util/testfactory" @@ -15,7 +16,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" ) func Test_MerkleMountainRangeHeights(t *testing.T) { @@ -96,7 +96,7 @@ func TestCreateCommitment(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - blob := &types.Blob{ + blob := &blob.Blob{ NamespaceId: tt.namespace.ID, Data: tt.blob, ShareVersion: uint32(tt.shareVersion), @@ -263,10 +263,10 @@ func validMsgPayForBlobs(t *testing.T) *types.MsgPayForBlobs { signer, err := testnode.NewOfflineSigner() require.NoError(t, err) ns1 := append(appns.NamespaceVersionZeroPrefix, bytes.Repeat([]byte{0x01}, appns.NamespaceVersionZeroIDSize)...) - blob := bytes.Repeat([]byte{2}, totalBlobSize(appconsts.ContinuationSparseShareContentSize*12)) + data := bytes.Repeat([]byte{2}, totalBlobSize(appconsts.ContinuationSparseShareContentSize*12)) - pblob := &tmproto.Blob{ - Data: blob, + pblob := &blob.Blob{ + Data: data, NamespaceId: ns1, NamespaceVersion: uint32(appns.NamespaceVersionZero), ShareVersion: uint32(appconsts.ShareVersionZero), @@ -283,16 +283,16 @@ func invalidNamespaceVersionMsgPayForBlobs(t *testing.T) *types.MsgPayForBlobs { signer, err := testnode.NewOfflineSigner() require.NoError(t, err) ns1 := append(appns.NamespaceVersionZeroPrefix, bytes.Repeat([]byte{0x01}, appns.NamespaceVersionZeroIDSize)...) - blob := bytes.Repeat([]byte{2}, totalBlobSize(appconsts.ContinuationSparseShareContentSize*12)) + data := bytes.Repeat([]byte{2}, totalBlobSize(appconsts.ContinuationSparseShareContentSize*12)) - pblob := &tmproto.Blob{ - Data: blob, + pblob := &blob.Blob{ + Data: data, NamespaceId: ns1, NamespaceVersion: uint32(255), ShareVersion: uint32(appconsts.ShareVersionZero), } - blobs := []*types.Blob{pblob} + blobs := []*blob.Blob{pblob} commitments, err := types.CreateCommitments(blobs) require.NoError(t, err) @@ -326,7 +326,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { type testCase struct { name string signer string - blobs []*tmproto.Blob + blobs []*blob.Blob expectedErr bool } ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)) @@ -336,7 +336,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { { name: "valid msg PFB with small blob", signer: testfactory.TestAccAddr, - blobs: []*tmproto.Blob{ + blobs: []*blob.Blob{ { NamespaceVersion: uint32(ns1.Version), NamespaceId: ns1.ID, @@ -349,7 +349,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { { name: "valid msg PFB with large blob", signer: testfactory.TestAccAddr, - blobs: []*tmproto.Blob{ + blobs: []*blob.Blob{ { NamespaceVersion: uint32(ns1.Version), NamespaceId: ns1.ID, @@ -362,7 +362,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { { name: "valid msg PFB with two blobs", signer: testfactory.TestAccAddr, - blobs: []*tmproto.Blob{ + blobs: []*blob.Blob{ { NamespaceVersion: uint32(ns1.Version), NamespaceId: ns1.ID, @@ -380,7 +380,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { { name: "unsupported share version returns an error", signer: testfactory.TestAccAddr, - blobs: []*tmproto.Blob{ + blobs: []*blob.Blob{ { NamespaceVersion: uint32(ns1.Version), NamespaceId: ns1.ID, @@ -393,7 +393,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { { name: "msg PFB with tx namespace returns an error", signer: testfactory.TestAccAddr, - blobs: []*tmproto.Blob{ + blobs: []*blob.Blob{ { NamespaceVersion: uint32(appns.TxNamespace.Version), NamespaceId: appns.TxNamespace.ID, @@ -406,7 +406,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { { name: "msg PFB with invalid signer returns an error", signer: testfactory.TestAccAddr[:10], - blobs: []*tmproto.Blob{ + blobs: []*blob.Blob{ { NamespaceVersion: uint32(ns1.Version), NamespaceId: ns1.ID, @@ -443,14 +443,14 @@ func TestNewMsgPayForBlobs(t *testing.T) { func TestValidateBlobs(t *testing.T) { type test struct { name string - blob *types.Blob + blob *blob.Blob expectError bool } tests := []test{ { name: "valid blob", - blob: &types.Blob{ + blob: &blob.Blob{ Data: []byte{1}, NamespaceId: appns.RandomBlobNamespace().ID, ShareVersion: uint32(appconsts.DefaultShareVersion), @@ -460,7 +460,7 @@ func TestValidateBlobs(t *testing.T) { }, { name: "invalid share version", - blob: &types.Blob{ + blob: &blob.Blob{ Data: []byte{1}, NamespaceId: appns.RandomBlobNamespace().ID, ShareVersion: uint32(10000), @@ -470,7 +470,7 @@ func TestValidateBlobs(t *testing.T) { }, { name: "empty blob", - blob: &types.Blob{ + blob: &blob.Blob{ Data: []byte{}, NamespaceId: appns.RandomBlobNamespace().ID, ShareVersion: uint32(appconsts.DefaultShareVersion), @@ -480,7 +480,7 @@ func TestValidateBlobs(t *testing.T) { }, { name: "invalid namespace", - blob: &types.Blob{ + blob: &blob.Blob{ Data: []byte{1}, NamespaceId: appns.TxNamespace.ID, ShareVersion: uint32(appconsts.DefaultShareVersion),