From ecf09a53c16b3f352db77987aeb71d724fc58849 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Thu, 15 Apr 2021 18:32:39 -0500 Subject: [PATCH] enforce a minimum square size (#282) * add a minimum to square size * switch to minsharecount * switch to new rsmt2d api * update rsmt2d to the latest commit * remove the rest of the graffiti * use rsmt2d version tag --- go.mod | 2 +- go.sum | 5 ++--- p2p/ipld/nmt_wrapper_test.go | 14 +++++++------- p2p/ipld/plugin/nodes/nodes_test.go | 2 +- p2p/ipld/read.go | 10 +--------- p2p/ipld/read_test.go | 8 ++++---- types/block.go | 10 ++++++++-- types/block_test.go | 6 ++++++ types/consts.go | 4 ++++ types/shares_test.go | 6 +----- 10 files changed, 35 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 814b58f0a1..7a20c45827 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.4.0 github.com/lazyledger/lazyledger-core/p2p/ipld/plugin v0.0.0-20210219190522-0eccfb24e2aa github.com/lazyledger/nmt v0.3.1 - github.com/lazyledger/rsmt2d v0.1.1-0.20210406153014-e1fd589bdb09 + github.com/lazyledger/rsmt2d v0.2.0 github.com/libp2p/go-buffer-pool v0.0.2 github.com/minio/highwayhash v1.0.1 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect diff --git a/go.sum b/go.sum index 49fb27c9de..e1a876d17d 100644 --- a/go.sum +++ b/go.sum @@ -614,8 +614,8 @@ github.com/lazyledger/merkletree v0.0.0-20201214195110-6901c4c3c75f/go.mod h1:10 github.com/lazyledger/nmt v0.3.1 h1:zP172RR33Es4dhb88GmUr9kBpAkH6Wcl7nQGJ3HQzu4= github.com/lazyledger/nmt v0.3.1/go.mod h1:tY7ypPX26Sbkt6F8EbPl3AT33B5N0BJe4OVPbq849YI= github.com/lazyledger/rsmt2d v0.1.1-0.20210327010029-ef1d6c54461e/go.mod h1:ORR2U7THCNr1fpUhwYqZN7QCFJ20iR2uiIWfXKz3KJ4= -github.com/lazyledger/rsmt2d v0.1.1-0.20210406153014-e1fd589bdb09 h1:5wpWhlalAm1vNpkR/L1BlWHyn8GT5XNieIZqTlKG9hc= -github.com/lazyledger/rsmt2d v0.1.1-0.20210406153014-e1fd589bdb09/go.mod h1:EbB1gGbX51gBNm0hC5lMcbkgEyO3Wj2RYwba9mCUvPA= +github.com/lazyledger/rsmt2d v0.2.0 h1:RrKkAd9WTewCnOmAtvM5bz05PC+XzIYSrD+3V3odPtg= +github.com/lazyledger/rsmt2d v0.2.0/go.mod h1:EbB1gGbX51gBNm0hC5lMcbkgEyO3Wj2RYwba9mCUvPA= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -1353,7 +1353,6 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/p2p/ipld/nmt_wrapper_test.go b/p2p/ipld/nmt_wrapper_test.go index 975340ad36..c5ef9e10fc 100644 --- a/p2p/ipld/nmt_wrapper_test.go +++ b/p2p/ipld/nmt_wrapper_test.go @@ -24,7 +24,7 @@ func TestPushErasuredNamespacedMerkleTree(t *testing.T) { tree := n.Constructor() // push test data to the tree - for i, d := range generateErasuredData(t, tc.squareSize) { + for i, d := range generateErasuredData(t, tc.squareSize, rsmt2d.NewRSGF8Codec()) { // push will panic if there's an error tree.Push(d, rsmt2d.SquareIndex{Axis: uint(0), Cell: uint(i)}) } @@ -61,7 +61,7 @@ func TestErasureNamespacedMerkleTreePanics(t *testing.T) { "push over square size", assert.PanicTestFunc( func() { - data := generateErasuredData(t, 16) + data := generateErasuredData(t, 16, rsmt2d.NewRSGF8Codec()) n := NewErasuredNamespacedMerkleTree(uint64(15)) tree := n.Constructor() for i, d := range data { @@ -73,7 +73,7 @@ func TestErasureNamespacedMerkleTreePanics(t *testing.T) { "push in incorrect lexigraphic order", assert.PanicTestFunc( func() { - data := generateErasuredData(t, 16) + data := generateErasuredData(t, 16, rsmt2d.NewRSGF8Codec()) n := NewErasuredNamespacedMerkleTree(uint64(16)) tree := n.Constructor() for i := len(data) - 1; i > 0; i-- { @@ -87,7 +87,7 @@ func TestErasureNamespacedMerkleTreePanics(t *testing.T) { assert.PanicTestFunc( func() { size := 16 - data := generateErasuredData(t, size) + data := generateErasuredData(t, size, rsmt2d.NewRSGF8Codec()) n := NewErasuredNamespacedMerkleTree(uint64(size)) tree := n.Constructor() for i, d := range data { @@ -116,19 +116,19 @@ func TestExtendedDataSquare(t *testing.T) { tree := NewErasuredNamespacedMerkleTree(uint64(squareSize)) - _, err := rsmt2d.ComputeExtendedDataSquare(raw, rsmt2d.RSGF8, tree.Constructor) + _, err := rsmt2d.ComputeExtendedDataSquare(raw, rsmt2d.NewRSGF8Codec(), tree.Constructor) assert.NoError(t, err) } // generateErasuredData produces a slice that is twice as long as it erasures // the data -func generateErasuredData(t *testing.T, numLeaves int) [][]byte { +func generateErasuredData(t *testing.T, numLeaves int, codec rsmt2d.Codec) [][]byte { raw := generateRandNamespacedRawData( numLeaves, types.NamespaceSize, types.MsgShareSize, ) - erasuredData, err := rsmt2d.Encode(raw, rsmt2d.RSGF8) + erasuredData, err := codec.Encode(raw) if err != nil { t.Error(err) } diff --git a/p2p/ipld/plugin/nodes/nodes_test.go b/p2p/ipld/plugin/nodes/nodes_test.go index ea65182f0f..a4bd6473a9 100644 --- a/p2p/ipld/plugin/nodes/nodes_test.go +++ b/p2p/ipld/plugin/nodes/nodes_test.go @@ -182,7 +182,7 @@ func generateExtendedRow(t *testing.T) [][]byte { origDataWithoutNamespaces[i] = share[namespaceSize:] } - extendedData, err := rsmt2d.ComputeExtendedDataSquare(origDataWithoutNamespaces, rsmt2d.RSGF8, rsmt2d.NewDefaultTree) + extendedData, err := rsmt2d.ComputeExtendedDataSquare(origDataWithoutNamespaces, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) if err != nil { t.Fatalf("rsmt2d.Encode(): %v", err) return nil diff --git a/p2p/ipld/read.go b/p2p/ipld/read.go index 531b43c2c8..e08136499a 100644 --- a/p2p/ipld/read.go +++ b/p2p/ipld/read.go @@ -16,10 +16,6 @@ import ( "github.com/lazyledger/lazyledger-core/types" ) -// /////////////////////////////////////// -// Retrieve Block Data -// //////////////////////////////////// - const baseErrorMsg = "failure to retrieve block data:" var ErrEncounteredTooManyErrors = fmt.Errorf("%s %s", baseErrorMsg, "encountered too many errors") @@ -31,7 +27,7 @@ func RetrieveBlockData( ctx context.Context, dah *types.DataAvailabilityHeader, api coreiface.CoreAPI, - codec rsmt2d.CodecType, + codec rsmt2d.Codec, ) (types.Data, error) { edsWidth := len(dah.RowsRoots) sc := newshareCounter(ctx, uint32(edsWidth)) @@ -219,10 +215,6 @@ func (sc *shareCounter) flatten() [][]byte { return flattended } -// ////////////////////////////////////// -// Get Leaf Data -// //////////////////////////////////// - // GetLeafData fetches and returns the data for leaf leafIndex of root rootCid. // It stops and returns an error if the provided context is cancelled before // finishing diff --git a/p2p/ipld/read_test.go b/p2p/ipld/read_test.go index 427fdc15b3..b25dde65a3 100644 --- a/p2p/ipld/read_test.go +++ b/p2p/ipld/read_test.go @@ -189,7 +189,7 @@ func TestBlockRecovery(t *testing.T) { tree := NewErasuredNamespacedMerkleTree(squareSize) recoverTree := NewErasuredNamespacedMerkleTree(squareSize) - eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.RSGF8, tree.Constructor) + eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.NewRSGF8Codec(), tree.Constructor) if err != nil { t.Error(err) } @@ -205,7 +205,7 @@ func TestBlockRecovery(t *testing.T) { rowRoots, colRoots, removeRandShares(flat, tc.d), - rsmt2d.RSGF8, + rsmt2d.NewRSGF8Codec(), recoverTree.Constructor, ) @@ -273,7 +273,7 @@ func TestRetrieveBlockData(t *testing.T) { rawData := shareData.RawShares() tree := NewErasuredNamespacedMerkleTree(uint64(tc.squareSize)) - eds, err := rsmt2d.ComputeExtendedDataSquare(rawData, rsmt2d.RSGF8, tree.Constructor) + eds, err := rsmt2d.ComputeExtendedDataSquare(rawData, rsmt2d.NewRSGF8Codec(), tree.Constructor) if err != nil { t.Fatal(err) } @@ -293,7 +293,7 @@ func TestRetrieveBlockData(t *testing.T) { ColumnRoots: colRoots, }, ipfsAPI, - rsmt2d.RSGF8, + rsmt2d.NewRSGF8Codec(), ) if tc.expectErr { diff --git a/types/block.go b/types/block.go index 491500f14a..f8f590a42c 100644 --- a/types/block.go +++ b/types/block.go @@ -238,7 +238,7 @@ func (b *Block) fillDataAvailabilityHeader() { // TODO(ismail): for better efficiency and a larger number shares // we should switch to the rsmt2d.LeopardFF16 codec: - extendedDataSquare, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree) + extendedDataSquare, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) if err != nil { panic(fmt.Sprintf("unexpected error: %v", err)) } @@ -310,7 +310,7 @@ func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error } // recompute the eds - eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree) + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) if err != nil { return fmt.Errorf("failure to recompute the extended data square: %w", err) } @@ -1384,6 +1384,12 @@ func (data *Data) ComputeShares() (NamespacedShares, int) { // FIXME(ismail): this is not a power of two // see: https://github.com/lazyledger/lazyledger-specs/issues/80 and wantLen := getNextSquareNum(curLen) + + // ensure that the min square size is used + if wantLen < minSharecount { + wantLen = minSharecount + } + tailShares := GenerateTailPaddingShares(wantLen-curLen, ShareSize) return append(append(append(append( diff --git a/types/block_test.go b/types/block_test.go index 5b8f37a6d7..47ca19419b 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -241,6 +241,12 @@ func TestNilDataAvailabilityHeaderHashDoesntCrash(t *testing.T) { assert.Equal(t, emptyBytes, new(DataAvailabilityHeader).Hash()) } +func TestEmptyBlockData(t *testing.T) { + blockData := Data{} + shares, _ := blockData.ComputeShares() + assert.Equal(t, GenerateTailPaddingShares(MinSquareSize, ShareSize), shares) +} + func TestCommit(t *testing.T) { lastID := makeBlockIDRandom() h := int64(3) diff --git a/types/consts.go b/types/consts.go index 52224a5bba..ea8e0550ec 100644 --- a/types/consts.go +++ b/types/consts.go @@ -30,6 +30,10 @@ const ( // 128*128*256 = 4 Megabytes // TODO(ismail): settle on a proper max square MaxSquareSize = 128 + + // MinSquareSize depicts the smallest original square width. + MinSquareSize = 1 + minSharecount = MinSquareSize * MinSquareSize ) var ( diff --git a/types/shares_test.go b/types/shares_test.go index ac5626a8dc..7bf2dc3b31 100644 --- a/types/shares_test.go +++ b/types/shares_test.go @@ -243,7 +243,7 @@ func TestDataFromSquare(t *testing.T) { shares, _ := data.ComputeShares() rawShares := shares.RawShares() - eds, err := rsmt2d.ComputeExtendedDataSquare(rawShares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree) + eds, err := rsmt2d.ComputeExtendedDataSquare(rawShares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) if err != nil { t.Error(err) } @@ -446,10 +446,6 @@ func Test_parseDelimiter(t *testing.T) { } } -// //////////////////////////// -// Test data generation -// //////////////////////////// - // generateRandomBlockData returns randomly generated block data for testing purposes func generateRandomBlockData(t *testing.T, txCount, isrCount, evdCount, msgCount, maxSize int) Data { var out Data