enforce a minimum square size (#282)

* add a minimum to square size

* switch to minsharecount

* switch to new rsmt2d api

* update rsmt2d to the latest commit

* remove the rest of the graffiti

* use rsmt2d version tag

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/ipfs/interface-go-ipfs-core v0.4.0
 	github.com/lazyledger/lazyledger-core/p2p/ipld/plugin v0.0.0-20210219190522-0eccfb24e2aa
 	github.com/lazyledger/nmt v0.3.1
-	github.com/lazyledger/rsmt2d v0.1.1-0.20210406153014-e1fd589bdb09
+	github.com/lazyledger/rsmt2d v0.2.0
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/minio/highwayhash v1.0.1
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect

go.sum

@@ -614,8 +614,8 @@ github.com/lazyledger/merkletree v0.0.0-20201214195110-6901c4c3c75f/go.mod h1:10
 github.com/lazyledger/nmt v0.3.1 h1:zP172RR33Es4dhb88GmUr9kBpAkH6Wcl7nQGJ3HQzu4=
 github.com/lazyledger/nmt v0.3.1/go.mod h1:tY7ypPX26Sbkt6F8EbPl3AT33B5N0BJe4OVPbq849YI=
 github.com/lazyledger/rsmt2d v0.1.1-0.20210327010029-ef1d6c54461e/go.mod h1:ORR2U7THCNr1fpUhwYqZN7QCFJ20iR2uiIWfXKz3KJ4=
-github.com/lazyledger/rsmt2d v0.1.1-0.20210406153014-e1fd589bdb09 h1:5wpWhlalAm1vNpkR/L1BlWHyn8GT5XNieIZqTlKG9hc=
-github.com/lazyledger/rsmt2d v0.1.1-0.20210406153014-e1fd589bdb09/go.mod h1:EbB1gGbX51gBNm0hC5lMcbkgEyO3Wj2RYwba9mCUvPA=
+github.com/lazyledger/rsmt2d v0.2.0 h1:RrKkAd9WTewCnOmAtvM5bz05PC+XzIYSrD+3V3odPtg=
+github.com/lazyledger/rsmt2d v0.2.0/go.mod h1:EbB1gGbX51gBNm0hC5lMcbkgEyO3Wj2RYwba9mCUvPA=
 github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
 github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU=
 github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E=
@@ -1353,7 +1353,6 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
 golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
 golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

p2p/ipld/nmt_wrapper_test.go

@@ -24,7 +24,7 @@ func TestPushErasuredNamespacedMerkleTree(t *testing.T) {
 		tree := n.Constructor()
 
 		// push test data to the tree
-		for i, d := range generateErasuredData(t, tc.squareSize) {
+		for i, d := range generateErasuredData(t, tc.squareSize, rsmt2d.NewRSGF8Codec()) {
 			// push will panic if there's an error
 			tree.Push(d, rsmt2d.SquareIndex{Axis: uint(0), Cell: uint(i)})
 		}
@@ -61,7 +61,7 @@ func TestErasureNamespacedMerkleTreePanics(t *testing.T) {
 			"push over square size",
 			assert.PanicTestFunc(
 				func() {
-					data := generateErasuredData(t, 16)
+					data := generateErasuredData(t, 16, rsmt2d.NewRSGF8Codec())
 					n := NewErasuredNamespacedMerkleTree(uint64(15))
 					tree := n.Constructor()
 					for i, d := range data {
@@ -73,7 +73,7 @@ func TestErasureNamespacedMerkleTreePanics(t *testing.T) {
 			"push in incorrect lexigraphic order",
 			assert.PanicTestFunc(
 				func() {
-					data := generateErasuredData(t, 16)
+					data := generateErasuredData(t, 16, rsmt2d.NewRSGF8Codec())
 					n := NewErasuredNamespacedMerkleTree(uint64(16))
 					tree := n.Constructor()
 					for i := len(data) - 1; i > 0; i-- {
@@ -87,7 +87,7 @@ func TestErasureNamespacedMerkleTreePanics(t *testing.T) {
 			assert.PanicTestFunc(
 				func() {
 					size := 16
-					data := generateErasuredData(t, size)
+					data := generateErasuredData(t, size, rsmt2d.NewRSGF8Codec())
 					n := NewErasuredNamespacedMerkleTree(uint64(size))
 					tree := n.Constructor()
 					for i, d := range data {
@@ -116,19 +116,19 @@ func TestExtendedDataSquare(t *testing.T) {
 
 	tree := NewErasuredNamespacedMerkleTree(uint64(squareSize))
 
-	_, err := rsmt2d.ComputeExtendedDataSquare(raw, rsmt2d.RSGF8, tree.Constructor)
+	_, err := rsmt2d.ComputeExtendedDataSquare(raw, rsmt2d.NewRSGF8Codec(), tree.Constructor)
 	assert.NoError(t, err)
 }
 
 // generateErasuredData produces a slice that is twice as long as it erasures
 // the data
-func generateErasuredData(t *testing.T, numLeaves int) [][]byte {
+func generateErasuredData(t *testing.T, numLeaves int, codec rsmt2d.Codec) [][]byte {
 	raw := generateRandNamespacedRawData(
 		numLeaves,
 		types.NamespaceSize,
 		types.MsgShareSize,
 	)
-	erasuredData, err := rsmt2d.Encode(raw, rsmt2d.RSGF8)
+	erasuredData, err := codec.Encode(raw)
 	if err != nil {
 		t.Error(err)
 	}

p2p/ipld/plugin/nodes/nodes_test.go

@@ -182,7 +182,7 @@ func generateExtendedRow(t *testing.T) [][]byte {
 		origDataWithoutNamespaces[i] = share[namespaceSize:]
 	}
 
-	extendedData, err := rsmt2d.ComputeExtendedDataSquare(origDataWithoutNamespaces, rsmt2d.RSGF8, rsmt2d.NewDefaultTree)
+	extendedData, err := rsmt2d.ComputeExtendedDataSquare(origDataWithoutNamespaces, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
 	if err != nil {
 		t.Fatalf("rsmt2d.Encode(): %v", err)
 		return nil

p2p/ipld/read.go

@@ -16,10 +16,6 @@ import (
 	"github.com/lazyledger/lazyledger-core/types"
 )
 
-// ///////////////////////////////////////
-//	Retrieve Block Data
-// ////////////////////////////////////
-
 const baseErrorMsg = "failure to retrieve block data:"
 
 var ErrEncounteredTooManyErrors = fmt.Errorf("%s %s", baseErrorMsg, "encountered too many errors")
@@ -31,7 +27,7 @@ func RetrieveBlockData(
 	ctx context.Context,
 	dah *types.DataAvailabilityHeader,
 	api coreiface.CoreAPI,
-	codec rsmt2d.CodecType,
+	codec rsmt2d.Codec,
 ) (types.Data, error) {
 	edsWidth := len(dah.RowsRoots)
 	sc := newshareCounter(ctx, uint32(edsWidth))
@@ -219,10 +215,6 @@ func (sc *shareCounter) flatten() [][]byte {
 	return flattended
 }
 
-// //////////////////////////////////////
-//	Get Leaf Data
-// ////////////////////////////////////
-
 // GetLeafData fetches and returns the data for leaf leafIndex of root rootCid.
 // It stops and returns an error if the provided context is cancelled before
 // finishing

p2p/ipld/read_test.go

@@ -189,7 +189,7 @@ func TestBlockRecovery(t *testing.T) {
 			tree := NewErasuredNamespacedMerkleTree(squareSize)
 			recoverTree := NewErasuredNamespacedMerkleTree(squareSize)
 
-			eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.RSGF8, tree.Constructor)
+			eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.NewRSGF8Codec(), tree.Constructor)
 			if err != nil {
 				t.Error(err)
 			}
@@ -205,7 +205,7 @@ func TestBlockRecovery(t *testing.T) {
 				rowRoots,
 				colRoots,
 				removeRandShares(flat, tc.d),
-				rsmt2d.RSGF8,
+				rsmt2d.NewRSGF8Codec(),
 				recoverTree.Constructor,
 			)
 
@@ -273,7 +273,7 @@ func TestRetrieveBlockData(t *testing.T) {
 			rawData := shareData.RawShares()
 
 			tree := NewErasuredNamespacedMerkleTree(uint64(tc.squareSize))
-			eds, err := rsmt2d.ComputeExtendedDataSquare(rawData, rsmt2d.RSGF8, tree.Constructor)
+			eds, err := rsmt2d.ComputeExtendedDataSquare(rawData, rsmt2d.NewRSGF8Codec(), tree.Constructor)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -293,7 +293,7 @@ func TestRetrieveBlockData(t *testing.T) {
 					ColumnRoots: colRoots,
 				},
 				ipfsAPI,
-				rsmt2d.RSGF8,
+				rsmt2d.NewRSGF8Codec(),
 			)
 
 			if tc.expectErr {

types/block.go

@@ -238,7 +238,7 @@ func (b *Block) fillDataAvailabilityHeader() {
 
 	// TODO(ismail): for better efficiency and a larger number shares
 	// we should switch to the rsmt2d.LeopardFF16 codec:
-	extendedDataSquare, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree)
+	extendedDataSquare, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
 	if err != nil {
 		panic(fmt.Sprintf("unexpected error: %v", err))
 	}
@@ -310,7 +310,7 @@ func (b *Block) PutBlock(ctx context.Context, nodeAdder format.NodeAdder) error
 	}
 
 	// recompute the eds
-	eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree)
+	eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
 	if err != nil {
 		return fmt.Errorf("failure to recompute the extended data square: %w", err)
 	}
@@ -1384,6 +1384,12 @@ func (data *Data) ComputeShares() (NamespacedShares, int) {
 	// FIXME(ismail): this is not a power of two
 	// see: https://github.com/lazyledger/lazyledger-specs/issues/80 and
 	wantLen := getNextSquareNum(curLen)
+
+	// ensure that the min square size is used
+	if wantLen < minSharecount {
+		wantLen = minSharecount
+	}
+
 	tailShares := GenerateTailPaddingShares(wantLen-curLen, ShareSize)
 
 	return append(append(append(append(

types/block_test.go

@@ -241,6 +241,12 @@ func TestNilDataAvailabilityHeaderHashDoesntCrash(t *testing.T) {
 	assert.Equal(t, emptyBytes, new(DataAvailabilityHeader).Hash())
 }
 
+func TestEmptyBlockData(t *testing.T) {
+	blockData := Data{}
+	shares, _ := blockData.ComputeShares()
+	assert.Equal(t, GenerateTailPaddingShares(MinSquareSize, ShareSize), shares)
+}
+
 func TestCommit(t *testing.T) {
 	lastID := makeBlockIDRandom()
 	h := int64(3)

types/consts.go

@@ -30,6 +30,10 @@ const (
 	// 128*128*256 = 4 Megabytes
 	// TODO(ismail): settle on a proper max square
 	MaxSquareSize = 128
+
+	// MinSquareSize depicts the smallest original square width.
+	MinSquareSize = 1
+	minSharecount = MinSquareSize * MinSquareSize
 )
 
 var (

types/shares_test.go

@@ -243,7 +243,7 @@ func TestDataFromSquare(t *testing.T) {
 			shares, _ := data.ComputeShares()
 			rawShares := shares.RawShares()
 
-			eds, err := rsmt2d.ComputeExtendedDataSquare(rawShares, rsmt2d.RSGF8, rsmt2d.NewDefaultTree)
+			eds, err := rsmt2d.ComputeExtendedDataSquare(rawShares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
 			if err != nil {
 				t.Error(err)
 			}
@@ -446,10 +446,6 @@ func Test_parseDelimiter(t *testing.T) {
 	}
 }
 
-// ////////////////////////////
-// Test data generation
-// ////////////////////////////
-
 // generateRandomBlockData returns randomly generated block data for testing purposes
 func generateRandomBlockData(t *testing.T, txCount, isrCount, evdCount, msgCount, maxSize int) Data {
 	var out Data