From efac24840aa1a8e832aa73c55bdb1a61a4dee118 Mon Sep 17 00:00:00 2001 From: sanaz Date: Mon, 4 Mar 2024 17:13:12 -0800 Subject: [PATCH 01/10] extends computeRoots benchmark --- datasquare_test.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/datasquare_test.go b/datasquare_test.go index 78331db..7046558 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -3,6 +3,7 @@ package rsmt2d import ( "crypto/sha256" "fmt" + "math" "reflect" "testing" @@ -402,13 +403,21 @@ func Test_setColSlice(t *testing.T) { } func BenchmarkEDSRoots(b *testing.B) { - for i := 32; i < 513; i *= 2 { - square, err := newDataSquare(genRandDS(i*2, int(shareSize)), NewDefaultTree, shareSize) + ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 1024 MB to bytes + totalNumberOfShares := ODSSizeByteUpperBound / shareSize + ODSShareSizeUpperBound := int(math.Ceil(math.Sqrt(float64( + totalNumberOfShares)))) + for i := 32; i < ODSShareSizeUpperBound; i *= 2 { + ds := genRandDS(i*2, shareSize) // i*2 X i*2 data square (number of shares in the corresponding EDS) + square, err := newDataSquare(ds, NewDefaultTree, shareSize) if err != nil { b.Errorf("Failure to create square of size %d: %s", i, err) } b.Run( - fmt.Sprintf("%dx%dx%d ODS", i, i, int(square.chunkSize)), + fmt.Sprintf("%dx%dx%d ODS=%dMB, EDS=%dMB", i, i, + int(square.chunkSize), + int(i*i*512/(1024*1024)), + int(2*2*i*i*512/(1024*1024))), func(b *testing.B) { for n := 0; n < b.N; n++ { square.resetRoots() From 0de1cbe8263f35ca17420ae7af9722b52e1b7203 Mon Sep 17 00:00:00 2001 From: sanaz Date: Tue, 5 Mar 2024 10:57:43 -0800 Subject: [PATCH 02/10] uses erasured nmt as tree constructor --- datasquare_test.go | 24 ++++++++++++++++++------ extendeddatacrossword_test.go | 2 +- extendeddatasquare_test.go | 15 +++++++++++++++ nmtwrapper_test.go | 7 ++++--- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/datasquare_test.go b/datasquare_test.go index 7046558..e9e4a59 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/celestiaorg/merkletree" + "github.com/celestiaorg/nmt" "github.com/stretchr/testify/assert" ) @@ -405,19 +406,30 @@ func Test_setColSlice(t *testing.T) { func BenchmarkEDSRoots(b *testing.B) { ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 1024 MB to bytes totalNumberOfShares := ODSSizeByteUpperBound / shareSize - ODSShareSizeUpperBound := int(math.Ceil(math.Sqrt(float64( + oDSShareSizeUpperBound := int(math.Ceil(math.Sqrt(float64( totalNumberOfShares)))) - for i := 32; i < ODSShareSizeUpperBound; i *= 2 { - ds := genRandDS(i*2, shareSize) // i*2 X i*2 data square (number of shares in the corresponding EDS) - square, err := newDataSquare(ds, NewDefaultTree, shareSize) + namespaceIDSize := 29 + for i := 32; i < oDSShareSizeUpperBound; i *= 2 { + // generate an EDS with i*2 X i*2 dimensions in terms of shares + // the generated EDS does not conform to celestia-app specs in terms + // of namespace version, also no erasure encoding takes place + // yet none of these would affect the benchmarking + ds := genRandSortedDS(i*2, shareSize, 29) + + // a tree constructor for erasured nmt + treeConstructor := newErasuredNamespacedMerkleTreeConstructor(uint64(i*2), + nmt.NamespaceIDSize(namespaceIDSize), nmt.IgnoreMaxNamespace(true), + nmt.InitialCapacity(i*2)) + + square, err := newDataSquare(ds, treeConstructor, shareSize) if err != nil { b.Errorf("Failure to create square of size %d: %s", i, err) } b.Run( fmt.Sprintf("%dx%dx%d ODS=%dMB, EDS=%dMB", i, i, int(square.chunkSize), - int(i*i*512/(1024*1024)), - int(2*2*i*i*512/(1024*1024))), + i*i*512/(1024*1024), + 2*2*i*i*512/(1024*1024)), func(b *testing.B) { for n := 0; n < b.N; n++ { square.resetRoots() diff --git a/extendeddatacrossword_test.go b/extendeddatacrossword_test.go index 6266b79..470ef4b 100644 --- a/extendeddatacrossword_test.go +++ b/extendeddatacrossword_test.go @@ -439,7 +439,7 @@ func createTestEdsWithNMT(t *testing.T, codec Codec, shareSize, namespaceSize in edsWidth := 4 // number of shares per row/column in the extended data square odsWidth := edsWidth / 2 // number of shares per row/column in the original data square - eds, err := ComputeExtendedDataSquare(shares, codec, newConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) + eds, err := ComputeExtendedDataSquare(shares, codec, newErasuredNamespacedMerkleTreeConstructor(uint64(odsWidth), nmt.NamespaceIDSize(namespaceSize))) require.NoError(t, err) return eds diff --git a/extendeddatasquare_test.go b/extendeddatasquare_test.go index 057e78f..2b959c7 100644 --- a/extendeddatasquare_test.go +++ b/extendeddatasquare_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "reflect" + "sort" "testing" "github.com/stretchr/testify/assert" @@ -347,6 +348,20 @@ func genRandDS(width int, chunkSize int) [][]byte { return ds } +func genRandSortedDS(width int, chunkSize int, namespaceSize int) [][]byte { + + ds := genRandDS(width, chunkSize) + + // Sort the shares in the square based on their namespace + sort.Slice(ds, func(i, j int) bool { + // Compare only the first namespaceSize bytes + return bytes.Compare(ds[i][:namespaceSize], ds[j][:namespaceSize]) < 0 + }) + + return ds + +} + // TestFlattened_EDS tests that eds.Flattened() returns all the shares in the // EDS. This function has the `_EDS` suffix to avoid a name collision with the // TestFlattened. diff --git a/nmtwrapper_test.go b/nmtwrapper_test.go index 3e107da..ffa1fbe 100644 --- a/nmtwrapper_test.go +++ b/nmtwrapper_test.go @@ -54,7 +54,8 @@ type nmtTree interface { // with an underlying NMT of namespace size `NamespaceSize` and with // `ignoreMaxNamespace=true`. axisIndex is the index of the row or column that // this tree is committing to. squareSize must be greater than zero. -func newErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, options ...nmt.Option) erasuredNamespacedMerkleTree { +func newErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, + options ...nmt.Option) erasuredNamespacedMerkleTree { if squareSize == 0 { panic("cannot create a erasuredNamespacedMerkleTree of squareSize == 0") } @@ -73,10 +74,10 @@ type constructor struct { opts []nmt.Option } -// newConstructor creates a tree constructor function as required by rsmt2d to +// newErasuredNamespacedMerkleTreeConstructor creates a tree constructor function as required by rsmt2d to // calculate the data root. It creates that tree using the // erasuredNamespacedMerkleTree. -func newConstructor(squareSize uint64, opts ...nmt.Option) TreeConstructorFn { +func newErasuredNamespacedMerkleTreeConstructor(squareSize uint64, opts ...nmt.Option) TreeConstructorFn { return constructor{ squareSize: squareSize, opts: opts, From 7c56d2c4402c53d8aee250d2adc6714504899965 Mon Sep 17 00:00:00 2001 From: sanaz Date: Tue, 5 Mar 2024 12:19:05 -0800 Subject: [PATCH 03/10] fixes linter issues --- extendeddatasquare_test.go | 2 -- nmtwrapper_test.go | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/extendeddatasquare_test.go b/extendeddatasquare_test.go index 2b959c7..a532399 100644 --- a/extendeddatasquare_test.go +++ b/extendeddatasquare_test.go @@ -349,7 +349,6 @@ func genRandDS(width int, chunkSize int) [][]byte { } func genRandSortedDS(width int, chunkSize int, namespaceSize int) [][]byte { - ds := genRandDS(width, chunkSize) // Sort the shares in the square based on their namespace @@ -359,7 +358,6 @@ func genRandSortedDS(width int, chunkSize int, namespaceSize int) [][]byte { }) return ds - } // TestFlattened_EDS tests that eds.Flattened() returns all the shares in the diff --git a/nmtwrapper_test.go b/nmtwrapper_test.go index ffa1fbe..4f2079c 100644 --- a/nmtwrapper_test.go +++ b/nmtwrapper_test.go @@ -55,7 +55,8 @@ type nmtTree interface { // `ignoreMaxNamespace=true`. axisIndex is the index of the row or column that // this tree is committing to. squareSize must be greater than zero. func newErasuredNamespacedMerkleTree(squareSize uint64, axisIndex uint, - options ...nmt.Option) erasuredNamespacedMerkleTree { + options ...nmt.Option, +) erasuredNamespacedMerkleTree { if squareSize == 0 { panic("cannot create a erasuredNamespacedMerkleTree of squareSize == 0") } From ae531dd48ad8b0ea827eea37a7aec55c7f594ce0 Mon Sep 17 00:00:00 2001 From: sanaz Date: Tue, 5 Mar 2024 12:20:26 -0800 Subject: [PATCH 04/10] brings back the old benchmarking --- datasquare_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/datasquare_test.go b/datasquare_test.go index e9e4a59..00c96d3 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -403,7 +403,26 @@ func Test_setColSlice(t *testing.T) { } } -func BenchmarkEDSRoots(b *testing.B) { +func BenchmarkEDSRootsWithDefaultTree(b *testing.B) { + for i := 32; i < 513; i *= 2 { + square, err := newDataSquare(genRandDS(i*2, int(shareSize)), NewDefaultTree, shareSize) + if err != nil { + b.Errorf("Failure to create square of size %d: %s", i, err) + } + b.Run( + fmt.Sprintf("%dx%dx%d ODS", i, i, int(square.chunkSize)), + func(b *testing.B) { + for n := 0; n < b.N; n++ { + square.resetRoots() + err := square.computeRoots() + assert.NoError(b, err) + } + }, + ) + } +} + +func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 1024 MB to bytes totalNumberOfShares := ODSSizeByteUpperBound / shareSize oDSShareSizeUpperBound := int(math.Ceil(math.Sqrt(float64( From 26e4985f8a3cd5cf04578819cc0b093c1343eb96 Mon Sep 17 00:00:00 2001 From: sanaz Date: Tue, 5 Mar 2024 15:29:11 -0800 Subject: [PATCH 05/10] takes closest power of two to the total number of shares --- datasquare_test.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/datasquare_test.go b/datasquare_test.go index 00c96d3..294c22a 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -423,17 +423,20 @@ func BenchmarkEDSRootsWithDefaultTree(b *testing.B) { } func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { - ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 1024 MB to bytes - totalNumberOfShares := ODSSizeByteUpperBound / shareSize - oDSShareSizeUpperBound := int(math.Ceil(math.Sqrt(float64( + ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 512 MB to bytes + totalNumberOfShares := float64(ODSSizeByteUpperBound) / shareSize + // the closest power of 2 of the square root of + // the total number of shares + oDSShareSizeUpperBound := math.Pow(2, math.Ceil(math.Log2(math.Sqrt( totalNumberOfShares)))) namespaceIDSize := 29 - for i := 32; i < oDSShareSizeUpperBound; i *= 2 { + + for i := 32; i <= int(oDSShareSizeUpperBound); i *= 2 { // generate an EDS with i*2 X i*2 dimensions in terms of shares // the generated EDS does not conform to celestia-app specs in terms - // of namespace version, also no erasure encoding takes place - // yet none of these would affect the benchmarking - ds := genRandSortedDS(i*2, shareSize, 29) + // of namespace version, also no erasure encoding takes place + // yet none of these should impact the benchmarking + ds := genRandSortedDS(i*2, shareSize, namespaceIDSize) // a tree constructor for erasured nmt treeConstructor := newErasuredNamespacedMerkleTreeConstructor(uint64(i*2), @@ -447,8 +450,8 @@ func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { b.Run( fmt.Sprintf("%dx%dx%d ODS=%dMB, EDS=%dMB", i, i, int(square.chunkSize), - i*i*512/(1024*1024), - 2*2*i*i*512/(1024*1024)), + i*i*shareSize/(1024*1024), + 2*i*2*i*shareSize/(1024*1024)), func(b *testing.B) { for n := 0; n < b.N; n++ { square.resetRoots() From f253377e775a3358b58d6f2057ced446985c2180 Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 6 Mar 2024 12:18:10 -0800 Subject: [PATCH 06/10] updates comment --- datasquare_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasquare_test.go b/datasquare_test.go index 294c22a..9ab3b45 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -423,7 +423,7 @@ func BenchmarkEDSRootsWithDefaultTree(b *testing.B) { } func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { - ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 512 MB to bytes + ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 512 MiB to bytes totalNumberOfShares := float64(ODSSizeByteUpperBound) / shareSize // the closest power of 2 of the square root of // the total number of shares From 9b26deb0e51a339f088057992c461f845da921c1 Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 6 Mar 2024 12:28:21 -0800 Subject: [PATCH 07/10] fixes comments and introduces new vars --- datasquare_test.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/datasquare_test.go b/datasquare_test.go index 9ab3b45..3288d5a 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -423,7 +423,7 @@ func BenchmarkEDSRootsWithDefaultTree(b *testing.B) { } func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { - ODSSizeByteUpperBound := 1024 * 1024 * 1024 // converting 512 MiB to bytes + ODSSizeByteUpperBound := 512 * 1024 * 1024 // converting 512 MiB to bytes totalNumberOfShares := float64(ODSSizeByteUpperBound) / shareSize // the closest power of 2 of the square root of // the total number of shares @@ -432,11 +432,15 @@ func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { namespaceIDSize := 29 for i := 32; i <= int(oDSShareSizeUpperBound); i *= 2 { - // generate an EDS with i*2 X i*2 dimensions in terms of shares + // number of shares in the original data square's row/column + odsSize := i + // number of shares in the extended data square's row/column + edsSize := 2 * odsSize + // generate an EDS with edsSize X edsSize dimensions in terms of shares. // the generated EDS does not conform to celestia-app specs in terms // of namespace version, also no erasure encoding takes place // yet none of these should impact the benchmarking - ds := genRandSortedDS(i*2, shareSize, namespaceIDSize) + ds := genRandSortedDS(edsSize, shareSize, namespaceIDSize) // a tree constructor for erasured nmt treeConstructor := newErasuredNamespacedMerkleTreeConstructor(uint64(i*2), @@ -447,11 +451,12 @@ func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { if err != nil { b.Errorf("Failure to create square of size %d: %s", i, err) } + odsSizeMiBytes := i * i * shareSize / (1024 * 1024) // size in MiB + edsSizeMiBytes := 4 * odsSizeMiBytes b.Run( fmt.Sprintf("%dx%dx%d ODS=%dMB, EDS=%dMB", i, i, int(square.chunkSize), - i*i*shareSize/(1024*1024), - 2*i*2*i*shareSize/(1024*1024)), + odsSizeMiBytes, edsSizeMiBytes), func(b *testing.B) { for n := 0; n < b.N; n++ { square.resetRoots() From b7d58d552494849919b989b7fd08871ea6424dea Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 6 Mar 2024 12:30:42 -0800 Subject: [PATCH 08/10] adds more clarifications --- datasquare_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasquare_test.go b/datasquare_test.go index 3288d5a..f61e2b2 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -451,7 +451,9 @@ func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { if err != nil { b.Errorf("Failure to create square of size %d: %s", i, err) } - odsSizeMiBytes := i * i * shareSize / (1024 * 1024) // size in MiB + // the total size of the ODS in MiB + odsSizeMiBytes := i * i * shareSize / (1024 * 1024) + // the total size of the EDS in MiB edsSizeMiBytes := 4 * odsSizeMiBytes b.Run( fmt.Sprintf("%dx%dx%d ODS=%dMB, EDS=%dMB", i, i, From d9b113064ba838abe0708ae56be930658c732b17 Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 6 Mar 2024 12:51:12 -0800 Subject: [PATCH 09/10] renames a var --- datasquare_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasquare_test.go b/datasquare_test.go index f61e2b2..4ffb4f7 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -427,11 +427,11 @@ func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { totalNumberOfShares := float64(ODSSizeByteUpperBound) / shareSize // the closest power of 2 of the square root of // the total number of shares - oDSShareSizeUpperBound := math.Pow(2, math.Ceil(math.Log2(math.Sqrt( + nearestPowerOf2ODSSize := math.Pow(2, math.Ceil(math.Log2(math.Sqrt( totalNumberOfShares)))) namespaceIDSize := 29 - for i := 32; i <= int(oDSShareSizeUpperBound); i *= 2 { + for i := 32; i <= int(nearestPowerOf2ODSSize); i *= 2 { // number of shares in the original data square's row/column odsSize := i // number of shares in the extended data square's row/column From a67997ec08f9079f99271fd78dbb84af4bea74f2 Mon Sep 17 00:00:00 2001 From: sanaz Date: Wed, 6 Mar 2024 15:51:56 -0800 Subject: [PATCH 10/10] introduces mebibyte const --- datasquare_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/datasquare_test.go b/datasquare_test.go index 4ffb4f7..69fe033 100644 --- a/datasquare_test.go +++ b/datasquare_test.go @@ -423,7 +423,8 @@ func BenchmarkEDSRootsWithDefaultTree(b *testing.B) { } func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { - ODSSizeByteUpperBound := 512 * 1024 * 1024 // converting 512 MiB to bytes + const mebibyte = 1024 * 1024 // bytes + ODSSizeByteUpperBound := 512 * mebibyte // converting 512 MiB to bytes totalNumberOfShares := float64(ODSSizeByteUpperBound) / shareSize // the closest power of 2 of the square root of // the total number of shares @@ -431,9 +432,9 @@ func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { totalNumberOfShares)))) namespaceIDSize := 29 - for i := 32; i <= int(nearestPowerOf2ODSSize); i *= 2 { + for squareSize := 32; squareSize <= int(nearestPowerOf2ODSSize); squareSize *= 2 { // number of shares in the original data square's row/column - odsSize := i + odsSize := squareSize // number of shares in the extended data square's row/column edsSize := 2 * odsSize // generate an EDS with edsSize X edsSize dimensions in terms of shares. @@ -443,20 +444,20 @@ func BenchmarkEDSRootsWithErasuredNMT(b *testing.B) { ds := genRandSortedDS(edsSize, shareSize, namespaceIDSize) // a tree constructor for erasured nmt - treeConstructor := newErasuredNamespacedMerkleTreeConstructor(uint64(i*2), + treeConstructor := newErasuredNamespacedMerkleTreeConstructor(uint64(edsSize), nmt.NamespaceIDSize(namespaceIDSize), nmt.IgnoreMaxNamespace(true), - nmt.InitialCapacity(i*2)) + nmt.InitialCapacity(odsSize*2)) square, err := newDataSquare(ds, treeConstructor, shareSize) if err != nil { - b.Errorf("Failure to create square of size %d: %s", i, err) + b.Errorf("Failure to create square of size %d: %s", odsSize, err) } // the total size of the ODS in MiB - odsSizeMiBytes := i * i * shareSize / (1024 * 1024) + odsSizeMiBytes := odsSize * odsSize * shareSize / mebibyte // the total size of the EDS in MiB edsSizeMiBytes := 4 * odsSizeMiBytes b.Run( - fmt.Sprintf("%dx%dx%d ODS=%dMB, EDS=%dMB", i, i, + fmt.Sprintf("%dx%dx%d ODS=%dMB, EDS=%dMB", odsSize, odsSize, int(square.chunkSize), odsSizeMiBytes, edsSizeMiBytes), func(b *testing.B) {