From 90a611b338b78463db1e3ed513eff1acd1b6ab48 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Wed, 28 Jun 2023 08:36:49 -0500 Subject: [PATCH] feat: create foundation for fraudulent block production (#1992) ## Overview creates a malicious package that allows for minimal changes to the rest of the application while also keeping the malicious portion of the code as separate as possible as to not accidentally trigger the malicious logic. part of #1953 ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --------- Co-authored-by: Rootul P --- go.mod | 2 +- go.sum | 7 +- pkg/wrapper/nmt_wrapper.go | 19 ++- pkg/wrapper/nmt_wrapper_test.go | 27 +--- test/util/malicious/app.go | 67 ++++++++++ test/util/malicious/app_test.go | 66 ++++++++++ test/util/malicious/hasher.go | 223 ++++++++++++++++++++++++++++++++ test/util/malicious/test_app.go | 81 ++++++++++++ test/util/malicious/tree.go | 55 ++++++++ test/util/testfactory/common.go | 28 ++++ 10 files changed, 548 insertions(+), 27 deletions(-) create mode 100644 test/util/malicious/app.go create mode 100644 test/util/malicious/app_test.go create mode 100644 test/util/malicious/hasher.go create mode 100644 test/util/malicious/test_app.go create mode 100644 test/util/malicious/tree.go diff --git a/go.mod b/go.mod index ab9d9fbdf4..ab71099a32 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/celestiaorg/celestia-app go 1.20 require ( - github.com/celestiaorg/nmt v0.16.0 + github.com/celestiaorg/nmt v0.17.0 github.com/celestiaorg/quantum-gravity-bridge v1.3.0 github.com/ethereum/go-ethereum v1.12.0 github.com/gogo/protobuf v1.3.3 diff --git a/go.sum b/go.sum index de6ea5dcc5..12ef3bec6a 100644 --- a/go.sum +++ b/go.sum @@ -176,8 +176,8 @@ github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 h1:vaQKgaOm0w58JAvOgn2iDo github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13/go.mod h1:G9XkhOJZde36FH0kt/1ayg4ZaioZEQmmRfMa/zQig0I= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.16.0 h1:4CX6d1Uwf1C+tGcAWskPve0HCDTnI4Ey8ffjiDwcGH0= -github.com/celestiaorg/nmt v0.16.0/go.mod h1:GfwIvQPhUakn1modWxJ+rv8dUjJzuXg5H+MLFM1o7nY= +github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU= +github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= github.com/celestiaorg/rsmt2d v0.9.0 h1:kon78I748ZqjNzI8OAqPN+2EImuZuanj/6gTh8brX3o= @@ -987,8 +987,11 @@ github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= diff --git a/pkg/wrapper/nmt_wrapper.go b/pkg/wrapper/nmt_wrapper.go index 3ef4fa06e7..bc63f84730 100644 --- a/pkg/wrapper/nmt_wrapper.go +++ b/pkg/wrapper/nmt_wrapper.go @@ -6,6 +6,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/nmt" + "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" ) @@ -25,7 +26,7 @@ var ( type ErasuredNamespacedMerkleTree struct { squareSize uint64 // note: this refers to the width of the original square before erasure-coded options []nmt.Option - tree *nmt.NamespacedMerkleTree + tree Tree // axisIndex is the index of the axis (row or column) that this tree is on. This is passed // by rsmt2d and used to help determine which quadrant each leaf belongs to. axisIndex uint64 @@ -37,6 +38,16 @@ type ErasuredNamespacedMerkleTree struct { shareIndex uint64 } +// Tree is an interface that wraps the methods of the underlying +// NamespaceMerkleTree that are used by ErasuredNamespacedMerkleTree. This +// interface is mainly used for testing. It is not recommended to use this +// interface by implementing a different implementation. +type Tree interface { + Root() ([]byte, error) + Push(namespacedData namespace.PrefixedData) error + ProveRange(start, end int) (nmt.Proof, error) +} + // NewErasuredNamespacedMerkleTree creates a new ErasuredNamespacedMerkleTree // with an underlying NMT of namespace size `appconsts.NamespaceSize` and with // `ignoreMaxNamespace=true`. axisIndex is the index of the row or column that @@ -127,3 +138,9 @@ func (w *ErasuredNamespacedMerkleTree) incrementShareIndex() { func (w *ErasuredNamespacedMerkleTree) isQuadrantZero() bool { return w.shareIndex < w.squareSize && w.axisIndex < w.squareSize } + +// SetTree sets the underlying tree to the provided tree. This is used for +// testing purposes only. +func (w *ErasuredNamespacedMerkleTree) SetTree(tree Tree) { + w.tree = tree +} diff --git a/pkg/wrapper/nmt_wrapper_test.go b/pkg/wrapper/nmt_wrapper_test.go index 38d00a6c2f..89a668e3a8 100644 --- a/pkg/wrapper/nmt_wrapper_test.go +++ b/pkg/wrapper/nmt_wrapper_test.go @@ -9,11 +9,11 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/namespace" appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/celestiaorg/celestia-app/test/util/testfactory" "github.com/celestiaorg/nmt" nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/stretchr/testify/assert" - tmrand "github.com/tendermint/tendermint/libs/rand" ) func TestPushErasuredNamespacedMerkleTree(t *testing.T) { @@ -48,7 +48,7 @@ func TestPushErasuredNamespacedMerkleTree(t *testing.T) { // to the second half of the tree. func TestRootErasuredNamespacedMerkleTree(t *testing.T) { size := 8 - data := generateRandNamespacedRawData(size) + data := testfactory.GenerateRandNamespacedRawData(size) nmtErasured := NewErasuredNamespacedMerkleTree(uint64(size), 0) nmtStandard := nmt.New(sha256.New(), nmt.NamespaceIDSize(namespace.NamespaceSize), nmt.IgnoreMaxNamespace(true)) @@ -130,7 +130,7 @@ func TestErasureNamespacedMerkleTreePushErrors(t *testing.T) { func TestComputeExtendedDataSquare(t *testing.T) { squareSize := 4 // data for a 4X4 square - data := generateRandNamespacedRawData(squareSize * squareSize) + data := testfactory.GenerateRandNamespacedRawData(squareSize * squareSize) _, err := rsmt2d.ComputeExtendedDataSquare(data, appconsts.DefaultCodec(), NewConstructor(uint64(squareSize))) assert.NoError(t, err) @@ -140,7 +140,7 @@ func TestComputeExtendedDataSquare(t *testing.T) { // returns a slice that is twice as long as numLeaves because it returns the // original data + erasured data. func generateErasuredData(t *testing.T, numLeaves int, codec rsmt2d.Codec) [][]byte { - raw := generateRandNamespacedRawData(numLeaves) + raw := testfactory.GenerateRandNamespacedRawData(numLeaves) erasuredData, err := codec.Encode(raw) if err != nil { t.Error(err) @@ -148,25 +148,6 @@ func generateErasuredData(t *testing.T, numLeaves int, codec rsmt2d.Codec) [][]b return append(raw, erasuredData...) } -// generateRandNamespacedRawData returns random data of length count. Each chunk -// of random data is of size shareSize and is prefixed with a random blob -// namespace. -func generateRandNamespacedRawData(count int) (result [][]byte) { - for i := 0; i < count; i++ { - rawData := tmrand.Bytes(appconsts.ShareSize) - namespace := appns.RandomBlobNamespace().Bytes() - copy(rawData, namespace) - result = append(result, rawData) - } - - sortByteArrays(result) - return result -} - -func sortByteArrays(src [][]byte) { - sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 }) -} - // TestErasuredNamespacedMerkleTree_ProveRange checks that the proof returned by the ProveRange for all the shares within the erasured data is non-empty. func TestErasuredNamespacedMerkleTree_ProveRange(t *testing.T) { for sqaureSize := 1; sqaureSize <= 16; sqaureSize++ { diff --git a/test/util/malicious/app.go b/test/util/malicious/app.go new file mode 100644 index 0000000000..be542d8883 --- /dev/null +++ b/test/util/malicious/app.go @@ -0,0 +1,67 @@ +package malicious + +import ( + "io" + + "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/app/encoding" + "github.com/cosmos/cosmos-sdk/baseapp" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/log" + dbm "github.com/tendermint/tm-db" +) + +const ( + // PrepareProposalHandlerKey is the key used to retrieve the PrepareProposal handler from the + // app options. + PrepareProposalHandlerKey = "prepare_proposal_handler" +) + +type App struct { + *app.App + preparePropsoalHandler func(req abci.RequestPrepareProposal) abci.ResponsePrepareProposal +} + +func New( + logger log.Logger, + db dbm.DB, + traceStore io.Writer, + loadLatest bool, + skipUpgradeHeights map[int64]bool, + homePath string, + invCheckPeriod uint, + encodingConfig encoding.Config, + appOpts servertypes.AppOptions, + baseAppOptions ...func(*baseapp.BaseApp), +) *App { + goodApp := app.New(logger, db, traceStore, loadLatest, skipUpgradeHeights, homePath, invCheckPeriod, encodingConfig, appOpts, baseAppOptions...) + badApp := &App{App: goodApp} + + // default to using the good app's handlers + badApp.SetPrepareProposalHandler(goodApp.PrepareProposal) + + // override the handler if it is set in the app options + if prepareHander := appOpts.Get(PrepareProposalHandlerKey); prepareHander != nil { + badApp.SetPrepareProposalHandler(prepareHander.(func(req abci.RequestPrepareProposal) abci.ResponsePrepareProposal)) + } + + return badApp +} + +func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePrepareProposal { + return app.preparePropsoalHandler(req) +} + +// SetPrepareProposalHandler sets the PrepareProposal handler. +func (app *App) SetPrepareProposalHandler(handler func(req abci.RequestPrepareProposal) abci.ResponsePrepareProposal) { + app.preparePropsoalHandler = handler +} + +// ProcessProposal overwrites the default app's method to auto accept any +// proposal. +func (app *App) ProcessProposal(_ abci.RequestProcessProposal) (resp abci.ResponseProcessProposal) { + return abci.ResponseProcessProposal{ + Result: abci.ResponseProcessProposal_ACCEPT, + } +} diff --git a/test/util/malicious/app_test.go b/test/util/malicious/app_test.go new file mode 100644 index 0000000000..05bef7a317 --- /dev/null +++ b/test/util/malicious/app_test.go @@ -0,0 +1,66 @@ +package malicious + +import ( + "testing" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/celestia-app/test/util/testfactory" + "github.com/celestiaorg/celestia-app/test/util/testnode" + "github.com/stretchr/testify/require" + tmrand "github.com/tendermint/tendermint/libs/rand" +) + +// TestOutOfOrderNMT tests that the malicious NMT implementation is able to +// generate the same root as the ordered NMT implementation when the leaves are +// added in the same order and can generate roots when leaves are out of +// order. +func TestOutOfOrderNMT(t *testing.T) { + squareSize := uint64(64) + c := NewConstructor(squareSize) + goodConstructor := wrapper.NewConstructor(squareSize) + + orderedTree := goodConstructor(0, 0) + maliciousOrderedTree := c(0, 0) + maliciousUnorderedTree := c(0, 0) + data := testfactory.GenerateRandNamespacedRawData(64) + + // compare the roots generated by pushing ordered data + for _, d := range data { + err := orderedTree.Push(d) + require.NoError(t, err) + err = maliciousOrderedTree.Push(d) + require.NoError(t, err) + } + + goodOrderedRoot, err := orderedTree.Root() + require.NoError(t, err) + malOrderedRoot, err := maliciousOrderedTree.Root() + require.NoError(t, err) + require.Equal(t, goodOrderedRoot, malOrderedRoot) + + // test the new tree with unordered data + for i := range data { + j := tmrand.Intn(len(data)) + data[i], data[j] = data[j], data[i] + } + + for _, d := range data { + err := maliciousUnorderedTree.Push(d) + require.NoError(t, err) + } + + root, err := maliciousUnorderedTree.Root() + require.NoError(t, err) + require.Len(t, root, 90) // two namespaces + 32 bytes of hash = 90 bytes + require.NotEqual(t, goodOrderedRoot, root) // quick sanity check to ensure the roots are different +} + +func TestMaliciousNodeTestNode(t *testing.T) { + // TODO: flesh out this test further + cfg := testnode.DefaultConfig(). + WithAppCreator(NewAppServer) + + cctx, _, _ := testnode.NewNetwork(t, cfg) + + require.NoError(t, cctx.WaitForNextBlock()) +} diff --git a/test/util/malicious/hasher.go b/test/util/malicious/hasher.go new file mode 100644 index 0000000000..bc1d661f7c --- /dev/null +++ b/test/util/malicious/hasher.go @@ -0,0 +1,223 @@ +package malicious + +import ( + "bytes" + "errors" + "hash" + + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/nmt/namespace" +) + +// NOTE: This file is a copy of the original nmt.Hasher implementation, but with +// all of the validation logic removed. This allows for malicious nodes to create nmt roots that are out of order. + +const ( + LeafPrefix = 0 + NodePrefix = 1 +) + +var _ hash.Hash = (*Hasher)(nil) + +var ( + ErrInvalidNodeLen = errors.New("invalid NMT node size") + ErrInvalidLeafLen = errors.New("invalid NMT leaf size") +) + +type Hasher struct { + baseHasher hash.Hash + NamespaceLen namespace.IDSize + + // The "ignoreMaxNs" flag influences the calculation of the namespace ID + // range for intermediate nodes in the tree i.e., HashNode method. This flag + // signals that, when determining the upper limit of the namespace ID range + // for a tree node, the maximum possible namespace ID (equivalent to + // "NamespaceLen" bytes of 0xFF, or 2^NamespaceLen-1) should be omitted if + // feasible. For a more in-depth understanding of this field, refer to the + // "HashNode". + ignoreMaxNs bool + precomputedMaxNs namespace.ID + + tp byte // keeps type of NMT node to be hashed + data []byte // written data of the NMT node +} + +func (n *Hasher) IsMaxNamespaceIDIgnored() bool { + return n.ignoreMaxNs +} + +func (n *Hasher) NamespaceSize() namespace.IDSize { + return n.NamespaceLen +} + +// NewBlindHasher returns a Hasher that performs no ordering validation when the Write method is invoked. +func NewBlindHasher(baseHasher hash.Hash, nidLen namespace.IDSize, ignoreMaxNamespace bool) *Hasher { + return &Hasher{ + baseHasher: baseHasher, + NamespaceLen: nidLen, + ignoreMaxNs: ignoreMaxNamespace, + precomputedMaxNs: bytes.Repeat([]byte{0xFF}, int(nidLen)), + } +} + +// Size returns the number of bytes Sum will return. +func (n *Hasher) Size() int { + return n.baseHasher.Size() + int(n.NamespaceLen)*2 +} + +// Write writes the namespaced data to be hashed. +// +// Requires data of fixed size to match leaf or inner NMT nodes. Only a single +// write is allowed. +// It panics if more than one single write is attempted. +// If the data does not match the format of an NMT non-leaf node or leaf node, an error will be returned. +func (n *Hasher) Write(data []byte) (int, error) { + if n.data != nil { + panic("only a single Write is allowed") + } + + ln := len(data) + switch ln { + // inner nodes are made up of the nmt hashes of the left and right children + case n.Size() * 2: + n.tp = NodePrefix + // leaf nodes contain the namespace length and a share + default: + n.tp = LeafPrefix + } + + n.data = data + return ln, nil +} + +// Sum computes the hash. Does not append the given suffix, violating the +// interface. +// It may panic if the data being hashed is invalid. This should never happen since the Write method refuses an invalid data and errors out. +func (n *Hasher) Sum([]byte) []byte { + switch n.tp { + case LeafPrefix: + res, err := n.HashLeaf(n.data) + if err != nil { + panic(err) // this should never happen since the data is already validated in the Write method + } + return res + case NodePrefix: + flagLen := int(n.NamespaceLen) * 2 + sha256Len := n.baseHasher.Size() + leftChild := n.data[:flagLen+sha256Len] + rightChild := n.data[flagLen+sha256Len:] + res, err := n.HashNode(leftChild, rightChild) + if err != nil { + panic(err) // this should never happen since the data is already validated in the Write method + } + return res + default: + panic("nmt node type wasn't set") + } +} + +// Reset resets the Hash to its initial state. +func (n *Hasher) Reset() { + n.tp, n.data = 255, nil // reset with an invalid node type, as zero value is a valid Leaf + n.baseHasher.Reset() +} + +// BlockSize returns the hash's underlying block size. +func (n *Hasher) BlockSize() int { + return n.baseHasher.BlockSize() +} + +func (n *Hasher) EmptyRoot() []byte { + n.baseHasher.Reset() + emptyNs := bytes.Repeat([]byte{0}, int(n.NamespaceLen)) + h := n.baseHasher.Sum(nil) + digest := append(append(emptyNs, emptyNs...), h...) + + return digest +} + +// HashLeaf computes namespace hash of the namespaced data item `ndata` as +// ns(ndata) || ns(ndata) || hash(leafPrefix || ndata), where ns(ndata) is the +// namespaceID inside the data item namely leaf[:n.NamespaceLen]). Note that for +// leaves minNs = maxNs = ns(leaf) = leaf[:NamespaceLen]. HashLeaf can return the ErrInvalidNodeLen error if the input is not namespaced. +// +//nolint:errcheck +func (n *Hasher) HashLeaf(ndata []byte) ([]byte, error) { + h := n.baseHasher + h.Reset() + + nID := ndata[:n.NamespaceLen] + resLen := int(2*n.NamespaceLen) + n.baseHasher.Size() + minMaxNIDs := make([]byte, 0, resLen) + minMaxNIDs = append(minMaxNIDs, nID...) // nID + minMaxNIDs = append(minMaxNIDs, nID...) // nID || nID + + // add LeafPrefix to the ndata + leafPrefixedNData := make([]byte, 0, len(ndata)+1) + leafPrefixedNData = append(leafPrefixedNData, LeafPrefix) + leafPrefixedNData = append(leafPrefixedNData, ndata...) + h.Write(leafPrefixedNData) + + // compute h(LeafPrefix || ndata) and append it to the minMaxNIDs + nameSpacedHash := h.Sum(minMaxNIDs) // nID || nID || h(LeafPrefix || ndata) + return nameSpacedHash, nil +} + +// MustHashLeaf is a wrapper around HashLeaf that panics if an error is +// encountered. The ndata must be a valid leaf node. +func (n *Hasher) MustHashLeaf(ndata []byte) []byte { + res, err := n.HashLeaf(ndata) + if err != nil { + panic(err) + } + return res +} + +// HashNode calculates a namespaced hash of a node using the supplied left and +// right children. The input values, `left` and `right,` are namespaced hash +// values with the format `minNID || maxNID || hash.` +// The HashNode function returns an error if the provided inputs are invalid. Specifically, it returns the ErrInvalidNodeLen error if the left and right inputs are not in the namespaced hash format, +// and the ErrUnorderedSiblings error if left.maxNID is greater than right.minNID. +// By default, the normal namespace hash calculation is +// followed, which is `res = min(left.minNID, right.minNID) || max(left.maxNID, +// right.maxNID) || H(NodePrefix, left, right)`. `res` refers to the return +// value of the HashNode. However, if the `ignoreMaxNs` property of the Hasher +// is set to true, the calculation of the namespace ID range of the node +// slightly changes. Let MAXNID be the maximum possible namespace ID value i.e., 2^NamespaceIDSize-1. +// If the namespace range of the right child is start=end=MAXNID, indicating that it represents the root of a subtree whose leaves all have the namespace ID of `MAXNID`, then exclude the right child from the namespace range calculation. Instead, +// assign the namespace range of the left child as the parent's namespace range. +func (n *Hasher) HashNode(left, right []byte) ([]byte, error) { + h := n.baseHasher + h.Reset() + + leftMinNs, leftMaxNs := nmt.MinNamespace(left, n.NamespaceLen), nmt.MaxNamespace(left, n.NamespaceLen) + rightMinNs, rightMaxNs := nmt.MinNamespace(right, n.NamespaceLen), nmt.MaxNamespace(right, n.NamespaceLen) + + // compute the namespace range of the parent node + minNs, maxNs := computeNsRange(leftMinNs, leftMaxNs, rightMinNs, rightMaxNs, n.ignoreMaxNs, n.precomputedMaxNs) + + res := make([]byte, 0) + res = append(res, minNs...) + res = append(res, maxNs...) + + // Note this seems a little faster than calling several Write()s on the + // underlying Hash function (see: + // https://github.com/google/trillian/pull/1503): + data := make([]byte, 0, 1+len(left)+len(right)) + data = append(data, NodePrefix) + data = append(data, left...) + data = append(data, right...) + //nolint:errcheck + h.Write(data) + return h.Sum(res), nil +} + +// computeNsRange computes the namespace range of the parent node based on the namespace ranges of its left and right children. +func computeNsRange(leftMinNs, leftMaxNs, rightMinNs, rightMaxNs []byte, ignoreMaxNs bool, precomputedMaxNs namespace.ID) (minNs []byte, maxNs []byte) { + minNs = leftMinNs + maxNs = rightMaxNs + if ignoreMaxNs && bytes.Equal(precomputedMaxNs, rightMinNs) { + maxNs = leftMaxNs + } + return minNs, maxNs +} diff --git a/test/util/malicious/test_app.go b/test/util/malicious/test_app.go new file mode 100644 index 0000000000..6cabf4413a --- /dev/null +++ b/test/util/malicious/test_app.go @@ -0,0 +1,81 @@ +package malicious + +import ( + "io" + "path/filepath" + + "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/app/encoding" + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/test/util" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + "github.com/cosmos/cosmos-sdk/snapshots" + snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types" + "github.com/cosmos/cosmos-sdk/store" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/spf13/cast" + "github.com/tendermint/tendermint/libs/log" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + dbm "github.com/tendermint/tm-db" +) + +func NewTestApp(cparams *tmproto.ConsensusParams, genAccounts ...string) *App { + app, _ := util.SetupTestAppWithGenesisValSet(cparams, genAccounts...) + badapp := &App{App: app} + badapp.SetPrepareProposalHandler(app.PrepareProposal) + return badapp +} + +func NewAppServer(logger log.Logger, db dbm.DB, traceStore io.Writer, appOpts servertypes.AppOptions) servertypes.Application { + var cache sdk.MultiStorePersistentCache + + if cast.ToBool(appOpts.Get(server.FlagInterBlockCache)) { + cache = store.NewCommitKVStoreCacheManager() + } + + skipUpgradeHeights := make(map[int64]bool) + for _, h := range cast.ToIntSlice(appOpts.Get(server.FlagUnsafeSkipUpgrades)) { + skipUpgradeHeights[int64(h)] = true + } + + pruningOpts, err := server.GetPruningOptionsFromFlags(appOpts) + if err != nil { + panic(err) + } + + // Add snapshots + snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots") + //nolint: staticcheck + snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir) + if err != nil { + panic(err) + } + snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir) + if err != nil { + panic(err) + } + + return New( + logger, db, traceStore, true, skipUpgradeHeights, + cast.ToString(appOpts.Get(flags.FlagHome)), + cast.ToUint(appOpts.Get(server.FlagInvCheckPeriod)), + encoding.MakeConfig(app.ModuleEncodingRegisters...), // Ideally, we would reuse the one created by NewRootCmd. + appOpts, + baseapp.SetPruning(pruningOpts), + baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))), + baseapp.SetMinRetainBlocks(cast.ToUint64(appOpts.Get(server.FlagMinRetainBlocks))), + baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))), + baseapp.SetHaltTime(cast.ToUint64(appOpts.Get(server.FlagHaltTime))), + baseapp.SetMinRetainBlocks(cast.ToUint64(appOpts.Get(server.FlagMinRetainBlocks))), + baseapp.SetInterBlockCache(cache), + baseapp.SetTrace(cast.ToBool(appOpts.Get(server.FlagTrace))), + baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))), + baseapp.SetSnapshot(snapshotStore, snapshottypes.NewSnapshotOptions(cast.ToUint64(appOpts.Get(server.FlagStateSyncSnapshotInterval)), cast.ToUint32(appOpts.Get(server.FlagStateSyncSnapshotKeepRecent)))), + func(b *baseapp.BaseApp) { + b.SetProtocolVersion(appconsts.LatestVersion) + }, + ) +} diff --git a/test/util/malicious/tree.go b/test/util/malicious/tree.go new file mode 100644 index 0000000000..98702ab138 --- /dev/null +++ b/test/util/malicious/tree.go @@ -0,0 +1,55 @@ +package malicious + +import ( + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/rsmt2d" +) + +// BlindTree is a wrapper around the erasured NMT that skips verification of +// namespace ordering when hashing and adding leaves to the tree. It does this +// by using a custom malicious hahser and wraps using the ForceAddLeaf method +// instead of the normal Push. +type BlindTree struct { + *nmt.NamespacedMerkleTree +} + +// Push overwrites the nmt method to skip namespace verification. +func (bt *BlindTree) Push(data namespace.PrefixedData) error { + return bt.ForceAddLeaf(data) +} + +type constructor struct { + squareSize uint64 + opts []nmt.Option +} + +// NewConstructor creates a tree constructor function as required by rsmt2d to +// calculate the data root. It creates that tree using a malicious version of +// the wrapper.ErasuredNamespacedMerkleTree. +func NewConstructor(squareSize uint64, opts ...nmt.Option) rsmt2d.TreeConstructorFn { + hasher := NewBlindHasher(appconsts.NewBaseHashFunc(), appconsts.NamespaceSize, true) + copts := []nmt.Option{ + nmt.CustomHasher(hasher), + nmt.NamespaceIDSize(appconsts.NamespaceSize), + nmt.IgnoreMaxNamespace(true), + } + opts = append(opts, copts...) + return constructor{ + squareSize: squareSize, + opts: opts, + }.NewTree +} + +// NewTree creates a new rsmt2d.Tree using the malicious +// wrapper.ErasuredNamespacedMerkleTree with predefined square size and +// nmt.Options. +func (c constructor) NewTree(_ rsmt2d.Axis, axisIndex uint) rsmt2d.Tree { + nmtTree := nmt.New(appconsts.NewBaseHashFunc(), c.opts...) + maliciousTree := &BlindTree{nmtTree} + newTree := wrapper.NewErasuredNamespacedMerkleTree(c.squareSize, axisIndex, c.opts...) + newTree.SetTree(maliciousTree) + return &newTree +} diff --git a/test/util/testfactory/common.go b/test/util/testfactory/common.go index 4dbc5f208c..f36194a3b3 100644 --- a/test/util/testfactory/common.go +++ b/test/util/testfactory/common.go @@ -1,5 +1,14 @@ package testfactory +import ( + "bytes" + "sort" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/namespace" + tmrand "github.com/tendermint/tendermint/libs/rand" +) + func Repeat[T any](s T, count int) []T { ss := make([]T, count) for i := 0; i < count; i++ { @@ -7,3 +16,22 @@ func Repeat[T any](s T, count int) []T { } return ss } + +// GenerateRandNamespacedRawData returns random data of length count. Each chunk +// of random data is of size shareSize and is prefixed with a random blob +// namespace. +func GenerateRandNamespacedRawData(count int) (result [][]byte) { + for i := 0; i < count; i++ { + rawData := tmrand.Bytes(appconsts.ShareSize) + namespace := namespace.RandomBlobNamespace().Bytes() + copy(rawData, namespace) + result = append(result, rawData) + } + + sortByteArrays(result) + return result +} + +func sortByteArrays(src [][]byte) { + sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 }) +}