feat: store and retrieve valid merkle roots in RLN db

waku/v2/protocol/rln/group_manager/dynamic/dynamic.go

@@ -95,6 +95,7 @@ func handler(gm *DynamicGroupManager, events []*contracts.RLNMemberRegistered) e
 		LastProcessedBlock: gm.lastBlockProcessed,
 		ChainID:            gm.web3Config.ChainID,
 		ContractAddress:    gm.web3Config.RegistryContract.Address,
+		ValidRootsPerBlock: gm.rootTracker.ValidRootsPerBlock(),
 	})
 	if err != nil {
 		// this is not a fatal error, hence we don't raise an exception

waku/v2/protocol/rln/group_manager/dynamic/metadata.go

@@ -6,13 +6,15 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/waku-org/go-waku/waku/v2/protocol/rln/group_manager"
 )
 
 // RLNMetadata persists attributes in the RLN database
 type RLNMetadata struct {
 	LastProcessedBlock uint64
 	ChainID            *big.Int
 	ContractAddress    common.Address
+	ValidRootsPerBlock []group_manager.RootsPerBlock
 }
 
 // Serialize converts a RLNMetadata into a binary format expected by zerokit's RLN
@@ -26,24 +28,49 @@ func (r RLNMetadata) Serialize() []byte {
 	result = binary.LittleEndian.AppendUint64(result, r.LastProcessedBlock)
 	result = binary.LittleEndian.AppendUint64(result, chainID.Uint64())
 	result = append(result, r.ContractAddress.Bytes()...)
+	result = binary.LittleEndian.AppendUint64(result, uint64(len(r.ValidRootsPerBlock)))
+	for _, v := range r.ValidRootsPerBlock {
+		result = append(result, v.Root[:]...)
+		result = binary.LittleEndian.AppendUint64(result, v.BlockNumber)
+	}
+
 	return result
 }
 
 const lastProcessedBlockOffset = 0
 const chainIDOffset = lastProcessedBlockOffset + 8
 const contractAddressOffset = chainIDOffset + 8
-const metadataByteLen = 8 + 8 + 20 // 2 uint64 fields and a 20bytes address
+const validRootsLenOffset = contractAddressOffset + 20
+const validRootsValOffset = validRootsLenOffset + 8
+const metadataByteLen = 8 + 8 + 20 + 8 // uint64 + uint64 + ethAddress + uint64
 
 // DeserializeMetadata converts a byte slice into a RLNMetadata instance
 func DeserializeMetadata(b []byte) (RLNMetadata, error) {
-	if len(b) != metadataByteLen {
+	if len(b) < metadataByteLen {
 		return RLNMetadata{}, errors.New("wrong size")
 	}
 
+	validRootLen := binary.LittleEndian.Uint64(b[validRootsLenOffset:validRootsValOffset])
+	if len(b) < int(metadataByteLen+validRootLen*(32+8)) { // len of a merkle node and len of a uint64 for the block number
+		return RLNMetadata{}, errors.New("wrong size")
+	}
+
+	validRoots := make([]group_manager.RootsPerBlock, 0, validRootLen)
+	for i := 0; i < int(validRootLen); i++ {
+		rootOffset := validRootsValOffset + (i * (32 + 8))
+		blockOffset := rootOffset + 32
+		root := group_manager.RootsPerBlock{
+			BlockNumber: binary.LittleEndian.Uint64(b[blockOffset : blockOffset+8]),
+		}
+		copy(root.Root[:], b[rootOffset:blockOffset])
+		validRoots = append(validRoots, root)
+	}
+
 	return RLNMetadata{
 		LastProcessedBlock: binary.LittleEndian.Uint64(b[lastProcessedBlockOffset:chainIDOffset]),
 		ChainID:            new(big.Int).SetUint64(binary.LittleEndian.Uint64(b[chainIDOffset:contractAddressOffset])),
-		ContractAddress:    common.BytesToAddress(b[contractAddressOffset:]),
+		ContractAddress:    common.BytesToAddress(b[contractAddressOffset:validRootsLenOffset]),
+		ValidRootsPerBlock: validRoots,
 	}, nil
 }
 

waku/v2/protocol/rln/group_manager/dynamic/metadata_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/require"
+	"github.com/waku-org/go-waku/waku/v2/protocol/rln/group_manager"
 )
 
 func TestMetadata(t *testing.T) {
@@ -14,6 +15,7 @@ func TestMetadata(t *testing.T) {
 		LastProcessedBlock: 128,
 		ChainID:            big.NewInt(1155511),
 		ContractAddress:    common.HexToAddress("0x9c09146844c1326c2dbc41c451766c7138f88155"),
+		ValidRootsPerBlock: []group_manager.RootsPerBlock{{Root: [32]byte{1}, BlockNumber: 100}, {Root: [32]byte{2}, BlockNumber: 200}},
 	}
 
 	serializedMetadata := metadata.Serialize()
@@ -23,9 +25,15 @@ func TestMetadata(t *testing.T) {
 	require.Equal(t, metadata.ChainID.Uint64(), unserializedMetadata.ChainID.Uint64())
 	require.Equal(t, metadata.LastProcessedBlock, unserializedMetadata.LastProcessedBlock)
 	require.Equal(t, metadata.ContractAddress.Hex(), unserializedMetadata.ContractAddress.Hex())
+	require.Len(t, unserializedMetadata.ValidRootsPerBlock, len(metadata.ValidRootsPerBlock))
+	require.Equal(t, metadata.ValidRootsPerBlock[0].BlockNumber, unserializedMetadata.ValidRootsPerBlock[0].BlockNumber)
+	require.Equal(t, metadata.ValidRootsPerBlock[0].Root, unserializedMetadata.ValidRootsPerBlock[0].Root)
+	require.Equal(t, metadata.ValidRootsPerBlock[1].BlockNumber, unserializedMetadata.ValidRootsPerBlock[1].BlockNumber)
+	require.Equal(t, metadata.ValidRootsPerBlock[1].Root, unserializedMetadata.ValidRootsPerBlock[1].Root)
 
-	// Handle cases where the chainId is not specified (for some reason?)
+	// Handle cases where the chainId is not specified (for some reason?) or no valid roots were specified
 	metadata.ChainID = nil
+	metadata.ValidRootsPerBlock = nil
 	serializedMetadata = metadata.Serialize()
 	unserializedMetadata, err = DeserializeMetadata(serializedMetadata)
 	require.NoError(t, err)

waku/v2/protocol/rln/group_manager/dynamic/web3.go

@@ -38,6 +38,8 @@ func (gm *DynamicGroupManager) HandleGroupUpdates(ctx context.Context, handler R
 		gm.log.Info("resuming onchain sync", zap.Uint64("fromBlock", fromBlock))
 	}
 
+	gm.rootTracker.SetValidRootsPerBlock(metadata.ValidRootsPerBlock)
+
 	err = gm.loadOldEvents(ctx, fromBlock, handler)
 	if err != nil {
 		return err

waku/v2/protocol/rln/group_manager/root_tracker.go

@@ -7,11 +7,14 @@ import (
 	"github.com/waku-org/go-zerokit-rln/rln"
 )
 
+// RootsPerBlock stores the merkle root generated at N block number
 type RootsPerBlock struct {
-	root        rln.MerkleNode
-	blockNumber uint64
+	Root        rln.MerkleNode
+	BlockNumber uint64
 }
 
+// MerkleRootTracker keeps track of the latest N merkle roots considered
+// valid for RLN proofs.
 type MerkleRootTracker struct {
 	sync.RWMutex
 
@@ -23,6 +26,7 @@ type MerkleRootTracker struct {
 
 const maxBufferSize = 20
 
+// NewMerkleRootTracker creates an instance of MerkleRootTracker
 func NewMerkleRootTracker(acceptableRootWindowSize int, rlnInstance *rln.RLN) (*MerkleRootTracker, error) {
 	result := &MerkleRootTracker{
 		acceptableRootWindowSize: acceptableRootWindowSize,
@@ -37,13 +41,14 @@ func NewMerkleRootTracker(acceptableRootWindowSize int, rlnInstance *rln.RLN) (*
 	return result, nil
 }
 
+// Backfill is used to pop merkle roots when there is a chain fork
 func (m *MerkleRootTracker) Backfill(fromBlockNumber uint64) {
 	m.Lock()
 	defer m.Unlock()
 
 	numBlocks := 0
 	for i := len(m.validMerkleRoots) - 1; i >= 0; i-- {
-		if m.validMerkleRoots[i].blockNumber >= fromBlockNumber {
+		if m.validMerkleRoots[i].BlockNumber >= fromBlockNumber {
 			numBlocks++
 		}
 	}
@@ -87,14 +92,16 @@ func (m *MerkleRootTracker) IndexOf(root [32]byte) int {
 	defer m.RUnlock()
 
 	for i := range m.validMerkleRoots {
-		if bytes.Equal(m.validMerkleRoots[i].root[:], root[:]) {
+		if bytes.Equal(m.validMerkleRoots[i].Root[:], root[:]) {
 			return i
 		}
 	}
 
 	return -1
 }
 
+// UpdateLatestRoot should be called when a block containing a new
+// IDCommitment is received so we can keep track of the merkle root change
 func (m *MerkleRootTracker) UpdateLatestRoot(blockNumber uint64) (rln.MerkleNode, error) {
 	m.Lock()
 	defer m.Unlock()
@@ -111,8 +118,8 @@ func (m *MerkleRootTracker) UpdateLatestRoot(blockNumber uint64) (rln.MerkleNode
 
 func (m *MerkleRootTracker) pushRoot(blockNumber uint64, root [32]byte) {
 	m.validMerkleRoots = append(m.validMerkleRoots, RootsPerBlock{
-		root:        root,
-		blockNumber: blockNumber,
+		Root:        root,
+		BlockNumber: blockNumber,
 	})
 
 	// Maintain valid merkle root window
@@ -125,29 +132,50 @@ func (m *MerkleRootTracker) pushRoot(blockNumber uint64, root [32]byte) {
 	if len(m.merkleRootBuffer) > maxBufferSize {
 		m.merkleRootBuffer = m.merkleRootBuffer[1:]
 	}
-
 }
 
+// Roots return the list of valid merkle roots
 func (m *MerkleRootTracker) Roots() []rln.MerkleNode {
 	m.RLock()
 	defer m.RUnlock()
 
 	result := make([]rln.MerkleNode, len(m.validMerkleRoots))
 	for i := range m.validMerkleRoots {
-		result[i] = m.validMerkleRoots[i].root
+		result[i] = m.validMerkleRoots[i].Root
 	}
 
 	return result
 }
 
+// Buffer is used as a repository of older merkle roots that although
+// they were valid once, they have left the acceptable window of
+// merkle roots. We keep track of them in case a chain fork occurs
+// and we need to restore the valid merkle roots to a previous point
+// of time
 func (m *MerkleRootTracker) Buffer() []rln.MerkleNode {
 	m.RLock()
 	defer m.RUnlock()
 
 	result := make([]rln.MerkleNode, len(m.merkleRootBuffer))
 	for i := range m.merkleRootBuffer {
-		result[i] = m.merkleRootBuffer[i].root
+		result[i] = m.merkleRootBuffer[i].Root
 	}
 
 	return result
 }
+
+// ValidRootsPerBlock returns the current valid merkle roots and block numbers
+func (m *MerkleRootTracker) ValidRootsPerBlock() []RootsPerBlock {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.validMerkleRoots
+}
+
+// SetValidRootsPErBlock is used to overwrite the valid merkle roots
+func (m *MerkleRootTracker) SetValidRootsPerBlock(roots []RootsPerBlock) {
+	m.Lock()
+	defer m.Unlock()
+
+	m.validMerkleRoots = roots
+}

waku/v2/protocol/rln/web3/web3.go

@@ -49,7 +49,7 @@ func NewConfig(ethClientAddress string, registryAddress common.Address) *Config
 
 // BuildConfig returns an instance of Web3Config with all the required elements for interaction with the RLN smart contracts
 func BuildConfig(ctx context.Context, ethClientAddress string, registryAddress common.Address) (*Config, error) {
-	ethClient, err := ethclient.Dial(ethClientAddress)
+	ethClient, err := ethclient.DialContext(ctx, ethClientAddress)
 	if err != nil {
 		return nil, err
 	}