Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(rln-relay): use batch operations API #617

Merged
merged 1 commit into from
Aug 12, 2023
Merged

feat(rln-relay): use batch operations API #617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion flake.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
];
doCheck = false;
# FIXME: This needs to be manually changed when updating modules.
vendorSha256 = "sha256-cMjGBz4/sDRcE55cfPBTl1lFkvHfZHeM4bTBcs5lO2s=";
vendorSha256 = "sha256-8khocBcz0klKwhoYKoPOvMSwFwa8rFcFSRM9jL8A0BU=";
# Fix for 'nix run' trying to execute 'go-waku'.
meta = { mainProgram = "waku"; };
};
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ require (
github.com/jackc/pgx/v5 v5.4.1
github.com/waku-org/go-libp2p-rendezvous v0.0.0-20230628220917-7b4e5ae4c0e7
github.com/waku-org/go-noise v0.0.4
github.com/waku-org/go-zerokit-rln v0.1.13-0.20230803113701-ea89e5d7eedb
github.com/waku-org/go-zerokit-rln v0.1.13
github.com/wk8/go-ordered-map v1.0.0
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1554,8 +1554,8 @@ github.com/waku-org/go-libp2p-rendezvous v0.0.0-20230628220917-7b4e5ae4c0e7 h1:0
github.com/waku-org/go-libp2p-rendezvous v0.0.0-20230628220917-7b4e5ae4c0e7/go.mod h1:pFvOZ9YTFsW0o5zJW7a0B5tr1owAijRWJctXJ2toL04=
github.com/waku-org/go-noise v0.0.4 h1:ZfQDcCw8pazm89EBl5SXY7GGAnzDQb9AHFXlw3Ktbvk=
github.com/waku-org/go-noise v0.0.4/go.mod h1:+PWRfs2eSOVwKrPcQlfhwDngSh3faL/1QoxvoqggEKc=
github.com/waku-org/go-zerokit-rln v0.1.13-0.20230803113701-ea89e5d7eedb h1:f1UbFvroZbAul1HTlzM4ZGgjQ3mtn0bW8Jvk7XZQ5V4=
github.com/waku-org/go-zerokit-rln v0.1.13-0.20230803113701-ea89e5d7eedb/go.mod h1:1S6g1KXC45HkDXhIWD9+mdAs9fdyzQy8gtmw4RLjVcM=
github.com/waku-org/go-zerokit-rln v0.1.13 h1:9Ti/my7dmpJGNxNpL1MvVurzAjSd6xwBOm2WUZi33Mo=
github.com/waku-org/go-zerokit-rln v0.1.13/go.mod h1:1S6g1KXC45HkDXhIWD9+mdAs9fdyzQy8gtmw4RLjVcM=
github.com/waku-org/go-zerokit-rln-apple v0.0.0-20230803113401-9a7ef94d120e h1:Ad0rJod5F1FuYCJ8SUB/bQZsQwirNHQRE0IcaVloxZo=
github.com/waku-org/go-zerokit-rln-apple v0.0.0-20230803113401-9a7ef94d120e/go.mod h1:KYykqtdApHVYZ3G0spwMnoxc5jH5eI3jyO9SwsSfi48=
github.com/waku-org/go-zerokit-rln-arm v0.0.0-20230801152407-8101ff87ee0a h1:10cre+P76QvnLeyeCVAM8WDbUCri/y5xY3LtwI9Y5DE=
Expand Down
43 changes: 26 additions & 17 deletions waku/v2/protocol/rln/group_manager/dynamic/dynamic.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,12 @@ func handler(gm *DynamicGroupManager, events []*contracts.RLNMemberRegistered) e
toInsertTable := om.New()
for _, event := range events {
if event.Raw.Removed {
var indexes []uint64
var indexes []uint
i_idx, ok := toRemoveTable.Get(event.Raw.BlockNumber)
if ok {
indexes = i_idx.([]uint64)
indexes = i_idx.([]uint)
}
indexes = append(indexes, event.Index.Uint64())
indexes = append(indexes, uint(event.Index.Uint64()))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this cause truncation in future if someone uses a 32bit machine and membership set size grows?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm. indeed. I'm gonna change go-zerokit-rln to receive uint64 instead of only uints

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nvm. Looking at the implementation in zerokit, the functions that accept indexes receive an uintptr_t value which will depend on the platform architecture...

@rymnc - do you think we should change zerokit's FFI api to use u64 instead of usize for the indexes? what's the maximum number of leaves that a merkle tree can have?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

by default, the tree_height is 20, so 1_048_576
I think we fixed the issue of serialization for usize's 🤔

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

indeed, serialization should be fine, but i wonder if in this case (ffi), it would be a problem, as you'd be limited to a maximum index of 2^32-1 in a 32b machine, due to using usize 🤔

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't believe we'd cross 2^32 ~ since 2^20 is the max tree height

toRemoveTable.Set(event.Raw.BlockNumber, indexes)
} else {
var eventsPerBlock []*contracts.RLNMemberRegistered
Expand Down Expand Up @@ -278,17 +278,28 @@ func (gm *DynamicGroupManager) persistCredentials() error {
func (gm *DynamicGroupManager) InsertMembers(toInsert *om.OrderedMap) error {
for pair := toInsert.Oldest(); pair != nil; pair = pair.Next() {
events := pair.Value.([]*contracts.RLNMemberRegistered) // TODO: should these be sortered by index? we assume all members arrive in order
var idCommitments []rln.IDCommitment
var oldestIndexInBlock *big.Int
for _, evt := range events {
pubkey := rln.Bytes32(evt.Pubkey.Bytes())
// TODO: should we track indexes to identify missing?
err := gm.rln.InsertMember(pubkey)
if err != nil {
gm.log.Error("inserting member into merkletree", zap.Error(err))
return err
if oldestIndexInBlock == nil {
oldestIndexInBlock = evt.Index
}
idCommitments = append(idCommitments, rln.Bytes32(evt.Pubkey.Bytes()))
}

_, err := gm.rootTracker.UpdateLatestRoot(pair.Key.(uint64))
if len(idCommitments) == 0 {
continue
}

// TODO: should we track indexes to identify missing?
startIndex := rln.MembershipIndex(uint(oldestIndexInBlock.Int64()))
err := gm.rln.InsertMembers(startIndex, idCommitments)
if err != nil {
gm.log.Error("inserting members into merkletree", zap.Error(err))
return err
}

_, err = gm.rootTracker.UpdateLatestRoot(pair.Key.(uint64))
if err != nil {
return err
}
Expand All @@ -298,13 +309,11 @@ func (gm *DynamicGroupManager) InsertMembers(toInsert *om.OrderedMap) error {

func (gm *DynamicGroupManager) RemoveMembers(toRemove *om.OrderedMap) error {
for pair := toRemove.Newest(); pair != nil; pair = pair.Prev() {
memberIndexes := pair.Value.([]uint64)
for _, index := range memberIndexes {
err := gm.rln.DeleteMember(uint(index))
if err != nil {
gm.log.Error("deleting member", zap.Error(err))
return err
}
memberIndexes := pair.Value.([]uint)
err := gm.rln.DeleteMembers(memberIndexes)
if err != nil {
gm.log.Error("deleting members", zap.Error(err))
return err
}
gm.rootTracker.Backfill(pair.Key.(uint64))
}
Expand Down
8 changes: 4 additions & 4 deletions waku/v2/protocol/rln/group_manager/dynamic/handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ func TestHandler(t *testing.T) {
roots = gm.rootTracker.Roots()

require.Len(t, roots, 2)
require.Equal(t, roots[0], root0)
require.Equal(t, roots[1], [32]byte{253, 232, 31, 10, 168, 25, 42, 0, 28, 221, 146, 119, 34, 212, 121, 51, 82, 55, 113, 181, 236, 3, 11, 190, 194, 144, 125, 59, 46, 171, 90, 43})
require.Equal(t, root0, roots[0])
require.Equal(t, [32]byte{0x96, 0x65, 0xdb, 0xbc, 0x28, 0x6b, 0x1f, 0xc6, 0xab, 0x2d, 0x82, 0xe, 0x30, 0xe0, 0xc1, 0x92, 0x32, 0x73, 0xd2, 0xd6, 0xe1, 0x21, 0xef, 0xa1, 0xc8, 0xa6, 0xa6, 0xd, 0x9, 0x7c, 0x85, 0x19}, roots[1])

events = []*contracts.RLNMemberRegistered{
eventBuilder(1, false, 0xbbbb, 2),
Expand All @@ -75,9 +75,9 @@ func TestHandler(t *testing.T) {
// Root[1] should become [0]
roots = gm.rootTracker.Roots()
require.Len(t, roots, 5)
require.Equal(t, roots[0], [32]byte{253, 232, 31, 10, 168, 25, 42, 0, 28, 221, 146, 119, 34, 212, 121, 51, 82, 55, 113, 181, 236, 3, 11, 190, 194, 144, 125, 59, 46, 171, 90, 43})
require.Equal(t, [32]byte{0x96, 0x65, 0xdb, 0xbc, 0x28, 0x6b, 0x1f, 0xc6, 0xab, 0x2d, 0x82, 0xe, 0x30, 0xe0, 0xc1, 0x92, 0x32, 0x73, 0xd2, 0xd6, 0xe1, 0x21, 0xef, 0xa1, 0xc8, 0xa6, 0xa6, 0xd, 0x9, 0x7c, 0x85, 0x19}, roots[0])
require.Len(t, rootTracker.Buffer(), 1)
require.Equal(t, rootTracker.Buffer()[0], [32]byte{62, 31, 25, 34, 223, 182, 113, 211, 249, 18, 247, 234, 70, 30, 10, 136, 238, 132, 143, 221, 225, 43, 108, 24, 171, 26, 210, 197, 106, 231, 52, 33})
require.Equal(t, root0, rootTracker.Buffer()[0])

// We detect a fork
//
Expand Down
25 changes: 13 additions & 12 deletions waku/v2/protocol/rln/group_manager/static/static.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type StaticGroupManager struct {

group []rln.IDCommitment
rootTracker *group_manager.MerkleRootTracker
nextIndex uint64
}

func NewStaticGroupManager(
Expand Down Expand Up @@ -46,33 +47,33 @@ func (gm *StaticGroupManager) Start(ctx context.Context, rlnInstance *rln.RLN, r
gm.rootTracker = rootTracker

// add members to the Merkle tree
for i, member := range gm.group {
err := gm.insertMember(member, uint64(i+1))
if err != nil {
return err
}

err := gm.insertMembers(gm.group)
if err != nil {
return err
}

gm.group = nil // Deleting group to release memory

return nil
}

func (gm *StaticGroupManager) insertMember(pubkey rln.IDCommitment, index uint64) error {
gm.log.Debug("a new key is added", zap.Binary("pubkey", pubkey[:]), zap.Uint64("index", index))

// assuming all the members arrive in order
err := gm.rln.InsertMember(pubkey)
func (gm *StaticGroupManager) insertMembers(idCommitments []rln.IDCommitment) error {
err := gm.rln.InsertMembers(rln.MembershipIndex(gm.nextIndex), idCommitments)
if err != nil {
gm.log.Error("inserting member into merkletree", zap.Error(err))
gm.log.Error("inserting members into merkletree", zap.Error(err))
return err
}

_, err = gm.rootTracker.UpdateLatestRoot(index)
latestIndex := gm.nextIndex + uint64(len(idCommitments))

_, err = gm.rootTracker.UpdateLatestRoot(latestIndex)
if err != nil {
return err
}

gm.nextIndex = latestIndex + 1

return nil
}

Expand Down