From c320b38cbecd3b0a20ec7e19d3451a9ea6c5e489 Mon Sep 17 00:00:00 2001 From: Richard Ramos Date: Fri, 28 Jul 2023 13:25:33 -0400 Subject: [PATCH] feat(rln-relay): use atomic operations API --- flake.nix | 2 +- go.mod | 2 +- go.sum | 4 +- .../rln/group_manager/dynamic/dynamic.go | 43 +++++++++++-------- .../rln/group_manager/dynamic/handler_test.go | 8 ++-- .../rln/group_manager/static/static.go | 25 +++++------ 6 files changed, 47 insertions(+), 37 deletions(-) diff --git a/flake.nix b/flake.nix index 82f9e994..3df40994 100644 --- a/flake.nix +++ b/flake.nix @@ -28,7 +28,7 @@ ]; doCheck = false; # FIXME: This needs to be manually changed when updating modules. - vendorSha256 = "sha256-cMjGBz4/sDRcE55cfPBTl1lFkvHfZHeM4bTBcs5lO2s="; + vendorSha256 = "sha256-8khocBcz0klKwhoYKoPOvMSwFwa8rFcFSRM9jL8A0BU="; # Fix for 'nix run' trying to execute 'go-waku'. meta = { mainProgram = "waku"; }; }; diff --git a/go.mod b/go.mod index fa062948..9c9661d2 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/jackc/pgx/v5 v5.4.1 github.com/waku-org/go-libp2p-rendezvous v0.0.0-20230628220917-7b4e5ae4c0e7 github.com/waku-org/go-noise v0.0.4 - github.com/waku-org/go-zerokit-rln v0.1.13-0.20230803113701-ea89e5d7eedb + github.com/waku-org/go-zerokit-rln v0.1.13 github.com/wk8/go-ordered-map v1.0.0 ) diff --git a/go.sum b/go.sum index 4e3fc03f..81baa177 100644 --- a/go.sum +++ b/go.sum @@ -1554,8 +1554,8 @@ github.com/waku-org/go-libp2p-rendezvous v0.0.0-20230628220917-7b4e5ae4c0e7 h1:0 github.com/waku-org/go-libp2p-rendezvous v0.0.0-20230628220917-7b4e5ae4c0e7/go.mod h1:pFvOZ9YTFsW0o5zJW7a0B5tr1owAijRWJctXJ2toL04= github.com/waku-org/go-noise v0.0.4 h1:ZfQDcCw8pazm89EBl5SXY7GGAnzDQb9AHFXlw3Ktbvk= github.com/waku-org/go-noise v0.0.4/go.mod h1:+PWRfs2eSOVwKrPcQlfhwDngSh3faL/1QoxvoqggEKc= -github.com/waku-org/go-zerokit-rln v0.1.13-0.20230803113701-ea89e5d7eedb h1:f1UbFvroZbAul1HTlzM4ZGgjQ3mtn0bW8Jvk7XZQ5V4= -github.com/waku-org/go-zerokit-rln v0.1.13-0.20230803113701-ea89e5d7eedb/go.mod h1:1S6g1KXC45HkDXhIWD9+mdAs9fdyzQy8gtmw4RLjVcM= +github.com/waku-org/go-zerokit-rln v0.1.13 h1:9Ti/my7dmpJGNxNpL1MvVurzAjSd6xwBOm2WUZi33Mo= +github.com/waku-org/go-zerokit-rln v0.1.13/go.mod h1:1S6g1KXC45HkDXhIWD9+mdAs9fdyzQy8gtmw4RLjVcM= github.com/waku-org/go-zerokit-rln-apple v0.0.0-20230803113401-9a7ef94d120e h1:Ad0rJod5F1FuYCJ8SUB/bQZsQwirNHQRE0IcaVloxZo= github.com/waku-org/go-zerokit-rln-apple v0.0.0-20230803113401-9a7ef94d120e/go.mod h1:KYykqtdApHVYZ3G0spwMnoxc5jH5eI3jyO9SwsSfi48= github.com/waku-org/go-zerokit-rln-arm v0.0.0-20230801152407-8101ff87ee0a h1:10cre+P76QvnLeyeCVAM8WDbUCri/y5xY3LtwI9Y5DE= diff --git a/waku/v2/protocol/rln/group_manager/dynamic/dynamic.go b/waku/v2/protocol/rln/group_manager/dynamic/dynamic.go index 9a8c04b0..79a11470 100644 --- a/waku/v2/protocol/rln/group_manager/dynamic/dynamic.go +++ b/waku/v2/protocol/rln/group_manager/dynamic/dynamic.go @@ -66,12 +66,12 @@ func handler(gm *DynamicGroupManager, events []*contracts.RLNMemberRegistered) e toInsertTable := om.New() for _, event := range events { if event.Raw.Removed { - var indexes []uint64 + var indexes []uint i_idx, ok := toRemoveTable.Get(event.Raw.BlockNumber) if ok { - indexes = i_idx.([]uint64) + indexes = i_idx.([]uint) } - indexes = append(indexes, event.Index.Uint64()) + indexes = append(indexes, uint(event.Index.Uint64())) toRemoveTable.Set(event.Raw.BlockNumber, indexes) } else { var eventsPerBlock []*contracts.RLNMemberRegistered @@ -278,17 +278,28 @@ func (gm *DynamicGroupManager) persistCredentials() error { func (gm *DynamicGroupManager) InsertMembers(toInsert *om.OrderedMap) error { for pair := toInsert.Oldest(); pair != nil; pair = pair.Next() { events := pair.Value.([]*contracts.RLNMemberRegistered) // TODO: should these be sortered by index? we assume all members arrive in order + var idCommitments []rln.IDCommitment + var oldestIndexInBlock *big.Int for _, evt := range events { - pubkey := rln.Bytes32(evt.Pubkey.Bytes()) - // TODO: should we track indexes to identify missing? - err := gm.rln.InsertMember(pubkey) - if err != nil { - gm.log.Error("inserting member into merkletree", zap.Error(err)) - return err + if oldestIndexInBlock == nil { + oldestIndexInBlock = evt.Index } + idCommitments = append(idCommitments, rln.Bytes32(evt.Pubkey.Bytes())) } - _, err := gm.rootTracker.UpdateLatestRoot(pair.Key.(uint64)) + if len(idCommitments) == 0 { + continue + } + + // TODO: should we track indexes to identify missing? + startIndex := rln.MembershipIndex(uint(oldestIndexInBlock.Int64())) + err := gm.rln.InsertMembers(startIndex, idCommitments) + if err != nil { + gm.log.Error("inserting members into merkletree", zap.Error(err)) + return err + } + + _, err = gm.rootTracker.UpdateLatestRoot(pair.Key.(uint64)) if err != nil { return err } @@ -298,13 +309,11 @@ func (gm *DynamicGroupManager) InsertMembers(toInsert *om.OrderedMap) error { func (gm *DynamicGroupManager) RemoveMembers(toRemove *om.OrderedMap) error { for pair := toRemove.Newest(); pair != nil; pair = pair.Prev() { - memberIndexes := pair.Value.([]uint64) - for _, index := range memberIndexes { - err := gm.rln.DeleteMember(uint(index)) - if err != nil { - gm.log.Error("deleting member", zap.Error(err)) - return err - } + memberIndexes := pair.Value.([]uint) + err := gm.rln.DeleteMembers(memberIndexes) + if err != nil { + gm.log.Error("deleting members", zap.Error(err)) + return err } gm.rootTracker.Backfill(pair.Key.(uint64)) } diff --git a/waku/v2/protocol/rln/group_manager/dynamic/handler_test.go b/waku/v2/protocol/rln/group_manager/dynamic/handler_test.go index e32757a1..94df42a4 100644 --- a/waku/v2/protocol/rln/group_manager/dynamic/handler_test.go +++ b/waku/v2/protocol/rln/group_manager/dynamic/handler_test.go @@ -59,8 +59,8 @@ func TestHandler(t *testing.T) { roots = gm.rootTracker.Roots() require.Len(t, roots, 2) - require.Equal(t, roots[0], root0) - require.Equal(t, roots[1], [32]byte{253, 232, 31, 10, 168, 25, 42, 0, 28, 221, 146, 119, 34, 212, 121, 51, 82, 55, 113, 181, 236, 3, 11, 190, 194, 144, 125, 59, 46, 171, 90, 43}) + require.Equal(t, root0, roots[0]) + require.Equal(t, [32]byte{0x96, 0x65, 0xdb, 0xbc, 0x28, 0x6b, 0x1f, 0xc6, 0xab, 0x2d, 0x82, 0xe, 0x30, 0xe0, 0xc1, 0x92, 0x32, 0x73, 0xd2, 0xd6, 0xe1, 0x21, 0xef, 0xa1, 0xc8, 0xa6, 0xa6, 0xd, 0x9, 0x7c, 0x85, 0x19}, roots[1]) events = []*contracts.RLNMemberRegistered{ eventBuilder(1, false, 0xbbbb, 2), @@ -75,9 +75,9 @@ func TestHandler(t *testing.T) { // Root[1] should become [0] roots = gm.rootTracker.Roots() require.Len(t, roots, 5) - require.Equal(t, roots[0], [32]byte{253, 232, 31, 10, 168, 25, 42, 0, 28, 221, 146, 119, 34, 212, 121, 51, 82, 55, 113, 181, 236, 3, 11, 190, 194, 144, 125, 59, 46, 171, 90, 43}) + require.Equal(t, [32]byte{0x96, 0x65, 0xdb, 0xbc, 0x28, 0x6b, 0x1f, 0xc6, 0xab, 0x2d, 0x82, 0xe, 0x30, 0xe0, 0xc1, 0x92, 0x32, 0x73, 0xd2, 0xd6, 0xe1, 0x21, 0xef, 0xa1, 0xc8, 0xa6, 0xa6, 0xd, 0x9, 0x7c, 0x85, 0x19}, roots[0]) require.Len(t, rootTracker.Buffer(), 1) - require.Equal(t, rootTracker.Buffer()[0], [32]byte{62, 31, 25, 34, 223, 182, 113, 211, 249, 18, 247, 234, 70, 30, 10, 136, 238, 132, 143, 221, 225, 43, 108, 24, 171, 26, 210, 197, 106, 231, 52, 33}) + require.Equal(t, root0, rootTracker.Buffer()[0]) // We detect a fork // diff --git a/waku/v2/protocol/rln/group_manager/static/static.go b/waku/v2/protocol/rln/group_manager/static/static.go index 346e2dde..c54fbc11 100644 --- a/waku/v2/protocol/rln/group_manager/static/static.go +++ b/waku/v2/protocol/rln/group_manager/static/static.go @@ -18,6 +18,7 @@ type StaticGroupManager struct { group []rln.IDCommitment rootTracker *group_manager.MerkleRootTracker + nextIndex uint64 } func NewStaticGroupManager( @@ -46,11 +47,10 @@ func (gm *StaticGroupManager) Start(ctx context.Context, rlnInstance *rln.RLN, r gm.rootTracker = rootTracker // add members to the Merkle tree - for i, member := range gm.group { - err := gm.insertMember(member, uint64(i+1)) - if err != nil { - return err - } + + err := gm.insertMembers(gm.group) + if err != nil { + return err } gm.group = nil // Deleting group to release memory @@ -58,21 +58,22 @@ func (gm *StaticGroupManager) Start(ctx context.Context, rlnInstance *rln.RLN, r return nil } -func (gm *StaticGroupManager) insertMember(pubkey rln.IDCommitment, index uint64) error { - gm.log.Debug("a new key is added", zap.Binary("pubkey", pubkey[:]), zap.Uint64("index", index)) - - // assuming all the members arrive in order - err := gm.rln.InsertMember(pubkey) +func (gm *StaticGroupManager) insertMembers(idCommitments []rln.IDCommitment) error { + err := gm.rln.InsertMembers(rln.MembershipIndex(gm.nextIndex), idCommitments) if err != nil { - gm.log.Error("inserting member into merkletree", zap.Error(err)) + gm.log.Error("inserting members into merkletree", zap.Error(err)) return err } - _, err = gm.rootTracker.UpdateLatestRoot(index) + latestIndex := gm.nextIndex + uint64(len(idCommitments)) + + _, err = gm.rootTracker.UpdateLatestRoot(latestIndex) if err != nil { return err } + gm.nextIndex = latestIndex + 1 + return nil }