Bump nim-chornicles, nim-libp2p, nimcrypto

Bump nim-web3, nim-eth and deps - on_chain/group_manager.nim adaption
Added status-im/nim-minilru submodule required by latest nim-eth
Added nim-quic and nim-ngtcp2 as new deps of nim-libp2p
Fixing tests.

 Changes:
	modified:   .gitmodules
	modified:   tests/waku_discv5/utils.nim
	modified:   tests/waku_enr/utils.nim
	modified:   tests/waku_rln_relay/test_rln_group_manager_onchain.nim
	modified:   tests/waku_rln_relay/utils.nim
	modified:   tests/waku_rln_relay/utils_onchain.nim

        modified:   vendor/nim-chronicles
	modified:   vendor/nim-eth
	modified:   vendor/nim-http-utils
	modified:   vendor/nim-json-rpc
	modified:   vendor/nim-json-serialization
	modified:   vendor/nim-libp2p
	modified:   vendor/nim-metrics
	new file:   vendor/nim-minilru
	modified:   vendor/nim-nat-traversal
	new file:   vendor/nim-ngtcp2
	modified:   vendor/nim-presto
	new file:   vendor/nim-quic
	modified:   vendor/nim-secp256k1
	modified:   vendor/nim-serialization
	modified:   vendor/nim-stew
	modified:   vendor/nim-taskpools
	modified:   vendor/nim-testutils
	modified:   vendor/nim-toml-serialization
	modified:   vendor/nim-unicodedb
	modified:   vendor/nim-unittest2
	modified:   vendor/nim-web3
	modified:   vendor/nim-websock
	modified:   vendor/nim-zlib
	modified:   vendor/nimcrypto
	modified:   waku.nimble

        modified:   waku/common/enr/builder.nim
	modified:   waku/common/enr/typed_record.nim
	modified:   waku/common/utils/nat.nim
	modified:   waku/discovery/waku_discv5.nim
	modified:   waku/waku_rln_relay/conversion_utils.nim
	modified:   waku/waku_rln_relay/group_manager/on_chain/group_manager.nim
	modified:   waku/waku_rln_relay/rln/wrappers.nim
	modified:   waku/waku_rln_relay/rln_relay.nim
This commit is contained in:
NagyZoltanPeter 2024-10-03 13:10:48 +02:00
parent 3ad613cad4
commit 7f1ea0890a
No known key found for this signature in database
GPG Key ID: 16EADB9673B65368
39 changed files with 142 additions and 81 deletions

15
.gitmodules vendored
View File

@ -179,3 +179,18 @@
branch = master
path = vendor/nph
url = https://github.com/arnetheduck/nph.git
[submodule "vendor/nim-minilru"]
path = vendor/nim-minilru
url = https://github.com/status-im/nim-minilru.git
ignore = untracked
branch = master
[submodule "vendor/nim-quic"]
path = vendor/nim-quic
url = https://github.com/status-im/nim-quic.git
ignore = untracked
branch = master
[submodule "vendor/nim-ngtcp2"]
path = vendor/nim-ngtcp2
url = https://github.com/vacp2p/nim-ngtcp2.git
ignore = untracked
branch = master

View File

@ -1,4 +1,5 @@
import
std/options,
stew/results,
stew/shims/net,
chronos,

View File

@ -1,4 +1,5 @@
import
std/options,
sequtils,
stew/results,
stew/shims/net,

View File

@ -42,14 +42,14 @@ suite "Onchain group manager":
manager.ethRpc.isSome()
manager.wakuRlnContract.isSome()
manager.initialized
manager.rlnContractDeployedBlockNumber > 0
manager.rlnContractDeployedBlockNumber > 0.Quantity
manager.rlnRelayMaxMessageLimit == 100
await manager.stop()
asyncTest "should error on initialization when chainId does not match":
let manager = await setup()
manager.chainId = CHAIN_ID + 1
manager.chainId = utils_onchain.CHAIN_ID + 1
(await manager.init()).isErrOr:
raiseAssert "Expected error when chainId does not match"

View File

@ -1,4 +1,4 @@
import web3, chronos, options, stint
import web3, chronos, options, stint, stew/byteutils
proc deployContract*(
web3: Web3, code: string, gasPrice = 0, contractInput = ""
@ -20,12 +20,13 @@ proc deployContract*(
var code = code
if code[1] notin {'x', 'X'}:
code = "0x" & code
var tr: EthSend
tr.source = web3.defaultAccount
tr.data = code & contractInput
tr.gas = Quantity(3000000000000).some
var tr: TransactionArgs
tr.`from` = Opt.some(web3.defaultAccount)
let sData = code & contractInput
tr.data = Opt.some(sData.toBytes())
tr.gas = Opt.some(Quantity(3000000000000))
if gasPrice != 0:
tr.gasPrice = some(gasPrice)
tr.gasPrice = Opt.some(gasPrice.Quantity)
let r = await web3.send(tr)
return await web3.getMinedTransactionReceipt(r)

View File

@ -10,6 +10,8 @@ import
chronicles,
stint,
web3,
web3/conversions,
web3/eth_api_types,
json,
libp2p/crypto/crypto,
eth/keys
@ -27,6 +29,18 @@ import
const CHAIN_ID* = 1337
template skip0xPrefix(hexStr: string): int =
## Returns the index of the first meaningful char in `hexStr` by skipping
## "0x" prefix
if hexStr.len > 1 and hexStr[0] == '0' and hexStr[1] in {'x', 'X'}: 2 else: 0
func strip0xPrefix(s: string): string =
let prefixLen = skip0xPrefix(s)
if prefixLen != 0:
s[prefixLen .. ^1]
else:
s
proc generateCredentials*(rlnInstance: ptr RLN): IdentityCredential =
let credRes = membershipKeyGen(rlnInstance)
return credRes.get()
@ -85,12 +99,13 @@ proc uploadRLNContract*(ethClientAddress: string): Future[Address] {.async.} =
debug "Address of the deployed rlnv2 contract: ", wakuRlnContractAddress
# need to send concat: impl & init_bytes
let contractInput = encode(wakuRlnContractAddress).data & Erc1967ProxyContractInput
let contractInput =
byteutils.toHex(encode(wakuRlnContractAddress)) & Erc1967ProxyContractInput
debug "contractInput", contractInput
let proxyReceipt =
await web3.deployContract(Erc1967Proxy, contractInput = contractInput)
debug "proxy receipt", proxyReceipt
debug "proxy receipt", contractAddress = proxyReceipt.contractAddress.get()
let proxyAddress = proxyReceipt.contractAddress.get()
let newBalance = await web3.provider.eth_getBalance(web3.defaultAccount, "latest")
@ -106,17 +121,17 @@ proc createEthAccount*(
): Future[(keys.PrivateKey, Address)] {.async.} =
let web3 = await newWeb3(EthClient)
let accounts = await web3.provider.eth_accounts()
let gasPrice = int(await web3.provider.eth_gasPrice())
let gasPrice = Quantity(await web3.provider.eth_gasPrice())
web3.defaultAccount = accounts[0]
let pk = keys.PrivateKey.random(rng[])
let acc = Address(toCanonicalAddress(pk.toPublicKey()))
var tx: EthSend
tx.source = accounts[0]
tx.value = some(ethToWei(ethAmount))
tx.to = some(acc)
tx.gasPrice = some(gasPrice)
var tx: TransactionArgs
tx.`from` = Opt.some(accounts[0])
tx.value = Opt.some(ethToWei(ethAmount))
tx.to = Opt.some(acc)
tx.gasPrice = Opt.some(gasPrice)
# Send ethAmount to acc
discard await web3.send(tx)

@ -1 +1 @@
Subproject commit a28bb9781ce74e725796c307ad05083e646872be
Subproject commit 4524912fcacfd3965e32a9fe23c9bb12d48177ff

2
vendor/nim-eth vendored

@ -1 +1 @@
Subproject commit bb5cb6a4d0dbb592b395ab41cda3fd3f1fdc443e
Subproject commit 84664b0fc0b9128b4316afb85274271389aab00f

@ -1 +1 @@
Subproject commit 8b88ad6dd9a6326c29f82067800c483d9410d873
Subproject commit e8fc71aee15203a852f6321e4dd8d87517502847

2
vendor/nim-json-rpc vendored

@ -1 +1 @@
Subproject commit 33360528d208aaa70a6ed914619fe76321760d54
Subproject commit be16a6528acfcfdaab381c4900240e454f7cdc63

@ -1 +1 @@
Subproject commit 8a4ed98bbd0a9479df15af2fa31da38a586ea6d5
Subproject commit ab1a061756bb6fc2e0f98cb57852f2bb0c6f9772

2
vendor/nim-libp2p vendored

@ -1 +1 @@
Subproject commit 18a2e79ce209d2f21d8c5db7a41659cc39dbe0b4
Subproject commit 09fe199b6bee6c3d0cc0e8750e67bbc0d52da138

2
vendor/nim-metrics vendored

@ -1 +1 @@
Subproject commit 29bb7ba63cd884770169891687595348a70cf166
Subproject commit f1f886957831e9f2f16ffa728abb4bf44bfd5b98

1
vendor/nim-minilru vendored Submodule

@ -0,0 +1 @@
Subproject commit 2682cffa8733f3b61751c65a963941315e887bac

@ -1 +1 @@
Subproject commit a3aa0c5f9d2a2870f1fd0f7a613d4fe025c84ab7
Subproject commit 05e76accbc7811273fadc23d8886ac1a2f83bb81

1
vendor/nim-ngtcp2 vendored Submodule

@ -0,0 +1 @@
Subproject commit 6834f4756b6af58356ac9c4fef3d71db3c3ae5fe

2
vendor/nim-presto vendored

@ -1 +1 @@
Subproject commit 2190421e09938696cd95d54b1f4753446c84c7a2
Subproject commit 5d5dc51bac4aafb26c03d2d813a47e80093bd0c7

1
vendor/nim-quic vendored Submodule

@ -0,0 +1 @@
Subproject commit ddcb31ffb74b5460ab37fd13547eca90594248bc

@ -1 +1 @@
Subproject commit 4470f49bcd6bcbfb59f0eeb67315ca9ddac0bdc0
Subproject commit 641902d492aff3910b0240217fa0b28b5471baad

@ -1 +1 @@
Subproject commit 298a9554a885b2df59737bb3461aac8d0d339724
Subproject commit 7950ac1e8c9f0019c798f97ab1fe1d9367cc2e6d

2
vendor/nim-stew vendored

@ -1 +1 @@
Subproject commit d4634c5405ac188e7050d348332edb6c3b09a527
Subproject commit b7b596955726a65788561d2d69978223d4484c95

@ -1 +1 @@
Subproject commit d4c43137c0590cb47f893a66ca8cb027fa6c217e
Subproject commit 66585e2e960b7695e48ea60377fb3aeac96406e8

@ -1 +1 @@
Subproject commit ae476c67314ac4b294d21040315e0f716189a70e
Subproject commit c36724c469b657435f40fb7d365ad14640341606

@ -1 +1 @@
Subproject commit cb1fc73f3519fed5f3a8fbfa90afc9a96d5f5f5c
Subproject commit 53ae081cc9a3e61095cab49862658a46b00eacf7

@ -1 +1 @@
Subproject commit 8c8959d84c12ecda6ea14c67bd68675b1936f8cf
Subproject commit 15c5e25e2a49a924bc97647481ff50125bba2c76

@ -1 +1 @@
Subproject commit e96f3215030cbfa13abc2f5827069b6f8ba87e38
Subproject commit 11f7cff9280933067fba1c736eac9d05b9bfca79

2
vendor/nim-web3 vendored

@ -1 +1 @@
Subproject commit 428b931e7c4f1284b4272bc2c11fca2bd70991cd
Subproject commit 62a0005b0907a64090827d4e5d691682587f5b2a

2
vendor/nim-websock vendored

@ -1 +1 @@
Subproject commit 63bcc2902d884c63101e144555ad99421734a70a
Subproject commit 179f81dedaddb5ba8d02534ccc8b7a8335981f49

2
vendor/nim-zlib vendored

@ -1 +1 @@
Subproject commit 45b06fca15ce0f09586067d950da30c10227865a
Subproject commit 02311a35623964a3ef37da8cd896ed95be06e6da

2
vendor/nimcrypto vendored

@ -1 +1 @@
Subproject commit 71bca15508e2c0548f32b42a69bcfb1ccd9ab9ff
Subproject commit a0b65f2a0dba987c75eadf03f811fb5508fd3372

View File

@ -23,7 +23,9 @@ requires "nim >= 2.0.8",
"web3",
"presto",
"regex",
"db_connector"
"db_connector",
"minilru",
"quic"
### Helper functions
proc buildModule(filePath, params = "", lang = "c"): bool =

View File

@ -41,9 +41,9 @@ proc build*(builder: EnrBuilder): EnrResult[enr.Record] =
enr.Record.init(
seqNum = builder.seqNumber,
pk = builder.privateKey,
ip = none(IpAddress),
tcpPort = none(Port),
udpPort = none(Port),
ip = Opt.none(IpAddress),
tcpPort = Opt.none(Port),
udpPort = Opt.none(Port),
extraFields = builder.fields,
)

View File

@ -4,6 +4,19 @@ import std/options, results, eth/keys as eth_keys, libp2p/crypto/crypto as libp2
import eth/p2p/discoveryv5/enr except TypedRecord, toTypedRecord
## Since enr changed to result.Opt[T] from Option[T] for intercompatibility introduce a conversion between
func toOpt*[T](o: Option[T]): Opt[T] =
if o.isSome():
return Opt.some(o.get())
else:
return Opt.none(T)
func toOption*[T](o: Opt[T]): Option[T] =
if o.isSome():
return some(o.get())
else:
return none(T)
## ENR typed record
# Record identity scheme
@ -31,7 +44,7 @@ proc init(T: type TypedRecord, record: Record): T =
TypedRecord(raw: record)
proc tryGet*(record: TypedRecord, field: string, T: type): Option[T] =
record.raw.tryGet(field, T)
return record.raw.tryGet(field, T).toOption()
func toTyped*(record: Record): EnrResult[TypedRecord] =
let tr = TypedRecord.init(record)

View File

@ -44,7 +44,7 @@ proc setupNat*(
endpoint.ip = some(extIp.get())
# RedirectPorts in considered a gcsafety violation
# because it obtains the address of a non-gcsafe proc?
var extPorts: Option[(Port, Port)]
var extPorts: Opt[(Port, Port)]
try:
extPorts = (
{.gcsafe.}:
@ -55,7 +55,7 @@ proc setupNat*(
except CatchableError:
# TODO: nat.nim Error: can raise an unlisted exception: Exception. Isolate here for now.
error "unable to determine external ports"
extPorts = none((Port, Port))
extPorts = Opt.none((Port, Port))
if extPorts.isSome():
let (extTcpPort, extUdpPort) = extPorts.get()

View File

@ -87,10 +87,10 @@ proc new*(
privKey = conf.privateKey,
bootstrapRecords = conf.bootstrapRecords,
enrAutoUpdate = conf.autoupdateRecord,
previousRecord = record,
enrIp = none(IpAddress),
enrTcpPort = none(Port),
enrUdpPort = none(Port),
previousRecord = record.toOpt(),
enrIp = Opt.none(IpAddress),
enrTcpPort = Opt.none(Port),
enrUdpPort = Opt.none(Port),
)
let shardPredOp =

View File

@ -128,3 +128,8 @@ proc fromEpoch*(epoch: Epoch): uint64 =
## decodes bytes of `epoch` (in little-endian) to uint64
let t = fromBytesLE(uint64, array[32, byte](epoch))
return t
func `+`*(a, b: Quantity): Quantity {.borrow.}
func u256*(n: Quantity): UInt256 {.inline.} =
n.uint64.stuint(256)

View File

@ -3,10 +3,11 @@
import
os,
web3,
web3/ethtypes,
web3/eth_api_types,
web3/primitives,
eth/keys as keys,
chronicles,
nimcrypto/keccak,
nimcrypto/keccak as keccak,
stint,
json,
std/tables,
@ -30,11 +31,11 @@ logScope:
# using the when predicate does not work within the contract macro, hence need to dupe
contract(WakuRlnContract):
# this serves as an entrypoint into the rln membership set
proc register(idCommitment: UInt256, userMessageLimit: UInt32)
proc register(idCommitment: UInt256, userMessageLimit: StUint[32])
# Initializes the implementation contract (only used in unit tests)
proc initialize(maxMessageLimit: UInt256)
# this event is raised when a new member is registered
proc MemberRegistered(rateCommitment: UInt256, index: Uint32) {.event.}
proc MemberRegistered(rateCommitment: UInt256, index: StUint[32]) {.event.}
# this function denotes existence of a given user
proc memberExists(idCommitment: Uint256): UInt256 {.view.}
@ -100,7 +101,7 @@ proc setMetadata*(
try:
let metadataSetRes = g.rlnInstance.setMetadata(
RlnMetadata(
lastProcessedBlock: normalizedBlock,
lastProcessedBlock: normalizedBlock.uint64,
chainId: g.chainId,
contractAddress: g.ethContractAddress,
validRoots: g.validRoots.toSeq(),
@ -190,11 +191,11 @@ method register*(
g.registrationTxHash = some(txHash)
# the receipt topic holds the hash of signature of the raised events
# TODO: make this robust. search within the event list for the event
debug "ts receipt", tsReceipt
# debug "ts receipt", tsReceipt
let firstTopic = tsReceipt.logs[0].topics[0]
# the hash of the signature of MemberRegistered(uint256,uint32) event is equal to the following hex value
if firstTopic !=
cast[FixedBytes[32]](keccak256.digest("MemberRegistered(uint256,uint32)").data):
cast[FixedBytes[32]](keccak.keccak256.digest("MemberRegistered(uint256,uint32)").data):
raise newException(ValueError, "unexpected event signature")
# the arguments of the raised event i.e., MemberRegistered are encoded inside the data field
@ -231,11 +232,10 @@ proc parseEvent(
## returns an error if it cannot parse the `data` parameter
var rateCommitment: UInt256
var index: UInt256
var data: string
# Remove the 0x prefix
var data: seq[byte]
try:
data = strip0xPrefix(log["data"].getStr())
except CatchableError:
data = hexToSeqByte(log["data"].getStr())
except ValueError:
return err(
"failed to parse the data field of the MemberRegistered event: " &
getCurrentExceptionMsg()
@ -243,9 +243,9 @@ proc parseEvent(
var offset = 0
try:
# Parse the rateCommitment
offset += decode(data, offset, rateCommitment)
offset += decode(data, 0, offset, rateCommitment)
# Parse the index
offset += decode(data, offset, index)
offset += decode(data, 0, offset, index)
return ok(
Membership(
rateCommitment: rateCommitment.toRateCommitment(),
@ -291,13 +291,18 @@ proc getRawEvents(
let ethRpc = g.ethRpc.get()
let wakuRlnContract = g.wakuRlnContract.get()
var events: JsonNode
g.retryWrapper(events, "Failed to get the events"):
var eventStrs: seq[JsonString]
g.retryWrapper(eventStrs, "Failed to get the events"):
await wakuRlnContract.getJsonLogs(
MemberRegistered,
fromBlock = some(fromBlock.blockId()),
toBlock = some(toBlock.blockId()),
fromBlock = Opt.some(fromBlock.blockId()),
toBlock = Opt.some(toBlock.blockId()),
)
var events: JsonNode
if events.len == 0:
for eventStr in eventStrs:
events.add(parseJson($eventStr))
return events
proc getBlockTable(
@ -314,7 +319,7 @@ proc getBlockTable(
return blockTable
for event in events:
let blockNumber = parseHexInt(event["blockNumber"].getStr()).uint
let blockNumber = parseHexInt(event["blockNumber"].getStr()).Quantity
let removed = event["removed"].getBool()
let parsedEventRes = parseEvent(MemberRegistered, event)
if parsedEventRes.isErr():
@ -450,13 +455,13 @@ proc startOnchainSync(
let ethRpc = g.ethRpc.get()
# static block chunk size
let blockChunkSize = 2_000
let blockChunkSize = 2_000.BlockNumber
# delay between rpc calls to not overload the rate limit
let rpcDelay = 200.milliseconds
# max number of futures to run concurrently
let maxFutures = 10
var fromBlock =
var fromBlock: BlockNumber =
if g.latestProcessedBlock > g.rlnContractDeployedBlockNumber:
info "syncing from last processed block", blockNumber = g.latestProcessedBlock
g.latestProcessedBlock + 1
@ -479,11 +484,11 @@ proc startOnchainSync(
if fromBlock >= currentLatestBlock:
break
if fromBlock + blockChunkSize.uint > currentLatestBlock.uint:
if fromBlock + blockChunkSize > currentLatestBlock:
g.retryWrapper(currentLatestBlock, "Failed to get the latest block number"):
cast[BlockNumber](await ethRpc.provider.eth_blockNumber())
let toBlock = min(fromBlock + BlockNumber(blockChunkSize), currentLatestBlock)
let toBlock = min(fromBlock + blockChunkSize, currentLatestBlock)
debug "fetching events", fromBlock = fromBlock, toBlock = toBlock
await sleepAsync(rpcDelay)
futs.add(g.getAndHandleEvents(fromBlock, toBlock))
@ -551,7 +556,7 @@ method init*(g: OnchainGroupManager): Future[GroupManagerResult[void]] {.async.}
let pk = g.ethPrivateKey.get()
let parsedPk = keys.PrivateKey.fromHex(pk).valueOr:
return err("failed to parse the private key" & ": " & $error)
ethRpc.privateKey = some(parsedPk)
ethRpc.privateKey = Opt.some(parsedPk)
ethRpc.defaultAccount =
ethRpc.privateKey.get().toPublicKey().toCanonicalAddress().Address
@ -605,7 +610,7 @@ method init*(g: OnchainGroupManager): Future[GroupManagerResult[void]] {.async.}
if metadata.contractAddress != g.ethContractAddress.toLower():
return err("persisted data: contract address mismatch")
g.latestProcessedBlock = metadata.lastProcessedBlock
g.latestProcessedBlock = metadata.lastProcessedBlock.BlockNumber
g.validRoots = metadata.validRoots.toDeque()
var deployedBlockNumber: Uint256
@ -661,10 +666,10 @@ method stop*(g: OnchainGroupManager): Future[void] {.async, gcsafe.} =
proc isSyncing*(g: OnchainGroupManager): Future[bool] {.async, gcsafe.} =
let ethRpc = g.ethRpc.get()
var syncing: JsonNode
var syncing: SyncingStatus
g.retryWrapper(syncing, "Failed to get the syncing status"):
await ethRpc.provider.eth_syncing()
return syncing.getBool()
return syncing.syncing
method isReady*(g: OnchainGroupManager): Future[bool] {.async.} =
initializedGuard(g)
@ -677,7 +682,7 @@ method isReady*(g: OnchainGroupManager): Future[bool] {.async.} =
cast[BlockNumber](await g.ethRpc.get().provider.eth_blockNumber())
# the node is still able to process messages if it is behind the latest block by a factor of the valid roots
if u256(g.latestProcessedBlock) < (u256(currentBlock) - u256(g.validRoots.len)):
if u256(g.latestProcessedBlock.uint64) < (u256(currentBlock) - u256(g.validRoots.len)):
return false
return not (await g.isSyncing())

View File

@ -8,7 +8,7 @@ import
stint,
web3,
json,
web3/ethtypes,
web3/eth_api_types,
eth/keys,
libp2p/protocols/pubsub/rpc/messages,
libp2p/protocols/pubsub/pubsub,