From 0eaf90465fe65954b42830c2d906d69a4d18930c Mon Sep 17 00:00:00 2001 From: fryorcraken <110212804+fryorcraken@users.noreply.github.com> Date: Tue, 1 Apr 2025 09:28:18 +1100 Subject: [PATCH] feat: introduce `preset` option (#3346) * feat: introduce `preset` option Overwriting config from cluster-id will be deprecated as a second step. * doc: improve preset doc * Change `default` preset to `twn` --- tests/all_tests_waku.nim | 3 +- tests/factory/test_config.nim | 157 +++++++++++++++++++++++++++++++ waku/factory/external_config.nim | 20 +++- waku/factory/internal_config.nim | 73 +++++++++++++- waku/factory/networks_config.nim | 1 + waku/factory/node_factory.nim | 8 +- waku/factory/waku.nim | 58 +++--------- 7 files changed, 261 insertions(+), 59 deletions(-) create mode 100644 tests/factory/test_config.nim diff --git a/tests/all_tests_waku.nim b/tests/all_tests_waku.nim index 39ac57caf..3e847ae86 100644 --- a/tests/all_tests_waku.nim +++ b/tests/all_tests_waku.nim @@ -30,7 +30,7 @@ import const os* {.strdefine.} = "" when os == "Linux" and # GitHub only supports container actions on Linux - # and we need to start a postgress database in a docker container + # and we need to start a postgres database in a docker container defined(postgres): import ./waku_archive/test_driver_postgres_query, @@ -106,3 +106,4 @@ import import ./waku_rln_relay/test_all # Node Factory +import ./factory/test_config diff --git a/tests/factory/test_config.nim b/tests/factory/test_config.nim new file mode 100644 index 000000000..4a1f6d78f --- /dev/null +++ b/tests/factory/test_config.nim @@ -0,0 +1,157 @@ +{.used.} + +import + std/options, + testutils/unittests, + chronos, + libp2p/crypto/[crypto, secp], + libp2p/multiaddress, + nimcrypto/utils, + secp256k1, + confutils +import + ../../waku/factory/external_config, + ../../waku/factory/internal_config, + ../../waku/factory/networks_config, + ../../waku/common/logging + +suite "Waku config - apply preset": + test "Default preset is TWN": + ## Setup + let expectedConf = ClusterConf.TheWakuNetworkConf() + + ## Given + let preConfig = WakuNodeConf(cmd: noCommand, preset: "default") + + ## When + let res = applyPresetConfiguration(preConfig) + assert res.isOk(), $res.error + + ## Then + let conf = res.get() + assert conf.maxMessageSize == expectedConf.maxMessageSize + assert conf.clusterId == expectedConf.clusterId + assert conf.rlnRelay == expectedConf.rlnRelay + assert conf.rlnRelayEthContractAddress == expectedConf.rlnRelayEthContractAddress + assert conf.rlnRelayDynamic == expectedConf.rlnRelayDynamic + assert conf.rlnRelayChainId == expectedConf.rlnRelayChainId + assert conf.rlnRelayBandwidthThreshold == expectedConf.rlnRelayBandwidthThreshold + assert conf.rlnEpochSizeSec == expectedConf.rlnEpochSizeSec + assert conf.rlnRelayUserMessageLimit == expectedConf.rlnRelayUserMessageLimit + assert conf.numShardsInNetwork == expectedConf.numShardsInNetwork + assert conf.discv5BootstrapNodes == expectedConf.discv5BootstrapNodes + + test "Subscribes to all valid shards in default network": + ## Setup + let expectedConf = ClusterConf.TheWakuNetworkConf() + + ## Given + let shards: seq[uint16] = @[0, 1, 2, 3, 4, 5, 6, 7] + let preConfig = WakuNodeConf(cmd: noCommand, preset: "default", shards: shards) + + ## When + let res = applyPresetConfiguration(preConfig) + assert res.isOk(), $res.error + + ## Then + let conf = res.get() + assert conf.shards.len == expectedConf.numShardsInNetwork.int + + test "Subscribes to some valid shards in default network": + ## Setup + let expectedConf = ClusterConf.TheWakuNetworkConf() + + ## Given + let shards: seq[uint16] = @[0, 4, 7] + let preConfig = WakuNodeConf(cmd: noCommand, preset: "default", shards: shards) + + ## When + let resConf = applyPresetConfiguration(preConfig) + let res = validateShards(resConf.get()) + assert res.isOk(), $res.error + + ## Then + let conf = resConf.get() + assert conf.shards.len() == shards.len() + for index, shard in shards: + assert shard in conf.shards + + test "Subscribes to invalid shards in default network": + ## Setup + + ## Given + let shards: seq[uint16] = @[0, 4, 7, 10] + let preConfig = WakuNodeConf(cmd: noCommand, preset: "default", shards: shards) + let postConfig = applyPresetConfiguration(preConfig) + + ## When + let res = validateShards(postConfig.get()) + + ## Then + assert res.isErr(), "Invalid shard was accepted" + +suite "Waku config - node key": + test "Passed node key is used": + ## Setup + let nodeKeyStr = + "0011223344556677889900aabbccddeeff0011223344556677889900aabbccddeeff" + let nodekey = block: + let key = SkPrivateKey.init(utils.fromHex(nodeKeyStr)).tryGet() + crypto.PrivateKey(scheme: Secp256k1, skkey: key) + + ## Given + let config = WakuNodeConf.load(version = "", cmdLine = @["--nodekey=" & nodeKeyStr]) + + ## When + let res = getNodeKey(config) + assert res.isOk(), $res.error + + ## Then + let resKey = res.get() + assert utils.toHex(resKey.getRawBytes().get()) == + utils.toHex(nodekey.getRawBytes().get()) + +suite "Waku config - Shards": + test "Shards are valid": + ## Setup + + ## Given + let shards: seq[uint16] = @[0, 2, 4] + let numShardsInNetwork = 5.uint32 + let config = WakuNodeConf( + cmd: noCommand, shards: shards, numShardsInNetwork: numShardsInNetwork + ) + + ## When + let res = validateShards(config) + + ## Then + assert res.isOk(), $res.error + + test "Shards are not in range": + ## Setup + + ## Given + let shards: seq[uint16] = @[0, 2, 5] + let numShardsInNetwork = 5.uint32 + let config = WakuNodeConf( + cmd: noCommand, shards: shards, numShardsInNetwork: numShardsInNetwork + ) + + ## When + let res = validateShards(config) + + ## Then + assert res.isErr(), "Invalid shard was accepted" + + test "Shard is passed without num shards": + ## Setup + + ## Given + let config = WakuNodeConf.load(version = "", cmdLine = @["--shard=32"]) + + ## When + let res = validateShards(config) + + ## Then + assert res.isOk(), $res.error diff --git a/waku/factory/external_config.nim b/waku/factory/external_config.nim index 9bc073426..100d1b644 100644 --- a/waku/factory/external_config.nim +++ b/waku/factory/external_config.nim @@ -76,7 +76,7 @@ type WakuNodeConf* = object .}: EthRpcUrl rlnRelayEthContractAddress* {. - desc: "Address of membership contract on an Ethereum testnet", + desc: "Address of membership contract on an Ethereum testnet.", defaultValue: "", name: "rln-relay-eth-contract-address" .}: string @@ -100,6 +100,7 @@ type WakuNodeConf* = object name: "rln-relay-eth-private-key" .}: string + # TODO: Remove "Default is" when it's already visible on the CLI rlnRelayUserMessageLimit* {. desc: "Set a user message limit for the rln membership registration. Must be a positive integer. Default is 1.", @@ -145,6 +146,13 @@ type WakuNodeConf* = object .}: seq[ProtectedShard] ## General node config + preset* {. + desc: + "Network preset to use. 'twn' is The RLN-protected Waku Network (cluster 1).", + defaultValue: "", + name: "preset" + .}: string + clusterId* {. desc: "Cluster id that the node is running in. Node in a different cluster id is disconnected.", @@ -276,7 +284,7 @@ hence would have reachability issues.""", .}: bool rlnRelay* {. - desc: "Enable spam protection through rln-relay: true|false", + desc: "Enable spam protection through rln-relay: true|false.", defaultValue: false, name: "rln-relay" .}: bool @@ -287,7 +295,7 @@ hence would have reachability issues.""", .}: Option[uint] rlnRelayDynamic* {. - desc: "Enable waku-rln-relay with on-chain dynamic group management: true|false", + desc: "Enable waku-rln-relay with on-chain dynamic group management: true|false.", defaultValue: false, name: "rln-relay-dynamic" .}: bool @@ -311,7 +319,8 @@ hence would have reachability issues.""", .}: string rlnRelayBandwidthThreshold* {. - desc: "Message rate in bytes/sec after which verification of proofs should happen", + desc: + "Message rate in bytes/sec after which verification of proofs should happen.", defaultValue: 0, # to maintain backwards compatibility name: "rln-relay-bandwidth-threshold" .}: int @@ -327,6 +336,7 @@ hence would have reachability issues.""", name: "keep-alive" .}: bool + # TODO: This is trying to do too much, this should only be used for autosharding, which itself should be configurable # If numShardsInNetwork is not set, we use the number of shards configured as numShardsInNetwork numShardsInNetwork* {. desc: "Number of shards in the network", @@ -590,7 +600,7 @@ with the drawback of consuming some more bandwidth.""", ## Discovery v5 config discv5Discovery* {. - desc: "Enable discovering nodes via Node Discovery v5", + desc: "Enable discovering nodes via Node Discovery v5.", defaultValue: false, name: "discv5-discovery" .}: bool diff --git a/waku/factory/internal_config.nim b/waku/factory/internal_config.nim index 064a03acd..08f11f1c5 100644 --- a/waku/factory/internal_config.nim +++ b/waku/factory/internal_config.nim @@ -4,7 +4,7 @@ import libp2p/crypto/crypto, libp2p/multiaddress, libp2p/nameresolving/dnsresolver, - std/[options, sequtils, net], + std/[options, sequtils, strutils, net], results import ./external_config, @@ -12,7 +12,8 @@ import ../node/config, ../waku_enr/capabilities, ../waku_enr, - ../waku_core + ../waku_core, + ./networks_config proc enrConfiguration*( conf: WakuNodeConf, netConfig: NetConfig, key: crypto.PrivateKey @@ -157,3 +158,71 @@ proc networkConfiguration*(conf: WakuNodeConf, clientId: string): NetConfigResul ) return netConfigRes + +proc applyPresetConfiguration*(srcConf: WakuNodeConf): Result[WakuNodeConf, string] = + var resConf = srcConf + + if resConf.clusterId == 1: + warn( + "TWN - The Waku Network configuration will not be applied when `--cluster-id=1` is passed in future releases. Use `--preset=twn` instead." + ) + resConf.preset = "twn" + + case toLowerAscii(resConf.preset) + of "twn": + let twnClusterConf = ClusterConf.TheWakuNetworkConf() + + # Override configuration + resConf.maxMessageSize = twnClusterConf.maxMessageSize + resConf.clusterId = twnClusterConf.clusterId + resConf.rlnRelay = twnClusterConf.rlnRelay + resConf.rlnRelayEthContractAddress = twnClusterConf.rlnRelayEthContractAddress + resConf.rlnRelayChainId = twnClusterConf.rlnRelayChainId + resConf.rlnRelayDynamic = twnClusterConf.rlnRelayDynamic + resConf.rlnRelayBandwidthThreshold = twnClusterConf.rlnRelayBandwidthThreshold + resConf.discv5Discovery = twnClusterConf.discv5Discovery + resConf.discv5BootstrapNodes = + resConf.discv5BootstrapNodes & twnClusterConf.discv5BootstrapNodes + resConf.rlnEpochSizeSec = twnClusterConf.rlnEpochSizeSec + resConf.rlnRelayUserMessageLimit = twnClusterConf.rlnRelayUserMessageLimit + resConf.numShardsInNetwork = twnClusterConf.numShardsInNetwork + + if resConf.relay: + resConf.rlnRelay = twnClusterConf.rlnRelay + else: + discard + + return ok(resConf) + +# TODO: numShardsInNetwork should be mandatory with autosharding, and unneeded otherwise +proc getNumShardsInNetwork*(conf: WakuNodeConf): uint32 = + if conf.numShardsInNetwork != 0: + return conf.numShardsInNetwork + # If conf.numShardsInNetwork is not set, use 1024 - the maximum possible as per the static sharding spec + # https://github.com/waku-org/specs/blob/master/standards/core/relay-sharding.md#static-sharding + return uint32(MaxShardIndex + 1) + +proc validateShards*(conf: WakuNodeConf): Result[void, string] = + let numShardsInNetwork = getNumShardsInNetwork(conf) + + for shard in conf.shards: + if shard >= numShardsInNetwork: + let msg = + "validateShards invalid shard: " & $shard & " when numShardsInNetwork: " & + $numShardsInNetwork # fmt doesn't work + error "validateShards failed", error = msg + return err(msg) + + return ok() + +proc getNodeKey*( + conf: WakuNodeConf, rng: ref HmacDrbgContext = crypto.newRng() +): Result[PrivateKey, string] = + if conf.nodekey.isSome(): + return ok(conf.nodekey.get()) + + warn "missing node key, generating new set" + let key = crypto.PrivateKey.random(Secp256k1, rng[]).valueOr: + error "Failed to generate key", error = error + return err("Failed to generate key: " & $error) + return ok(key) diff --git a/waku/factory/networks_config.nim b/waku/factory/networks_config.nim index 41678f590..8dfefbf34 100644 --- a/waku/factory/networks_config.nim +++ b/waku/factory/networks_config.nim @@ -30,6 +30,7 @@ proc TheWakuNetworkConf*(T: type ClusterConf): ClusterConf = rlnRelayUserMessageLimit: 100, numShardsInNetwork: 8, discv5Discovery: true, + # TODO: Why is this part of the conf? eg an edge node would not have this discv5BootstrapNodes: @[ "enr:-QESuED0qW1BCmF-oH_ARGPr97Nv767bl_43uoy70vrbah3EaCAdK3Q0iRQ6wkSTTpdrg_dU_NC2ydO8leSlRpBX4pxiAYJpZIJ2NIJpcIRA4VDAim11bHRpYWRkcnO4XAArNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwAtNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQOTd-h5owwj-cx7xrmbvQKU8CV3Fomfdvcv1MBc-67T5oN0Y3CCdl-DdWRwgiMohXdha3UyDw", diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index d2d6b1d99..3142ff766 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -137,13 +137,6 @@ proc initNode( ## Mount protocols -proc getNumShardsInNetwork*(conf: WakuNodeConf): uint32 = - if conf.numShardsInNetwork != 0: - return conf.numShardsInNetwork - # If conf.numShardsInNetwork is not set, use 1024 - the maximum possible as per the static sharding spec - # https://github.com/waku-org/specs/blob/master/standards/core/relay-sharding.md#static-sharding - return uint32(MaxShardIndex + 1) - proc getAutoshards*( node: WakuNode, contentTopics: seq[string] ): Result[seq[RelayShard], string] = @@ -265,6 +258,7 @@ proc setupProtocols( if conf.numShardsInNetwork == 0: warn "Number of shards in network not configured, setting it to", + # TODO: If not configured, it mounts 1024 shards! Make it a mandatory configuration instead numShardsInNetwork = $numShardsInNetwork node.mountSharding(conf.clusterId, numShardsInNetwork).isOkOr: diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim index 546253176..854df8dde 100644 --- a/waku/factory/waku.nim +++ b/waku/factory/waku.nim @@ -99,19 +99,6 @@ proc logConfig(conf: WakuNodeConf) = func version*(waku: Waku): string = waku.version -proc validateShards(conf: WakuNodeConf): Result[void, string] = - let numShardsInNetwork = getNumShardsInNetwork(conf) - - for shard in conf.shards: - if shard >= numShardsInNetwork: - let msg = - "validateShards invalid shard: " & $shard & " when numShardsInNetwork: " & - $numShardsInNetwork # fmt doesn't work - error "validateShards failed", error = msg - return err(msg) - - return ok() - proc setupSwitchServices( waku: Waku, conf: WakuNodeConf, circuitRelay: Relay, rng: ref HmacDrbgContext ) = @@ -214,46 +201,28 @@ proc new*( shards.add(shard) confCopy.shards = shards - case confCopy.clusterId + # Why can't I replace this block with a concise `.valueOr`? + confCopy = block: + let res = applyPresetConfiguration(confCopy) + if res.isErr(): + error "Failed to complete the config", error = res.error + return err("Failed to complete the config:" & $res.error) + res.get() - # cluster-id=1 (aka The Waku Network) - of 1: - let twnClusterConf = ClusterConf.TheWakuNetworkConf() - - # Override configuration - confCopy.maxMessageSize = twnClusterConf.maxMessageSize - confCopy.clusterId = twnClusterConf.clusterId - confCopy.rlnRelayEthContractAddress = twnClusterConf.rlnRelayEthContractAddress - confCopy.rlnRelayChainId = twnClusterConf.rlnRelayChainId - confCopy.rlnRelayDynamic = twnClusterConf.rlnRelayDynamic - confCopy.rlnRelayBandwidthThreshold = twnClusterConf.rlnRelayBandwidthThreshold - confCopy.discv5Discovery = twnClusterConf.discv5Discovery - confCopy.discv5BootstrapNodes = - confCopy.discv5BootstrapNodes & twnClusterConf.discv5BootstrapNodes - confCopy.rlnEpochSizeSec = twnClusterConf.rlnEpochSizeSec - confCopy.rlnRelayUserMessageLimit = twnClusterConf.rlnRelayUserMessageLimit - confCopy.numShardsInNetwork = twnClusterConf.numShardsInNetwork - - # Only set rlnRelay to true if relay is configured - if confCopy.relay: - confCopy.rlnRelay = twnClusterConf.rlnRelay - else: - discard + logConfig(confCopy) info "Running nwaku node", version = git_version - logConfig(confCopy) let validateShardsRes = validateShards(confCopy) if validateShardsRes.isErr(): error "Failed validating shards", error = $validateShardsRes.error return err("Failed validating shards: " & $validateShardsRes.error) - if not confCopy.nodekey.isSome(): - let keyRes = crypto.PrivateKey.random(Secp256k1, rng[]) - if keyRes.isErr(): - error "Failed to generate key", error = $keyRes.error - return err("Failed to generate key: " & $keyRes.error) - confCopy.nodekey = some(keyRes.get()) + let keyRes = getNodeKey(confCopy, rng) + if keyRes.isErr(): + error "Failed to generate key", error = $keyRes.error + return err("Failed to generate key: " & $keyRes.error) + confCopy.nodeKey = some(keyRes.get()) var relay = newCircuitRelay(confCopy.isRelayClient) @@ -284,6 +253,7 @@ proc new*( var waku = Waku( version: git_version, + # TODO: WakuNodeConf is re-used for too many context, `conf` here should be a dedicated subtype conf: confCopy, rng: rng, key: confCopy.nodekey.get(),