feat!: introduce preset option

Overwriting config from cluster-id will be deprecated as a second step.

tests/all_tests_waku.nim

@@ -30,7 +30,7 @@ import
 const os* {.strdefine.} = ""
 when os == "Linux" and
     # GitHub only supports container actions on Linux
-    # and we need to start a postgress database in a docker container
+    # and we need to start a postgres database in a docker container
     defined(postgres):
   import
     ./waku_archive/test_driver_postgres_query,
@@ -106,3 +106,4 @@ import
 import ./waku_rln_relay/test_all
 
 # Node Factory
+import ./factory/test_config

tests/factory/test_config.nim

@@ -0,0 +1,157 @@
+{.used.}
+
+import
+  std/options,
+  testutils/unittests,
+  chronos,
+  libp2p/crypto/[crypto, secp],
+  libp2p/multiaddress,
+  nimcrypto/utils,
+  secp256k1,
+  confutils
+import
+  ../../waku/factory/external_config,
+  ../../waku/factory/internal_config,
+  ../../waku/factory/networks_config,
+  ../../waku/common/logging
+
+suite "Waku config - apply preset":
+  test "Default preset is TWN":
+    ## Setup
+    let expectedConf = ClusterConf.TheWakuNetworkConf()
+
+    ## Given
+    let preConfig = WakuNodeConf(cmd: noCommand, preset: "default")
+
+    ## When
+    let res = applyPresetConfiguration(preConfig)
+    assert res.isOk(), $res.error
+
+    ## Then
+    let conf = res.get()
+    assert conf.maxMessageSize == expectedConf.maxMessageSize
+    assert conf.clusterId == expectedConf.clusterId
+    assert conf.rlnRelay == expectedConf.rlnRelay
+    assert conf.rlnRelayEthContractAddress == expectedConf.rlnRelayEthContractAddress
+    assert conf.rlnRelayDynamic == expectedConf.rlnRelayDynamic
+    assert conf.rlnRelayChainId == expectedConf.rlnRelayChainId
+    assert conf.rlnRelayBandwidthThreshold == expectedConf.rlnRelayBandwidthThreshold
+    assert conf.rlnEpochSizeSec == expectedConf.rlnEpochSizeSec
+    assert conf.rlnRelayUserMessageLimit == expectedConf.rlnRelayUserMessageLimit
+    assert conf.numShardsInNetwork == expectedConf.numShardsInNetwork
+    assert conf.discv5BootstrapNodes == expectedConf.discv5BootstrapNodes
+
+  test "Subscribes to all valid shards in default network":
+    ## Setup
+    let expectedConf = ClusterConf.TheWakuNetworkConf()
+
+    ## Given
+    let shards: seq[uint16] = @[0, 1, 2, 3, 4, 5, 6, 7]
+    let preConfig = WakuNodeConf(cmd: noCommand, preset: "default", shards: shards)
+
+    ## When
+    let res = applyPresetConfiguration(preConfig)
+    assert res.isOk(), $res.error
+
+    ## Then
+    let conf = res.get()
+    assert conf.shards.len == expectedConf.numShardsInNetwork.int
+
+  test "Subscribes to some valid shards in default network":
+    ## Setup
+    let expectedConf = ClusterConf.TheWakuNetworkConf()
+
+    ## Given
+    let shards: seq[uint16] = @[0, 4, 7]
+    let preConfig = WakuNodeConf(cmd: noCommand, preset: "default", shards: shards)
+
+    ## When
+    let resConf = applyPresetConfiguration(preConfig)
+    let res = validateShards(resConf.get())
+    assert res.isOk(), $res.error
+
+    ## Then
+    let conf = resConf.get()
+    assert conf.shards.len() == shards.len()
+    for index, shard in shards:
+      assert shard in conf.shards
+
+  test "Subscribes to invalid shards in default network":
+    ## Setup
+
+    ## Given
+    let shards: seq[uint16] = @[0, 4, 7, 10]
+    let preConfig = WakuNodeConf(cmd: noCommand, preset: "default", shards: shards)
+    let postConfig = applyPresetConfiguration(preConfig)
+
+    ## When
+    let res = validateShards(postConfig.get())
+
+    ## Then
+    assert res.isErr(), "Invalid shard was accepted"
+
+suite "Waku config - node key":
+  test "Passed node key is used":
+    ## Setup
+    let nodeKeyStr =
+      "0011223344556677889900aabbccddeeff0011223344556677889900aabbccddeeff"
+    let nodekey = block:
+      let key = SkPrivateKey.init(utils.fromHex(nodeKeyStr)).tryGet()
+      crypto.PrivateKey(scheme: Secp256k1, skkey: key)
+
+    ## Given
+    let config = WakuNodeConf.load(version = "", cmdLine = @["--nodekey=" & nodeKeyStr])
+
+    ## When
+    let res = nodeKeyConfiguration(config)
+    assert res.isOk(), $res.error
+
+    ## Then
+    let resKey = res.get()
+    assert utils.toHex(resKey.getRawBytes().get()) ==
+      utils.toHex(nodekey.getRawBytes().get())
+
+suite "Waku config - Shards":
+  test "Shards are valid":
+    ## Setup
+
+    ## Given
+    let shards: seq[uint16] = @[0, 2, 4]
+    let numShardsInNetwork = 5.uint32
+    let config = WakuNodeConf(
+      cmd: noCommand, shards: shards, numShardsInNetwork: numShardsInNetwork
+    )
+
+    ## When
+    let res = validateShards(config)
+
+    ## Then
+    assert res.isOk(), $res.error
+
+  test "Shards are not in range":
+    ## Setup
+
+    ## Given
+    let shards: seq[uint16] = @[0, 2, 5]
+    let numShardsInNetwork = 5.uint32
+    let config = WakuNodeConf(
+      cmd: noCommand, shards: shards, numShardsInNetwork: numShardsInNetwork
+    )
+
+    ## When
+    let res = validateShards(config)
+
+    ## Then
+    assert res.isErr(), "Invalid shard was accepted"
+
+  test "Shard is passed without num shards":
+    ## Setup
+
+    ## Given
+    let config = WakuNodeConf.load(version = "", cmdLine = @["--shard=32"])
+
+    ## When
+    let res = validateShards(config)
+
+    ## Then
+    assert res.isOk(), $res.error

waku/factory/external_config.nim

@@ -76,7 +76,7 @@ type WakuNodeConf* = object
   .}: EthRpcUrl
 
   rlnRelayEthContractAddress* {.
-    desc: "Address of membership contract on an Ethereum testnet",
+    desc: "Address of membership contract on an Ethereum testnet. Part of presets.",
     defaultValue: "",
     name: "rln-relay-eth-contract-address"
   .}: string
@@ -102,21 +102,21 @@ type WakuNodeConf* = object
 
   rlnRelayUserMessageLimit* {.
     desc:
-      "Set a user message limit for the rln membership registration. Must be a positive integer. Default is 1.",
+      "Set a user message limit for the rln membership registration. Must be a positive integer. Default is 1. Part of presets.",
     defaultValue: 1,
     name: "rln-relay-user-message-limit"
   .}: uint64
 
   rlnEpochSizeSec* {.
     desc:
-      "Epoch size in seconds used to rate limit RLN memberships. Default is 1 second.",
+      "Epoch size in seconds used to rate limit RLN memberships. Default is 1 second. Part of presets.",
     defaultValue: 1,
     name: "rln-relay-epoch-sec"
   .}: uint64
 
   maxMessageSize* {.
     desc:
-      "Maximum message size. Accepted units: KiB, KB, and B. e.g. 1024KiB; 1500 B; etc.",
+      "Maximum message size. Accepted units: KiB, KB, and B. e.g. 1024KiB; 1500 B; etc. Part of presets.",
     defaultValue: DefaultMaxWakuMessageSizeStr,
     name: "max-msg-size"
   .}: string
@@ -145,9 +145,15 @@ type WakuNodeConf* = object
     .}: seq[ProtectedShard]
 
     ## General node config
+    preset* {.
+      desc: "Network preset to use." & "Must be one of 'default', ''",
+      defaultValue: "",
+      name: "preset"
+    .}: string
+
     clusterId* {.
       desc:
-        "Cluster id that the node is running in. Node in a different cluster id is disconnected.",
+        "Cluster id that the node is running in. Node in a different cluster id is disconnected. Part of presets.",
       defaultValue: 0,
       name: "cluster-id"
     .}: uint16
@@ -276,7 +282,7 @@ hence would have reachability issues.""",
     .}: bool
 
     rlnRelay* {.
-      desc: "Enable spam protection through rln-relay: true|false",
+      desc: "Enable spam protection through rln-relay: true|false. Part of presets.",
       defaultValue: false,
       name: "rln-relay"
     .}: bool
@@ -287,7 +293,8 @@ hence would have reachability issues.""",
     .}: Option[uint]
 
     rlnRelayDynamic* {.
-      desc: "Enable  waku-rln-relay with on-chain dynamic group management: true|false",
+      desc:
+        "Enable  waku-rln-relay with on-chain dynamic group management: true|false. Part of presets.",
       defaultValue: false,
       name: "rln-relay-dynamic"
     .}: bool
@@ -311,7 +318,8 @@ hence would have reachability issues.""",
     .}: string
 
     rlnRelayBandwidthThreshold* {.
-      desc: "Message rate in bytes/sec after which verification of proofs should happen",
+      desc:
+        "Message rate in bytes/sec after which verification of proofs should happen. Part of presets.",
       defaultValue: 0, # to maintain backwards compatibility
       name: "rln-relay-bandwidth-threshold"
     .}: int
@@ -327,6 +335,7 @@ hence would have reachability issues.""",
       name: "keep-alive"
     .}: bool
 
+    # TODO: This is trying to do too much, this should only be used for autosharding, which itself should be configurable
     # If numShardsInNetwork is not set, we use the number of shards configured as numShardsInNetwork
     numShardsInNetwork* {.
       desc: "Number of shards in the network",
@@ -590,7 +599,7 @@ with the drawback of consuming some more bandwidth.""",
 
     ## Discovery v5 config
     discv5Discovery* {.
-      desc: "Enable discovering nodes via Node Discovery v5",
+      desc: "Enable discovering nodes via Node Discovery v5. Part of presets.",
       defaultValue: false,
       name: "discv5-discovery"
     .}: bool
@@ -603,7 +612,7 @@ with the drawback of consuming some more bandwidth.""",
 
     discv5BootstrapNodes* {.
       desc:
-        "Text-encoded ENR for bootstrap node. Used when connecting to the network. Argument may be repeated.",
+        "Text-encoded ENR for bootstrap node. Used when connecting to the network. Argument may be repeated. Part of presets.",
       name: "discv5-bootstrap-node"
     .}: seq[string]
 

waku/factory/internal_config.nim

@@ -12,7 +12,8 @@ import
   ../node/config,
   ../waku_enr/capabilities,
   ../waku_enr,
-  ../waku_core
+  ../waku_core,
+  ./networks_config
 
 proc enrConfiguration*(
     conf: WakuNodeConf, netConfig: NetConfig, key: crypto.PrivateKey
@@ -157,3 +158,81 @@ proc networkConfiguration*(conf: WakuNodeConf, clientId: string): NetConfigResul
   )
 
   return netConfigRes
+
+proc applyPresetConfiguration*(srcConf: WakuNodeConf): Result[WakuNodeConf, string] =
+  var resConf = srcConf
+
+  if resConf.clusterId == 1:
+    warn(
+      "Sandbox (The Waku Network) configuration will not be applied when `--cluster-id=1` is passed in future releases. Use `--preset=sandbox` instead."
+    )
+    resConf.preset = "default"
+
+  case resConf.preset
+  of "default":
+    let twnClusterConf = ClusterConf.TheWakuNetworkConf()
+
+    # Override configuration
+    resConf.maxMessageSize = twnClusterConf.maxMessageSize
+    resConf.clusterId = twnClusterConf.clusterId
+    resConf.rlnRelay = twnClusterConf.rlnRelay
+    resConf.rlnRelayEthContractAddress = twnClusterConf.rlnRelayEthContractAddress
+    resConf.rlnRelayChainId = twnClusterConf.rlnRelayChainId
+    resConf.rlnRelayDynamic = twnClusterConf.rlnRelayDynamic
+    resConf.rlnRelayBandwidthThreshold = twnClusterConf.rlnRelayBandwidthThreshold
+    resConf.discv5Discovery = twnClusterConf.discv5Discovery
+    resConf.discv5BootstrapNodes =
+      resConf.discv5BootstrapNodes & twnClusterConf.discv5BootstrapNodes
+    resConf.rlnEpochSizeSec = twnClusterConf.rlnEpochSizeSec
+    resConf.rlnRelayUserMessageLimit = twnClusterConf.rlnRelayUserMessageLimit
+    resConf.numShardsInNetwork = twnClusterConf.numShardsInNetwork
+
+    if resConf.relay:
+      resConf.rlnRelay = twnClusterConf.rlnRelay
+  else:
+    discard
+
+  return ok(resConf)
+
+# TODO: numShardsInNetwork should be mandatory with autosharding, and unneeded otherwise
+proc getNumShardsInNetwork*(conf: WakuNodeConf): uint32 =
+  if conf.numShardsInNetwork != 0:
+    return conf.numShardsInNetwork
+  # If conf.numShardsInNetwork is not set, use 1024 - the maximum possible as per the static sharding spec
+  # https://github.com/waku-org/specs/blob/master/standards/core/relay-sharding.md#static-sharding
+  return uint32(MaxShardIndex + 1)
+
+proc validateShards*(conf: WakuNodeConf): Result[void, string] =
+  let numShardsInNetwork = getNumShardsInNetwork(conf)
+
+  for shard in conf.shards:
+    if shard >= numShardsInNetwork:
+      let msg =
+        "validateShards invalid shard: " & $shard & " when numShardsInNetwork: " &
+        $numShardsInNetwork # fmt doesn't work
+      error "validateShards failed", error = msg
+      return err(msg)
+
+  return ok()
+
+proc nodeKeyConfiguration*(
+    conf: WakuNodeConf, rng: Option[ref HmacDrbgContext] = none(ref HmacDrbgContext)
+): Result[PrivateKey, string] =
+  if conf.nodeKey.isSome:
+    return ok(conf.nodeKey.get())
+
+  let key =
+    if conf.nodeKey.isSome() and rng.isSome():
+      conf.nodeKey.get()
+    else:
+      warn "missing key or rng, generating new set"
+      var nodeRng =
+        if rng.isSome():
+          rng.get()
+        else:
+          crypto.newRng()
+      crypto.PrivateKey.random(Secp256k1, nodeRng[]).valueOr:
+        error "Failed to generate key", error = error
+        return err("Failed to generate key: " & $error)
+
+  return ok(key)

waku/factory/networks_config.nim

@@ -30,6 +30,7 @@ proc TheWakuNetworkConf*(T: type ClusterConf): ClusterConf =
     rlnRelayUserMessageLimit: 100,
     numShardsInNetwork: 8,
     discv5Discovery: true,
+      # TODO: Why is this part of the conf? eg an edge node would not have this
     discv5BootstrapNodes:
       @[
         "enr:-QESuED0qW1BCmF-oH_ARGPr97Nv767bl_43uoy70vrbah3EaCAdK3Q0iRQ6wkSTTpdrg_dU_NC2ydO8leSlRpBX4pxiAYJpZIJ2NIJpcIRA4VDAim11bHRpYWRkcnO4XAArNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwAtNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQOTd-h5owwj-cx7xrmbvQKU8CV3Fomfdvcv1MBc-67T5oN0Y3CCdl-DdWRwgiMohXdha3UyDw",

waku/factory/node_factory.nim

@@ -137,13 +137,6 @@ proc initNode(
 
 ## Mount protocols
 
-proc getNumShardsInNetwork*(conf: WakuNodeConf): uint32 =
-  if conf.numShardsInNetwork != 0:
-    return conf.numShardsInNetwork
-  # If conf.numShardsInNetwork is not set, use 1024 - the maximum possible as per the static sharding spec
-  # https://github.com/waku-org/specs/blob/master/standards/core/relay-sharding.md#static-sharding
-  return uint32(MaxShardIndex + 1)
-
 proc getAutoshards*(
     node: WakuNode, contentTopics: seq[string]
 ): Result[seq[RelayShard], string] =
@@ -265,6 +258,7 @@ proc setupProtocols(
 
   if conf.numShardsInNetwork == 0:
     warn "Number of shards in network not configured, setting it to",
+      # TODO: If not configured, it mounts 1024 shards! Make it a mandatory configuration instead
       numShardsInNetwork = $numShardsInNetwork
 
   node.mountSharding(conf.clusterId, numShardsInNetwork).isOkOr:
@@ -484,15 +478,9 @@ proc startNode*(
 proc setupNode*(
     conf: WakuNodeConf, rng: ref HmacDrbgContext = crypto.newRng(), relay: Relay
 ): Result[WakuNode, string] =
-  # Use provided key only if corresponding rng is also provided
-  let key =
-    if conf.nodeKey.isSome():
-      conf.nodeKey.get()
-    else:
-      warn "missing key, generating new"
-      crypto.PrivateKey.random(Secp256k1, rng[]).valueOr:
-        error "Failed to generate key", error = error
-        return err("Failed to generate key: " & $error)
+  let key = nodeKeyConfiguration(conf).valueOr:
+    error "Failed to set node key", error = error
+    return err("Failed to set node key: " & error)
 
   let netConfig = networkConfiguration(conf, clientId).valueOr:
     error "failed to create internal config", error = error

waku/factory/waku.nim

@@ -99,19 +99,6 @@ proc logConfig(conf: WakuNodeConf) =
 func version*(waku: Waku): string =
   waku.version
 
-proc validateShards(conf: WakuNodeConf): Result[void, string] =
-  let numShardsInNetwork = getNumShardsInNetwork(conf)
-
-  for shard in conf.shards:
-    if shard >= numShardsInNetwork:
-      let msg =
-        "validateShards invalid shard: " & $shard & " when numShardsInNetwork: " &
-        $numShardsInNetwork # fmt doesn't work
-      error "validateShards failed", error = msg
-      return err(msg)
-
-  return ok()
-
 proc setupSwitchServices(
     waku: Waku, conf: WakuNodeConf, circuitRelay: Relay, rng: ref HmacDrbgContext
 ) =
@@ -184,16 +171,16 @@ proc setupAppCallbacks(
   return ok()
 
 proc new*(
-    T: type Waku, confCopy: var WakuNodeConf, appCallbacks: AppCallbacks = nil
+    T: type Waku, srcConf: var WakuNodeConf, appCallbacks: AppCallbacks = nil
 ): Result[Waku, string] =
   let rng = crypto.newRng()
 
-  logging.setupLog(confCopy.logLevel, confCopy.logFormat)
+  logging.setupLog(srcConf.logLevel, srcConf.logFormat)
 
   # TODO: remove after pubsubtopic config gets removed
   var shards = newSeq[uint16]()
-  if confCopy.pubsubTopics.len > 0:
-    let shardsRes = topicsToRelayShards(confCopy.pubsubTopics)
+  if srcConf.pubsubTopics.len > 0:
+    let shardsRes = topicsToRelayShards(srcConf.pubsubTopics)
     if shardsRes.isErr():
       error "failed to parse pubsub topic, please format according to static shard specification",
         error = shardsRes.error
@@ -203,75 +190,59 @@ proc new*(
 
     if shardsOpt.isSome():
       let relayShards = shardsOpt.get()
-      if relayShards.clusterId != confCopy.clusterId:
+      if relayShards.clusterId != srcConf.clusterId:
         error "clusterId of the pubsub topic should match the node's cluster. e.g. --pubsub-topic=/waku/2/rs/22/1 and --cluster-id=22",
-          nodeCluster = confCopy.clusterId, pubsubCluster = relayShards.clusterId
+          nodeCluster = srcConf.clusterId, pubsubCluster = relayShards.clusterId
         return err(
           "clusterId of the pubsub topic should match the node's cluster. e.g. --pubsub-topic=/waku/2/rs/22/1 and --cluster-id=22"
         )
 
       for shard in relayShards.shardIds:
         shards.add(shard)
-      confCopy.shards = shards
+      srcConf.shards = shards
 
-  case confCopy.clusterId
+  # Why can't I replace this block with a concise `.valueOr`?
+  let finalConf = block:
+    let res = applyPresetConfiguration(srcConf)
+    if res.isErr():
+      error "Failed to complete the config", error = res.error
+      return err("Failed to complete the config:" & $res.error)
+    res.get()
 
-  # cluster-id=1 (aka The Waku Network)
-  of 1:
-    let twnClusterConf = ClusterConf.TheWakuNetworkConf()
-
-    # Override configuration
-    confCopy.maxMessageSize = twnClusterConf.maxMessageSize
-    confCopy.clusterId = twnClusterConf.clusterId
-    confCopy.rlnRelayEthContractAddress = twnClusterConf.rlnRelayEthContractAddress
-    confCopy.rlnRelayChainId = twnClusterConf.rlnRelayChainId
-    confCopy.rlnRelayDynamic = twnClusterConf.rlnRelayDynamic
-    confCopy.rlnRelayBandwidthThreshold = twnClusterConf.rlnRelayBandwidthThreshold
-    confCopy.discv5Discovery = twnClusterConf.discv5Discovery
-    confCopy.discv5BootstrapNodes =
-      confCopy.discv5BootstrapNodes & twnClusterConf.discv5BootstrapNodes
-    confCopy.rlnEpochSizeSec = twnClusterConf.rlnEpochSizeSec
-    confCopy.rlnRelayUserMessageLimit = twnClusterConf.rlnRelayUserMessageLimit
-    confCopy.numShardsInNetwork = twnClusterConf.numShardsInNetwork
-
-    # Only set rlnRelay to true if relay is configured
-    if confCopy.relay:
-      confCopy.rlnRelay = twnClusterConf.rlnRelay
-  else:
-    discard
+  logConfig(finalConf)
 
   info "Running nwaku node", version = git_version
-  logConfig(confCopy)
 
-  let validateShardsRes = validateShards(confCopy)
+  let validateShardsRes = validateShards(finalConf)
   if validateShardsRes.isErr():
     error "Failed validating shards", error = $validateShardsRes.error
     return err("Failed validating shards: " & $validateShardsRes.error)
 
-  if not confCopy.nodekey.isSome():
+  # TODO: We are also generating a node key in setupNode
+  if not finalConf.nodekey.isSome():
     let keyRes = crypto.PrivateKey.random(Secp256k1, rng[])
     if keyRes.isErr():
       error "Failed to generate key", error = $keyRes.error
       return err("Failed to generate key: " & $keyRes.error)
-    confCopy.nodekey = some(keyRes.get())
+    finalConf.nodekey = some(keyRes.get())
 
-  var relay = newCircuitRelay(confCopy.isRelayClient)
+  var relay = newCircuitRelay(finalConf.isRelayClient)
 
-  let nodeRes = setupNode(confCopy, rng, relay)
+  let nodeRes = setupNode(finalConf, rng, relay)
   if nodeRes.isErr():
     error "Failed setting up node", error = nodeRes.error
     return err("Failed setting up node: " & nodeRes.error)
 
   let node = nodeRes.get()
 
-  node.setupAppCallbacks(confCopy, appCallbacks).isOkOr:
+  node.setupAppCallbacks(finalConf, appCallbacks).isOkOr:
     error "Failed setting up app callbacks", error = error
     return err("Failed setting up app callbacks: " & $error)
 
   ## Delivery Monitor
   var deliveryMonitor: DeliveryMonitor
-  if confCopy.reliabilityEnabled:
-    if confCopy.storenode == "":
+  if finalConf.reliabilityEnabled:
+    if finalConf.storenode == "":
       return err("A storenode should be set when reliability mode is on")
 
     let deliveryMonitorRes = DeliveryMonitor.new(
@@ -284,15 +255,15 @@ proc new*(
 
   var waku = Waku(
     version: git_version,
-    conf: confCopy,
+    conf: finalConf,
     rng: rng,
-    key: confCopy.nodekey.get(),
+    key: finalConf.nodekey.get(),
     node: node,
     deliveryMonitor: deliveryMonitor,
     appCallbacks: appCallbacks,
   )
 
-  waku.setupSwitchServices(confCopy, relay, rng)
+  waku.setupSwitchServices(finalConf, relay, rng)
 
   ok(waku)