Add support for TOML config files (--config-file) (#3442)

.gitmodules

@@ -228,3 +228,8 @@
 	url = https://github.com/status-im/nim-zlib.git
 	ignore = untracked
 	branch = master
+[submodule "vendor/nim-toml-serialization"]
+	path = vendor/nim-toml-serialization
+	url = https://github.com/status-im/nim-toml-serialization.git
+	ignore = untracked
+	branch = master

beacon_chain/conf.nim

@@ -12,13 +12,17 @@ import
   metrics,
 
   chronicles, chronicles/options as chroniclesOptions,
-  confutils, confutils/defs, confutils/std/net, stew/shims/net as stewNet,
+  confutils, confutils/defs, confutils/std/net,
+  confutils/toml/defs as confTomlDefs,
+  confutils/toml/std/net as confTomlNet,
+  confutils/toml/std/uri as confTomlUri,
+  serialization/errors, stew/shims/net as stewNet,
   stew/[io2, byteutils], unicodedb/properties, normalize,
   eth/common/eth_types as commonEthTypes, eth/net/nat,
   eth/p2p/discoveryv5/enr,
   json_serialization, web3/[ethtypes, confutils_defs],
 
-  ./spec/[keystore, network],
+  ./spec/[keystore, network, crypto],
   ./spec/datatypes/base,
   ./networking/network_metadata,
   ./validators/slashing_protection_common,
@@ -28,7 +32,8 @@ export
   uri, nat, enr,
   defaultEth2TcpPort, enabledLogLevel, ValidIpAddress,
   defs, parseCmdArg, completeCmdArg, network_metadata,
-  network, BlockHashOrNumber
+  network, BlockHashOrNumber,
+  confTomlDefs, confTomlNet, confTomlUri
 
 declareGauge network_name, "network name", ["name"]
 
@@ -93,6 +98,10 @@ type
     # migrate = "Export and remove specified validators from Nimbus."
 
   BeaconNodeConf* = object
+    configFile* {.
+      desc: "Loads the configuration from a TOML file"
+      name: "config-file" }: Option[InputFile]
+
     logLevel* {.
       desc: "Sets the log level for process and topics (e.g. \"DEBUG; TRACE:discv5,libp2p; REQUIRED:none; DISABLED:none\")"
       defaultValue: "INFO"
@@ -674,6 +683,10 @@ type
         name: "backfill"}: bool
 
   ValidatorClientConf* = object
+    configFile* {.
+      desc: "Loads the configuration from a TOML file"
+      name: "config-file" }: Option[InputFile]
+
     logLevel* {.
       desc: "Sets the log level"
       defaultValue: "INFO"
@@ -751,6 +764,10 @@ type
         name: "beacon-node" }: seq[string]
 
   SigningNodeConf* = object
+    configFile* {.
+      desc: "Loads the configuration from a TOML file"
+      name: "config-file" }: Option[InputFile]
+
     logLevel* {.
       desc: "Sets the log level"
       defaultValue: "INFO"
@@ -993,6 +1010,64 @@ template writeValue*(writer: var JsonWriter,
                      value: TypedInputFile|InputFile|InputDir|OutPath|OutDir|OutFile) =
   writer.writeValue(string value)
 
+template raiseUnexpectedValue(r: var TomlReader, msg: string) =
+  # TODO: We need to implement `raiseUnexpectedValue` for TOML,
+  # so the correct line and column information can be included
+  # in error messages:
+  raise newException(SerializationError, msg)
+
+proc readValue*(r: var TomlReader, value: var Epoch)
+               {.raises: [Defect, SerializationError, IOError].} =
+  value = Epoch r.parseInt(uint64)
+
+proc readValue*(r: var TomlReader, value: var GraffitiBytes)
+               {.raises: [Defect, SerializationError, IOError].} =
+  try:
+    value = GraffitiBytes.init(r.readValue(string))
+  except ValueError as err:
+    r.raiseUnexpectedValue("A printable string or 0x-prefixed hex-encoded raw bytes expected")
+
+proc readValue*(r: var TomlReader, val: var NatConfig)
+               {.raises: [Defect, IOError, SerializationError].} =
+  val = try: parseCmdArg(NatConfig, TaintedString r.readValue(string))
+        except CatchableError as err:
+          raise newException(SerializationError, err.msg)
+
+proc readValue*(r: var TomlReader, a: var Eth2Digest)
+               {.raises: [Defect, IOError, SerializationError].} =
+  try:
+    a = fromHex(type(a), r.readValue(string))
+  except ValueError:
+    r.raiseUnexpectedValue("Hex string expected")
+
+proc readValue*(reader: var TomlReader, value: var ValidatorPubKey)
+               {.raises: [Defect, IOError, SerializationError].} =
+  let keyAsString = try:
+    reader.readValue(string)
+  except CatchableError:
+    raiseUnexpectedValue(reader, "A hex-encoded string expected")
+
+  let key = ValidatorPubKey.fromHex(keyAsString)
+  if key.isOk:
+    value = key.get
+  else:
+    # TODO: Can we provide better diagnostic?
+    raiseUnexpectedValue(reader, "Valid hex-encoded public key expected")
+
+proc readValue*(r: var TomlReader, a: var PubKey0x)
+               {.raises: [Defect, IOError, SerializationError].} =
+  try:
+    a = parseCmdArg(PubKey0x, TaintedString r.readValue(string))
+  except CatchableError:
+    r.raiseUnexpectedValue("a 0x-prefixed hex-encoded string expected")
+
+proc readValue*(r: var TomlReader, a: var WalletName)
+               {.raises: [Defect, IOError, SerializationError].} =
+  try:
+    a = parseCmdArg(WalletName, TaintedString r.readValue(string))
+  except CatchableError:
+    r.raiseUnexpectedValue("string expected")
+
 proc loadEth2Network*(config: BeaconNodeConf): Eth2NetworkMetadata {.raises: [Defect, IOError].} =
   network_name.set(2, labelValues = [config.eth2Network.get(otherwise = "mainnet")])
   when not defined(gnosisChainBinary):

beacon_chain/nimbus_binary_common.nim

@@ -14,7 +14,7 @@ import
   std/[os, tables, strutils, terminal, typetraits],
 
   # Nimble packages
-  chronos, confutils,
+  chronos, confutils, toml_serialization,
   chronicles, chronicles/helpers as chroniclesHelpers, chronicles/topics_registry,
   stew/io2,
 
@@ -26,7 +26,8 @@ import
 when defined(posix):
   import termios
 
-export beacon_clock, beacon_node_status, conf, confutils
+export
+  confutils, toml_serialization, beacon_clock, beacon_node_status, conf
 
 type
   SlotStartProc*[T] = proc(node: T, wallTime: BeaconTime,
@@ -180,11 +181,16 @@ template makeBannerAndConfig*(clientId: string, ConfType: type): untyped =
     version = clientId & "\p" & copyrights & "\p\p" &
       "eth2 specification v" & SPEC_VERSION & "\p\p" &
       nimBanner
+
   # TODO for some reason, copyrights are printed when doing `--help`
   {.push warning[ProveInit]: off.}
   let config = ConfType.load(
-    version = version,
+    version = version, # but a short version string makes more sense...
-    copyrightBanner = clientId) # but a short version string makes more sense...
+    copyrightBanner = clientId,
+    secondarySources = proc (config: ConfType, sources: auto) =
+      if config.configFile.isSome:
+        sources.addConfigFile(Toml, config.configFile.get)
+  )
   {.pop.}
   config
 

scripts/launch_local_testnet.sh

@@ -438,6 +438,20 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
   fi
 done
 
+CLI_CONF_FILE="$CONTAINER_DATA_DIR/config.toml"
+
+cat > "$CLI_CONF_FILE" <<END_CLI_CONFIG
+non-interactive = true
+nat = "extip:127.0.0.1"
+network = "${CONTAINER_DATA_DIR}"
+log-level = "${LOG_LEVEL}"
+log-format = "json"
+rest = true
+rest-address = "127.0.0.1"
+metrics = true
+metrics-address = "127.0.0.1"
+END_CLI_CONFIG
+
 for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
   NODE_DATA_DIR="${DATA_DIR}/node${NUM_NODE}"
   CONTAINER_NODE_DATA_DIR="${CONTAINER_DATA_DIR}/node${NUM_NODE}"
@@ -471,11 +485,7 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
   fi
 
   $BEACON_NODE_COMMAND \
-    --non-interactive \
+    --config-file="$CLI_CONF_FILE" \
-    --nat:extip:127.0.0.1 \
-    --network="${CONTAINER_DATA_DIR}" \
-    --log-level="${LOG_LEVEL}" \
-    --log-format=json \
     --tcp-port=$(( BASE_PORT + NUM_NODE )) \
     --udp-port=$(( BASE_PORT + NUM_NODE )) \
     --max-peers=$(( NUM_NODES - 1 )) \
@@ -483,11 +493,7 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
     ${BOOTSTRAP_ARG} \
     ${WEB3_ARG} \
     ${STOP_AT_EPOCH_FLAG} \
-    --rest \
-    --rest-address="127.0.0.1" \
     --rest-port="$(( BASE_REST_PORT + NUM_NODE ))" \
-    --metrics \
-    --metrics-address="127.0.0.1" \
     --metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \
     ${EXTRA_ARGS} \
     > "${DATA_DIR}/log${NUM_NODE}.txt" 2>&1 &

vendor/nim-confutils

@@ -1 +1 @@
-Subproject commit 6a56d01381f434d5fbcc61b6e497b9409155bcbc
+Subproject commit 0fc26c5b25a931fdd15a74f2d9028112ffd621ba

vendor/nim-faststreams

@@ -1 +1 @@
-Subproject commit 3a0ab42573e566ce52625760f6bbf7e0bbb6ebc4
+Subproject commit 37a183153c071539ab870f427c09a1376ba311b9

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit fcd0eadadde0ee000a63df8ab21dc4e9f015a790
+Subproject commit 37bc0db558d85711967acb16e9bb822b06911d46

vendor/nim-toml-serialization

@@ -0,0 +1 @@
+Subproject commit f4fb6c6ef833bff6649f26830c6a8401c945aa38