From cdeae90806d78d4f088ffd6dfcb9411f1782ea25 Mon Sep 17 00:00:00 2001 From: zah Date: Sat, 5 Mar 2022 04:33:15 +0200 Subject: [PATCH] Add support for TOML config files (--config-file) (#3442) --- .gitmodules | 5 ++ beacon_chain/conf.nim | 81 ++++++++++++++++++++++++++- beacon_chain/nimbus_binary_common.nim | 14 +++-- scripts/launch_local_testnet.sh | 24 +++++--- vendor/nim-confutils | 2 +- vendor/nim-faststreams | 2 +- vendor/nim-serialization | 2 +- vendor/nim-toml-serialization | 1 + 8 files changed, 112 insertions(+), 19 deletions(-) create mode 160000 vendor/nim-toml-serialization diff --git a/.gitmodules b/.gitmodules index f8965d418..94943580f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -228,3 +228,8 @@ url = https://github.com/status-im/nim-zlib.git ignore = untracked branch = master +[submodule "vendor/nim-toml-serialization"] + path = vendor/nim-toml-serialization + url = https://github.com/status-im/nim-toml-serialization.git + ignore = untracked + branch = master diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 4cc775f52..db7e3822a 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -12,13 +12,17 @@ import metrics, chronicles, chronicles/options as chroniclesOptions, - confutils, confutils/defs, confutils/std/net, stew/shims/net as stewNet, + confutils, confutils/defs, confutils/std/net, + confutils/toml/defs as confTomlDefs, + confutils/toml/std/net as confTomlNet, + confutils/toml/std/uri as confTomlUri, + serialization/errors, stew/shims/net as stewNet, stew/[io2, byteutils], unicodedb/properties, normalize, eth/common/eth_types as commonEthTypes, eth/net/nat, eth/p2p/discoveryv5/enr, json_serialization, web3/[ethtypes, confutils_defs], - ./spec/[keystore, network], + ./spec/[keystore, network, crypto], ./spec/datatypes/base, ./networking/network_metadata, ./validators/slashing_protection_common, @@ -28,7 +32,8 @@ export uri, nat, enr, defaultEth2TcpPort, enabledLogLevel, ValidIpAddress, defs, parseCmdArg, completeCmdArg, network_metadata, - network, BlockHashOrNumber + network, BlockHashOrNumber, + confTomlDefs, confTomlNet, confTomlUri declareGauge network_name, "network name", ["name"] @@ -93,6 +98,10 @@ type # migrate = "Export and remove specified validators from Nimbus." BeaconNodeConf* = object + configFile* {. + desc: "Loads the configuration from a TOML file" + name: "config-file" }: Option[InputFile] + logLevel* {. desc: "Sets the log level for process and topics (e.g. \"DEBUG; TRACE:discv5,libp2p; REQUIRED:none; DISABLED:none\")" defaultValue: "INFO" @@ -674,6 +683,10 @@ type name: "backfill"}: bool ValidatorClientConf* = object + configFile* {. + desc: "Loads the configuration from a TOML file" + name: "config-file" }: Option[InputFile] + logLevel* {. desc: "Sets the log level" defaultValue: "INFO" @@ -751,6 +764,10 @@ type name: "beacon-node" }: seq[string] SigningNodeConf* = object + configFile* {. + desc: "Loads the configuration from a TOML file" + name: "config-file" }: Option[InputFile] + logLevel* {. desc: "Sets the log level" defaultValue: "INFO" @@ -993,6 +1010,64 @@ template writeValue*(writer: var JsonWriter, value: TypedInputFile|InputFile|InputDir|OutPath|OutDir|OutFile) = writer.writeValue(string value) +template raiseUnexpectedValue(r: var TomlReader, msg: string) = + # TODO: We need to implement `raiseUnexpectedValue` for TOML, + # so the correct line and column information can be included + # in error messages: + raise newException(SerializationError, msg) + +proc readValue*(r: var TomlReader, value: var Epoch) + {.raises: [Defect, SerializationError, IOError].} = + value = Epoch r.parseInt(uint64) + +proc readValue*(r: var TomlReader, value: var GraffitiBytes) + {.raises: [Defect, SerializationError, IOError].} = + try: + value = GraffitiBytes.init(r.readValue(string)) + except ValueError as err: + r.raiseUnexpectedValue("A printable string or 0x-prefixed hex-encoded raw bytes expected") + +proc readValue*(r: var TomlReader, val: var NatConfig) + {.raises: [Defect, IOError, SerializationError].} = + val = try: parseCmdArg(NatConfig, TaintedString r.readValue(string)) + except CatchableError as err: + raise newException(SerializationError, err.msg) + +proc readValue*(r: var TomlReader, a: var Eth2Digest) + {.raises: [Defect, IOError, SerializationError].} = + try: + a = fromHex(type(a), r.readValue(string)) + except ValueError: + r.raiseUnexpectedValue("Hex string expected") + +proc readValue*(reader: var TomlReader, value: var ValidatorPubKey) + {.raises: [Defect, IOError, SerializationError].} = + let keyAsString = try: + reader.readValue(string) + except CatchableError: + raiseUnexpectedValue(reader, "A hex-encoded string expected") + + let key = ValidatorPubKey.fromHex(keyAsString) + if key.isOk: + value = key.get + else: + # TODO: Can we provide better diagnostic? + raiseUnexpectedValue(reader, "Valid hex-encoded public key expected") + +proc readValue*(r: var TomlReader, a: var PubKey0x) + {.raises: [Defect, IOError, SerializationError].} = + try: + a = parseCmdArg(PubKey0x, TaintedString r.readValue(string)) + except CatchableError: + r.raiseUnexpectedValue("a 0x-prefixed hex-encoded string expected") + +proc readValue*(r: var TomlReader, a: var WalletName) + {.raises: [Defect, IOError, SerializationError].} = + try: + a = parseCmdArg(WalletName, TaintedString r.readValue(string)) + except CatchableError: + r.raiseUnexpectedValue("string expected") + proc loadEth2Network*(config: BeaconNodeConf): Eth2NetworkMetadata {.raises: [Defect, IOError].} = network_name.set(2, labelValues = [config.eth2Network.get(otherwise = "mainnet")]) when not defined(gnosisChainBinary): diff --git a/beacon_chain/nimbus_binary_common.nim b/beacon_chain/nimbus_binary_common.nim index 2aced746b..be67b3917 100644 --- a/beacon_chain/nimbus_binary_common.nim +++ b/beacon_chain/nimbus_binary_common.nim @@ -14,7 +14,7 @@ import std/[os, tables, strutils, terminal, typetraits], # Nimble packages - chronos, confutils, + chronos, confutils, toml_serialization, chronicles, chronicles/helpers as chroniclesHelpers, chronicles/topics_registry, stew/io2, @@ -26,7 +26,8 @@ import when defined(posix): import termios -export beacon_clock, beacon_node_status, conf, confutils +export + confutils, toml_serialization, beacon_clock, beacon_node_status, conf type SlotStartProc*[T] = proc(node: T, wallTime: BeaconTime, @@ -180,11 +181,16 @@ template makeBannerAndConfig*(clientId: string, ConfType: type): untyped = version = clientId & "\p" & copyrights & "\p\p" & "eth2 specification v" & SPEC_VERSION & "\p\p" & nimBanner + # TODO for some reason, copyrights are printed when doing `--help` {.push warning[ProveInit]: off.} let config = ConfType.load( - version = version, - copyrightBanner = clientId) # but a short version string makes more sense... + version = version, # but a short version string makes more sense... + copyrightBanner = clientId, + secondarySources = proc (config: ConfType, sources: auto) = + if config.configFile.isSome: + sources.addConfigFile(Toml, config.configFile.get) + ) {.pop.} config diff --git a/scripts/launch_local_testnet.sh b/scripts/launch_local_testnet.sh index 850c8c739..63db45df4 100755 --- a/scripts/launch_local_testnet.sh +++ b/scripts/launch_local_testnet.sh @@ -438,6 +438,20 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do fi done +CLI_CONF_FILE="$CONTAINER_DATA_DIR/config.toml" + +cat > "$CLI_CONF_FILE" < "${DATA_DIR}/log${NUM_NODE}.txt" 2>&1 & diff --git a/vendor/nim-confutils b/vendor/nim-confutils index 6a56d0138..0fc26c5b2 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit 6a56d01381f434d5fbcc61b6e497b9409155bcbc +Subproject commit 0fc26c5b25a931fdd15a74f2d9028112ffd621ba diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams index 3a0ab4257..37a183153 160000 --- a/vendor/nim-faststreams +++ b/vendor/nim-faststreams @@ -1 +1 @@ -Subproject commit 3a0ab42573e566ce52625760f6bbf7e0bbb6ebc4 +Subproject commit 37a183153c071539ab870f427c09a1376ba311b9 diff --git a/vendor/nim-serialization b/vendor/nim-serialization index fcd0eadad..37bc0db55 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit fcd0eadadde0ee000a63df8ab21dc4e9f015a790 +Subproject commit 37bc0db558d85711967acb16e9bb822b06911d46 diff --git a/vendor/nim-toml-serialization b/vendor/nim-toml-serialization new file mode 160000 index 000000000..f4fb6c6ef --- /dev/null +++ b/vendor/nim-toml-serialization @@ -0,0 +1 @@ +Subproject commit f4fb6c6ef833bff6649f26830c6a8401c945aa38