status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Backwards compatible handling of the web3-url parameter in TOML

This commit is contained in:
Zahary Karadjov 2023-03-14 17:40:37 +02:00
parent 3a35809a02
commit 46f48269ef
No known key found for this signature in database
GPG Key ID: C1F42EAFF38D570F
9 changed files with 226 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
AllTests-mainnet.md
View File

@ -161,10 +161,19 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
+ Testing uints inputs - valid OK
```
OK: 10/12 Fail: 0/12 Skip: 2/12
## EL Configuration
```diff
+ Empty config file OK
+ Invalid URls OK
+ New style config files OK
+ Old style config files OK
+ URL parsing OK
```
OK: 5/5 Fail: 0/5 Skip: 0/5
## Eth1 monitor
```diff
+ Deposits chain OK
+ Rewrite HTTPS Infura URLs OK
+ Rewrite URLs OK
+ Roundtrip engine RPC V1 and bellatrix ExecutionPayload representations OK
+ Roundtrip engine RPC V2 and capella ExecutionPayload representations OK
+ Roundtrip engine RPC V3 and deneb ExecutionPayload representations OK
@ -626,4 +635,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
OK: 9/9 Fail: 0/9 Skip: 0/9
---TOTAL---
OK: 347/352 Fail: 0/352 Skip: 5/352
OK: 352/357 Fail: 0/357 Skip: 5/357

25
beacon_chain/eth1/el_conf.nim
View File

@ -1,10 +1,16 @@
import
std/[options, strutils, uri],
stew/results, chronicles, confutils,
confutils/toml/defs as confTomlDefs,
confutils/toml/std/net as confTomlNet,
confutils/toml/std/uri as confTomlUri,
json_serialization, # for logging
toml_serialization, toml_serialization/lexer,
../spec/engine_authentication
export
toml_serialization, confTomlDefs, confTomlNet, confTomlUri
type
EngineApiRole* = enum
DepositSyncing = "sync-deposits"
@ -20,8 +26,8 @@ type
EngineApiUrlConfigValue* = object
url*: string # TODO: Use the URI type here
jwtSecret*: Option[string]
jwtSecretFile*: Option[InputFile]
jwtSecret* {.serializedFieldName: "jwt-secret".}: Option[string]
jwtSecretFile* {.serializedFieldName: "jwt-secret-file".}: Option[InputFile]
roles*: Option[EngineApiRoles]
const
@ -97,9 +103,9 @@ proc parseCmdArg*(T: type EngineApiUrlConfigValue, input: string): T
if uri.anchor != "":
for key, value in decodeQuery(uri.anchor):
case key
of "jwtSecret":
of "jwtSecret", "jwt-secret":
jwtSecret = some value
of "jwtSecretFile":
of "jwtSecretFile", "jwt-secret-file":
jwtSecretFile = some InputFile.parseCmdArg(value)
of "roles":
var uriRoles: EngineApiRoles = {}
@ -126,6 +132,17 @@ proc parseCmdArg*(T: type EngineApiUrlConfigValue, input: string): T
jwtSecretFile: jwtSecretFile,
roles: roles)
proc readValue*(reader: var TomlReader, value: var EngineApiUrlConfigValue)
{.raises: [Defect, SerializationError, IOError].} =
if reader.lex.readable and reader.lex.peekChar in ['\'', '"']:
# If the input is a string, we'll reuse the command-line parsing logic
value = try: parseCmdArg(EngineApiUrlConfigValue, reader.readValue(string))
except ValueError as err:
reader.lex.raiseUnexpectedValue("Valid Engine API URL expected: " & err.msg)
else:
# Else, we'll use the standard object-serializer in TOML
toml_serialization.readValue(reader, value)
proc fixupWeb3Urls*(web3Url: var string) =
var normalizedUrl = toLowerAscii(web3Url)
if not (normalizedUrl.startsWith("https://") or

16
docs/the_nimbus_book/src/eth1.md
View File

@ -104,7 +104,21 @@ You can increase the resilience of your setup and eliminate any downtime during
```
!!! tip
You can use a different secret for each connection by specifying `jwtSecret` or `jwtSecretFile` as a query parameter in the anchor section of the URL (e.g. `http://127.0.0.1:8551/#jwtSecret=0x12345...` or `http://127.0.0.1:8551/#jwtSecretFile=/tmp/jwtsecret`).
You can use a different secret for each connection by specifying `jwt-secret` or `jwt-secret-file` as a query parameter in the anchor section of the URL (e.g. `http://127.0.0.1:8551/#jwt-secret=0x12345...` or `http://127.0.0.1:8551/#jwt-secret-file=/tmp/jwtsecret`). If you use a [TOML config file](./options.html#configuration-files), you can also use the following more natural syntax:
```toml
data-dir = "my-data-dir"
rest = true
...
[[el]]
url = "http://127.0.0.1:8551"
jwt-secret-file="/path/to/jwt/file"
[[el]]
url = "http://192.168.1.2:8551"
jwt-secret = ""
```
As long as only one of execution clients remains operational and fully synced, Nimbus will keep performing all validator duties.

17
ncli/deposit_downloader.nim
View File

@ -10,15 +10,18 @@ import
type
CliFlags = object
network {.
network* {.
defaultValue: "mainnet"
name: "network".}: string
elUrls {.
elUrls* {.
name: "el".}: seq[EngineApiUrlConfigValue]
jwtSecret {.
jwtSecret* {.
name: "jwt-secret".}: Option[InputFile]
outDepositsFile {.
outDepositsFile* {.
name: "out-deposits-file".}: Option[OutFile]
configFile* {.
desc: "Loads the configuration from a TOML file"
name: "config-file" .}: Option[InputFile]
proc main(flags: CliFlags) {.async.} =
let
@ -70,4 +73,8 @@ proc main(flags: CliFlags) {.async.} =
info "All deposits downloaded"
waitFor main(load CliFlags)
waitFor main(
load(CliFlags,
secondarySources = proc (config: CliFlags, sources: auto) =
if config.configFile.isSome:
sources.addConfigFile(Toml, config.configFile.get)))

1
tests/all_tests.nim
View File

@ -24,6 +24,7 @@ import # Unit test
./test_discovery,
./test_engine_authentication,
./test_eth1_monitor,
./test_el_conf,
./test_eth2_ssz_serialization,
./test_exit_pool,
./test_forks,

163
tests/test_el_conf.nim Normal file
View File

@ -0,0 +1,163 @@
# beacon_chain
# Copyright (c) 2021-2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2, confutils,
stew/byteutils,
../beacon_chain/eth1/el_conf,
../beacon_chain/spec/engine_authentication
type
ExampleConfigFile = object
dataDir* {.name: "data-dir".}: string
el* {.name: "el".}: seq[EngineApiUrlConfigValue]
proc loadExampleConfig(content: string, cmdLine = newSeq[string]()): ExampleConfigFile =
ExampleConfigFile.load(
cmdLine = cmdLine,
secondarySources = proc (config: ExampleConfigFile, sources: auto) =
sources.addConfigFileContent(Toml, content))
const
validJwtToken = parseJwtTokenValue(
"aa95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098cc").get
suite "EL Configuration":
test "URL parsing":
let url1 = EngineApiUrlConfigValue.parseCmdArg("localhost:8484")
check:
url1.url == "localhost:8484"
url1.roles.isNone
url1.jwtSecret.isNone
url1.jwtSecretFile.isNone
let
url1Final1 = url1.toFinalUrl(some validJwtToken)
url1Final2 = url1.toFinalUrl(none seq[byte])
check:
url1Final1.isOk
url1Final1.get.url == "ws://localhost:8484"
url1Final1.get.jwtSecret.get == validJwtToken
url1Final1.get.roles == defaultEngineApiRoles
url1Final2.isOk
url1Final2.get.url == "ws://localhost:8484"
url1Final2.get.jwtSecret.isNone
url1Final2.get.roles == defaultEngineApiRoles
let url2 = EngineApiUrlConfigValue.parseCmdArg(
"https://eth-node.io:2020#jwt-secret-file=jwt.hex")
check:
url2.url == "https://eth-node.io:2020"
url2.roles.isNone
url2.jwtSecret.isNone
url2.jwtSecretFile.get.string == "jwt.hex"
let url3 = EngineApiUrlConfigValue.parseCmdArg(
"http://localhost/#roles=sync-deposits&jwt-secret=ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba")
check:
url3.url == "http://localhost/"
url3.roles == some({DepositSyncing})
url3.jwtSecret == some("ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba")
url3.jwtSecretFile.isNone
let url3Final = url3.toFinalUrl(some validJwtToken)
check:
url3Final.isOk
url3Final.get.jwtSecret.get.toHex == "ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba"
url3Final.get.roles == {DepositSyncing}
let url4 = EngineApiUrlConfigValue.parseCmdArg(
"localhost#roles=sync-deposits,validate-blocks&jwt-secret=ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba23")
check:
url4.url == "localhost"
url4.roles == some({DepositSyncing, BlockValidation})
url4.jwtSecret == some("ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba23")
url4.jwtSecretFile.isNone
let url4Final = url4.toFinalUrl(some validJwtToken)
check:
not url4Final.isOk # the JWT secret is invalid
let url5 = EngineApiUrlConfigValue.parseCmdArg(
"http://127.0.0.1:9090/#roles=sync-deposits,validate-blocks,produce-blocks,sync-deposits")
check:
url5.url == "http://127.0.0.1:9090/"
url5.roles == some({DepositSyncing, BlockValidation, BlockProduction})
url5.jwtSecret.isNone
url5.jwtSecretFile.isNone
test "Invalid URls":
template testInvalidUrl(url: string) =
expect ValueError:
echo "This URL should be invalid: ", EngineApiUrlConfigValue.parseCmdArg(url)
testInvalidUrl "http://127.0.0.1:9090/#roles="
testInvalidUrl "http://127.0.0.1:9090/#roles=sy"
testInvalidUrl "http://127.0.0.1:9090/#roles=sync-deposits,"
testInvalidUrl "http://127.0.0.1:9090/#roles=sync-deposits;validate-blocks"
testInvalidUrl "http://127.0.0.1:9090/#roles=validate-blocks,sync-deps"
test "Old style config files":
let cfg = loadExampleConfig """
data-dir = "/foo"
el = ["http://localhost:8585", "eth-data.io#roles=sync-deposits", "wss://eth-nodes.io/21312432"]
"""
check:
cfg.dataDir == "/foo"
cfg.el.len == 3
cfg.el[0].url == "http://localhost:8585"
cfg.el[1].url == "eth-data.io"
cfg.el[1].roles == some({DepositSyncing})
cfg.el[2].url == "wss://eth-nodes.io/21312432"
test "New style config files":
let cfg = loadExampleConfig """
data-dir = "my-data-dir"
[[el]]
url = "http://localhost:8585"
jwt-secret-file = "jwt.hex"
[[el]]
url = "eth-data.io"
roles = ["sync-deposits", "produce-blocks"]
[[el]]
url = "wss://eth-nodes.io/21312432"
jwt-secret = "0xee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba"
"""
check:
cfg.dataDir == "my-data-dir"
cfg.el.len == 3
cfg.el[0].url == "http://localhost:8585"
cfg.el[0].roles.isNone
cfg.el[0].jwtSecret.isNone
cfg.el[0].jwtSecretFile.get.string == "jwt.hex"
cfg.el[1].url == "eth-data.io"
cfg.el[1].roles == some({DepositSyncing, BlockProduction})
cfg.el[1].jwtSecret.isNone
cfg.el[1].jwtSecretFile.isNone
cfg.el[2].url == "wss://eth-nodes.io/21312432"
cfg.el[2].roles.isNone
cfg.el[2].jwtSecret.get == "0xee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba"
cfg.el[2].jwtSecretFile.isNone
test "Empty config file":
let cfg = loadExampleConfig("", cmdLine = @["--data-dir=foo"])
check:
cfg.dataDir == "foo"
cfg.el.len == 0

22
tests/test_eth1_monitor.nim
View File

@ -23,39 +23,19 @@ from ../beacon_chain/spec/presets import
MAX_BYTES_PER_TRANSACTION, MAX_EXTRA_DATA_BYTES, MAX_TRANSACTIONS_PER_PAYLOAD
suite "Eth1 monitor":
test "Rewrite HTTPS Infura URLs":
test "Rewrite URLs":
var
mainnetWssUrl = "wss://mainnet.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
mainnetHttpUrl = "http://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
mainnetHttpsUrl = "https://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
goerliWssUrl = "wss://goerli.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
goerliHttpUrl = "http://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
goerliHttpsUrl = "https://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
gethHttpUrl = "http://localhost:8545"
gethHttpsUrl = "https://localhost:8545"
gethWsUrl = "ws://localhost:8545"
unspecifiedProtocolUrl = "localhost:8545"
fixupWeb3Urls mainnetWssUrl
fixupWeb3Urls mainnetHttpUrl
fixupWeb3Urls mainnetHttpsUrl
fixupWeb3Urls goerliWssUrl
fixupWeb3Urls goerliHttpUrl
fixupWeb3Urls goerliHttpsUrl
fixupWeb3Urls gethHttpUrl
fixupWeb3Urls gethHttpsUrl
fixupWeb3Urls gethWsUrl
fixupWeb3Urls unspecifiedProtocolUrl
check:
mainnetWssUrl == "wss://mainnet.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
mainnetHttpUrl == "http://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
mainnetHttpsUrl == "https://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
goerliWssUrl == "wss://goerli.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
goerliHttpUrl == "http://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
goerliHttpsUrl == "https://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
gethHttpUrl == "http://localhost:8545"
gethHttpsUrl == "https://localhost:8545"
unspecifiedProtocolUrl == "ws://localhost:8545"

2
vendor/nim-confutils vendored

@ -1 +1 @@
Subproject commit 56f4db90f7923a4d6814837dda9f44c8955c52a4
Subproject commit c4c11c52ce5c5f48188069243d29b48b37e41e53

2
vendor/nim-toml-serialization vendored

@ -1 +1 @@
Subproject commit a243648f241c205b5b7fc72abb0f9c14a2812b3a
Subproject commit 86d477136f105f04bfd0dd7c0e939593d81fc581