status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make portal_bridge cli options consistent and improve logging (#2437) 

- Use --portal-rpc-url as url option to connect to Portal JSON-RPC
interface, just as --web3-url for EL JSON RPC interface.
- Improve logging to know beter which call on which JSON-RPC
interface fails
This commit is contained in:
Kim De Mey 2024-07-02 11:24:41 +02:00 committed by GitHub
parent e163b69261
commit d7b849db3d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 75 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
fluffy/tools/portal_bridge/portal_bridge_beacon.nim
View File

@ -19,7 +19,7 @@ import
../../network/beacon/beacon_content,
../../rpc/portal_rpc_client,
../eth_data_exporter/cl_data_exporter,
./portal_bridge_conf
./[portal_bridge_conf, portal_bridge_common]
const restRequestsTimeout = 30.seconds
@ -29,7 +29,7 @@ proc sleepAsync(t: TimeDiff): Future[void] =
proc gossipLCBootstrapUpdate(
restClient: RestClientRef,
portalRpcClient: RpcHttpClient,
portalRpcClient: RpcClient,
trustedBlockRoot: Eth2Digest,
cfg: RuntimeConfig,
forkDigests: ref ForkDigests,
@ -75,7 +75,7 @@ proc gossipLCBootstrapUpdate(
proc gossipLCUpdates(
restClient: RestClientRef,
portalRpcClient: RpcHttpClient,
portalRpcClient: RpcClient,
startPeriod: uint64,
count: uint64,
cfg: RuntimeConfig,
@ -135,7 +135,7 @@ proc gossipLCUpdates(
proc gossipLCFinalityUpdate(
restClient: RestClientRef,
portalRpcClient: RpcHttpClient,
portalRpcClient: RpcClient,
cfg: RuntimeConfig,
forkDigests: ref ForkDigests,
): Future[Result[Slot, string]] {.async.} =
@ -182,7 +182,7 @@ proc gossipLCFinalityUpdate(
proc gossipLCOptimisticUpdate(
restClient: RestClientRef,
portalRpcClient: RpcHttpClient,
portalRpcClient: RpcClient,
cfg: RuntimeConfig,
forkDigests: ref ForkDigests,
): Future[Result[Slot, string]] {.async.} =
@ -234,23 +234,20 @@ proc runBeacon*(config: PortalBridgeConf) {.raises: [CatchableError].} =
let
(cfg, forkDigests, beaconClock) = getBeaconData()
getBeaconTime = beaconClock.getBeaconTimeFn()
portalRpcClient = newRpcHttpClient()
portalRpcClient = newRpcClientConnect(config.portalRpcUrl)
restClient = RestClientRef.new(config.restUrl).valueOr:
fatal "Cannot connect to server", error = $error
quit QuitFailure
proc backfill(
beaconRestClient: RestClientRef,
rpcAddress: string,
rpcPort: Port,
portalRpcClient: RpcClient,
backfillAmount: uint64,
trustedBlockRoot: Option[TrustedDigest],
) {.async.} =
# Bootstrap backfill, currently just one bootstrap selected by
# trusted-block-root, could become a selected list, or some other way.
if trustedBlockRoot.isSome():
await portalRpcClient.connect(rpcAddress, rpcPort, false)
let res = await gossipLCBootstrapUpdate(
beaconRestClient, portalRpcClient, trustedBlockRoot.get(), cfg, forkDigests
)
@ -274,8 +271,6 @@ proc runBeacon*(config: PortalBridgeConf) {.raises: [CatchableError].} =
leftOver = backfillAmount mod updatesPerRequest
for i in 0 ..< requestAmount:
await portalRpcClient.connect(rpcAddress, rpcPort, false)
let res = await gossipLCUpdates(
beaconRestClient,
portalRpcClient,
@ -291,8 +286,6 @@ proc runBeacon*(config: PortalBridgeConf) {.raises: [CatchableError].} =
await portalRpcClient.close()
if leftOver > 0:
await portalRpcClient.connect(rpcAddress, rpcPort, false)
let res = await gossipLCUpdates(
beaconRestClient,
portalRpcClient,
@ -340,8 +333,6 @@ proc runBeacon*(config: PortalBridgeConf) {.raises: [CatchableError].} =
# Or basically `lightClientOptimisticUpdateSlotOffset`
await sleepAsync((SECONDS_PER_SLOT div INTERVALS_PER_SLOT).int.seconds)
await portalRpcClient.connect(config.rpcAddress, Port(config.rpcPort), false)
let res =
await gossipLCOptimisticUpdate(restClient, portalRpcClient, cfg, forkDigests)
@ -394,8 +385,7 @@ proc runBeacon*(config: PortalBridgeConf) {.raises: [CatchableError].} =
timeToNextSlot = nextSlot.start_beacon_time() - getBeaconTime()
waitFor backfill(
restClient, config.rpcAddress, config.rpcPort, config.backfillAmount,
config.trustedBlockRoot,
restClient, portalRpcClient, config.backfillAmount, config.trustedBlockRoot
)
asyncSpawn runOnSlotLoop()

30
fluffy/tools/portal_bridge/portal_bridge_common.nim Normal file
View File

@ -0,0 +1,30 @@
# Fluffy
# Copyright (c) 2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import chronicles, json_rpc/rpcclient, ./portal_bridge_conf
proc newRpcClientConnect*(url: JsonRpcUrl): RpcClient =
## Instantiate a new JSON-RPC client and try to connect. Will quit on failure.
case url.kind
of HttpUrl:
let client = newRpcHttpClient()
try:
waitFor client.connect(url.value)
except CatchableError as e:
fatal "Failed to connect to JSON-RPC server", error = $e.msg, url = url.value
quit QuitFailure
client
of WsUrl:
let client = newRpcWebSocketClient()
try:
waitFor client.connect(url.value)
except CatchableError as e:
fatal "Failed to connect to JSON-RPC server", error = $e.msg, url = url.value
quit QuitFailure
client

35
fluffy/tools/portal_bridge/portal_bridge_conf.nim
View File

@ -29,13 +29,13 @@ proc defaultEra1DataDir*(): string =
type
TrustedDigest* = MDigest[32 * 8]
Web3UrlKind* = enum
JsonRpcUrlKind* = enum
HttpUrl
WsUrl
Web3Url* = object
kind*: Web3UrlKind
url*: string
JsonRpcUrl* = object
kind*: JsonRpcUrlKind
value*: string
PortalBridgeCmd* = enum
beacon = "Run a Portal bridge for the beacon network"
@ -55,18 +55,8 @@ type
name: "log-format"
.}: StdoutLogKind
# Portal JSON-RPC API server to connect to
rpcAddress* {.
desc: "Listening address of the Portal JSON-RPC server",
defaultValue: "127.0.0.1",
name: "rpc-address"
.}: string
rpcPort* {.
desc: "Listening port of the Portal JSON-RPC server",
defaultValue: 8545,
name: "rpc-port"
.}: Port
portalRpcUrl* {.desc: "Portal node JSON-RPC API URL", name: "portal-rpc-url".}:
JsonRpcUrl
case cmd* {.command, desc: "".}: PortalBridgeCmd
of PortalBridgeCmd.beacon:
@ -91,7 +81,8 @@ type
name: "trusted-block-root"
.}: Option[TrustedDigest]
of PortalBridgeCmd.history:
web3Url* {.desc: "Execution layer JSON-RPC API URL", name: "web3-url".}: Web3Url
web3Url* {.desc: "Execution layer JSON-RPC API URL", name: "web3-url".}:
JsonRpcUrl
blockVerify* {.
desc: "Verify the block header, body and receipts",
@ -128,7 +119,7 @@ type
.}: InputDir
of PortalBridgeCmd.state:
web3UrlState* {.desc: "Execution layer JSON-RPC API URL", name: "web3-url".}:
Web3Url
JsonRpcUrl
func parseCmdArg*(T: type TrustedDigest, input: string): T {.raises: [ValueError].} =
TrustedDigest.fromHex(input)
@ -136,20 +127,20 @@ func parseCmdArg*(T: type TrustedDigest, input: string): T {.raises: [ValueError
func completeCmdArg*(T: type TrustedDigest, input: string): seq[string] =
return @[]
proc parseCmdArg*(T: type Web3Url, p: string): T {.raises: [ValueError].} =
proc parseCmdArg*(T: type JsonRpcUrl, p: string): T {.raises: [ValueError].} =
let
url = parseUri(p)
normalizedScheme = url.scheme.toLowerAscii()
if (normalizedScheme == "http" or normalizedScheme == "https"):
Web3Url(kind: HttpUrl, url: p)
JsonRpcUrl(kind: HttpUrl, value: p)
elif (normalizedScheme == "ws" or normalizedScheme == "wss"):
Web3Url(kind: WsUrl, url: p)
JsonRpcUrl(kind: WsUrl, value: p)
else:
raise newException(
ValueError,
"The Web3 URL must specify one of following protocols: http/https/ws/wss",
)
proc completeCmdArg*(T: type Web3Url, val: string): seq[string] =
proc completeCmdArg*(T: type JsonRpcUrl, val: string): seq[string] =
return @[]

58
fluffy/tools/portal_bridge/portal_bridge_history.nim
View File

@ -23,7 +23,7 @@ import
../../network_metadata,
../../eth_data/[era1, history_data_ssz_e2s, history_data_seeding],
../../database/era1_db,
./portal_bridge_conf
./[portal_bridge_conf, portal_bridge_common]
from stew/objects import checkedEnumAssign
@ -127,11 +127,11 @@ proc getBlockByNumber(
try:
let res = await client.eth_getBlockByNumber(blockTag, fullTransactions)
if res.isNil:
return err("failed to get latest blockHeader")
return err("EL failed to provide requested block")
res
except CatchableError as e:
return err("JSON-RPC eth_getBlockByNumber failed: " & e.msg)
return err("EL JSON-RPC eth_getBlockByNumber failed: " & e.msg)
return ok(blck)
@ -142,9 +142,9 @@ proc getBlockReceipts(
try:
await client.eth_getBlockReceipts(blockId(blockNumber))
except CatchableError as e:
return err("JSON-RPC eth_getBlockReceipts failed: " & e.msg)
return err("EL JSON-RPC eth_getBlockReceipts failed: " & e.msg)
if res.isNone():
err("Failed getting receipts")
err("EL failed to provided requested receipts")
else:
ok(res.get())
@ -165,7 +165,7 @@ proc gossipBlockHeader(
encodedContentKeyHex, SSZ.encode(headerWithProof).toHex()
)
except CatchableError as e:
return err("JSON-RPC error: " & $e.msg)
return err("JSON-RPC portal_historyGossip failed: " & $e.msg)
info "Block header gossiped", peers, contentKey = encodedContentKeyHex
return ok()
@ -185,7 +185,7 @@ proc gossipBlockBody(
encodedContentKeyHex, SSZ.encode(body).toHex()
)
except CatchableError as e:
return err("JSON-RPC error: " & $e.msg)
return err("JSON-RPC portal_historyGossip failed: " & $e.msg)
info "Block body gossiped", peers, contentKey = encodedContentKeyHex
return ok()
@ -204,7 +204,7 @@ proc gossipReceipts(
encodedContentKeyHex, SSZ.encode(receipts).toHex()
)
except CatchableError as e:
return err("JSON-RPC error: " & $e.msg)
return err("JSON-RPC portal_historyGossip failed: " & $e.msg)
info "Receipts gossiped", peers, contentKey = encodedContentKeyHex
return ok()
@ -266,7 +266,7 @@ proc runLatestLoop(
# gossip block header
(await portalClient.gossipBlockHeader(hash, headerWithProof)).isOkOr:
error "Failed to gossip block header", error
error "Failed to gossip block header", error, hash
# For bodies & receipts to get verified, the header needs to be available
# on the network. Wait a little to get the headers propagated through
@ -275,11 +275,11 @@ proc runLatestLoop(
# gossip block body
(await portalClient.gossipBlockBody(hash, body)).isOkOr:
error "Failed to gossip block body", error
error "Failed to gossip block body", error, hash
# gossip receipts
(await portalClient.gossipReceipts(hash, portalReceipts)).isOkOr:
error "Failed to gossip receipts", error
error "Failed to gossip receipts", error, hash
# Making sure here that we poll enough times not to miss a block.
# We could also do some work without awaiting it, e.g. the gossiping or
@ -319,7 +319,7 @@ proc gossipHeadersWithProof(
contentKey.asSeq.toHex(), contentValue.toHex()
)
except CatchableError as e:
return err("JSON-RPC error: " & $e.msg)
return err("JSON-RPC portal_historyGossip failed: " & $e.msg)
info "Block header gossiped", peers, contentKey
ok()
@ -339,7 +339,7 @@ proc gossipBlockContent(
contentKey.asSeq.toHex(), contentValue.toHex()
)
except CatchableError as e:
return err("JSON-RPC error: " & $e.msg)
return err("JSON-RPC portal_historyGossip failed: " & $e.msg)
info "Block content gossiped", peers, contentKey
ok()
@ -377,7 +377,7 @@ proc runBackfillLoop(
try:
await portalClient.portal_historyGossipHeaders(eraFile)
except CatchableError as e:
error "JSON-RPC method failed", error = e.msg
error "JSON-RPC portal_historyGossipHeaders failed", error = e.msg
false
if headerRes:
@ -386,7 +386,7 @@ proc runBackfillLoop(
try:
await portalClient.portal_historyGossipBlockContent(eraFile)
except CatchableError as e:
error "JSON-RPC method failed", error = e.msg
error "JSON-RPC portal_historyGossipBlockContent failed", error = e.msg
false
if res:
error "Failed to gossip block content from era1 file", eraFile
@ -455,7 +455,7 @@ proc runBackfillLoopAuditMode(
error "Block hash mismatch", blockNumber
break headerBlock
info "Retrieved block header from Portal network"
info "Retrieved block header from Portal network", blockHash
headerSuccess = true
# body
@ -523,44 +523,28 @@ proc runBackfillLoopAuditMode(
raiseAssert "Failed to build header with proof: " & error
(await portalClient.gossipBlockHeader(blockHash, headerWithProof)).isOkOr:
error "Failed to gossip block header", error
error "Failed to gossip block header", error, blockHash
if not bodySuccess:
(
await portalClient.gossipBlockBody(
blockHash, PortalBlockBodyLegacy.fromBlockBody(body)
)
).isOkOr:
error "Failed to gossip block body", error
error "Failed to gossip block body", error, blockHash
if not receiptsSuccess:
(
await portalClient.gossipReceipts(
blockHash, PortalReceipts.fromReceipts(receipts)
)
).isOkOr:
error "Failed to gossip receipts", error
error "Failed to gossip receipts", error, blockHash
await sleepAsync(2.seconds)
proc runHistory*(config: PortalBridgeConf) =
let
portalClient = newRpcHttpClient()
# TODO: Use Web3 object?
web3Client: RpcClient =
case config.web3Url.kind
of HttpUrl:
newRpcHttpClient()
of WsUrl:
newRpcWebSocketClient()
try:
waitFor portalClient.connect(config.rpcAddress, Port(config.rpcPort), false)
except CatchableError as e:
error "Failed to connect to portal RPC", error = $e.msg
if config.web3Url.kind == HttpUrl:
try:
waitFor (RpcHttpClient(web3Client)).connect(config.web3Url.url)
except CatchableError as e:
error "Failed to connect to web3 RPC", error = $e.msg
portalClient = newRpcClientConnect(config.portalRpcUrl)
web3Client = newRpcClientConnect(config.web3Url)
if config.latest:
asyncSpawn runLatestLoop(portalClient, web3Client, config.blockVerify)

22
fluffy/tools/portal_bridge/portal_bridge_state.nim
View File

@ -7,28 +7,12 @@
{.push raises: [].}
import chronos, chronicles, ../../rpc/portal_rpc_client, ./portal_bridge_conf
import chronicles, ./[portal_bridge_conf, portal_bridge_common]
proc runState*(config: PortalBridgeConf) =
let
portalClient = newRpcHttpClient()
# TODO: Use Web3 object?
web3Client: RpcClient =
case config.web3UrlState.kind
of HttpUrl:
newRpcHttpClient()
of WsUrl:
newRpcWebSocketClient()
try:
waitFor portalClient.connect(config.rpcAddress, Port(config.rpcPort), false)
except CatchableError as e:
error "Failed to connect to portal RPC", error = $e.msg
if config.web3UrlState.kind == HttpUrl:
try:
waitFor (RpcHttpClient(web3Client)).connect(config.web3UrlState.url)
except CatchableError as e:
error "Failed to connect to web3 RPC", error = $e.msg
portalClient = newRpcClientConnect(config.portalRpcUrl)
web3Client = newRpcClientConnect(config.web3Url)
# TODO:
# Here we'd want to implement initially a loop that backfills the state