Fluffy: Configure RPC APIs via CLI parameter (#2657)

* Support RPC API namespaces as cli parameter.

* Fluffy now uses rpcFlags on startup.

* Update testnet script to enable all RPC APIs.

* Update Fluffy book and move web3 call into eth calls.
This commit is contained in:
web3-developer 2024-09-25 22:44:46 +08:00 committed by GitHub
parent 3820b15f28
commit 69d58e8215
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 206 additions and 123 deletions

View File

@ -46,8 +46,17 @@ const
defaultBucketIpLimitDesc* = $defaultPortalProtocolConfig.tableIpLimits.bucketIpLimit
defaultBitsPerHopDesc* = $defaultPortalProtocolConfig.bitsPerHop
defaultMaxGossipNodesDesc* = $defaultPortalProtocolConfig.maxGossipNodes
defaultRpcApis* = @["eth", "portal"]
defaultRpcApisDesc* = "eth,portal"
type
RpcFlag* {.pure.} = enum
eth
debug
portal
portal_debug
discovery
TrustedDigest* = MDigest[32 * 8]
PortalCmd* = enum
@ -179,10 +188,6 @@ type
desc: "Enable the HTTP JSON-RPC server", defaultValue: false, name: "rpc"
.}: bool
rpcPort* {.
desc: "Port for the HTTP JSON-RPC server", defaultValue: 8545, name: "rpc-port"
.}: Port
rpcAddress* {.
desc: "Listening address of the RPC server",
defaultValue: defaultAdminListenAddress,
@ -190,6 +195,18 @@ type
name: "rpc-address"
.}: IpAddress
rpcPort* {.
desc: "Port for the HTTP JSON-RPC server", defaultValue: 8545, name: "rpc-port"
.}: Port
rpcApi* {.
desc:
"Enable specific set of RPC APIs (available: eth, debug, portal, portal_debug, discovery)",
defaultValue: defaultRpcApis,
defaultValueDesc: $defaultRpcApisDesc,
name: "rpc-api"
.}: seq[string]
wsEnabled* {.
desc: "Enable the WebSocket JSON-RPC server", defaultValue: false, name: "ws"
.}: bool
@ -367,3 +384,41 @@ chronicles.formatIt(OutDir):
$it
chronicles.formatIt(InputFile):
$it
func processList(v: string, o: var seq[string]) =
## Process comma-separated list of strings.
if len(v) > 0:
for n in v.split({' ', ','}):
if len(n) > 0:
o.add(n)
iterator repeatingList(listOfList: openArray[string]): string =
for strList in listOfList:
var list = newSeq[string]()
processList(strList, list)
for item in list:
yield item
proc getRpcFlags*(rpcApis: openArray[string]): set[RpcFlag] =
if rpcApis.len == 0:
error "No RPC APIs specified"
quit QuitFailure
var rpcFlags: set[RpcFlag]
for apiStr in rpcApis.repeatingList():
case apiStr.toLowerAscii()
of "eth":
rpcFlags.incl RpcFlag.eth
of "debug":
rpcFlags.incl RpcFlag.debug
of "portal":
rpcFlags.incl RpcFlag.portal
of "portal_debug":
rpcFlags.incl RpcFlag.portal_debug
of "discovery":
rpcFlags.incl RpcFlag.discovery
else:
error "Unknown RPC API: ", name = apiStr
quit QuitFailure
rpcFlags

View File

@ -103,7 +103,7 @@ Run Fluffy and trigger the propagation of data with the
`portal_history_propagateEpochRecords` JSON-RPC API call:
```bash
./build/fluffy --rpc
./build/fluffy --rpc --rpc-api:portal,portal_debug
# From another terminal
curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_history_propagateEpochRecords","params":["./user_data_dir/"]}' http://localhost:8545 | jq
@ -116,7 +116,7 @@ accumulators are available on the history network:
Make sure you still have a fluffy instance running, if not run:
```bash
./build/fluffy --rpc
./build/fluffy --rpc --rpc-api:portal,portal_debug
```
Run the `content_verifier` tool and see if all epoch accumulators are found:
@ -146,7 +146,7 @@ This will store blocks 1 to 10 into a json file located at
`portal_history_propagate` JSON-RPC API call:
```bash
./build/fluffy --rpc
./build/fluffy --rpc --rpc-api:portal,portal_debug
# From another shell
curl -s -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":"1","method":"portal_history_propagate","params":["./user_data_dir/eth-history-data.json"]}' http://localhost:8545 | jq

View File

@ -16,7 +16,7 @@ First build Fluffy as explained [here](./quick-start.md#build-the-fluffy-client)
Next run it with the JSON-RPC server enabled:
```bash
./build/fluffy --rpc --bootstrap-node:enr:<base64 encoding of ENR>
./build/fluffy --rpc --rpc-api:portal,discovery --bootstrap-node:enr:<base64 encoding of ENR>
```
### Testing Discovery v5 Layer

View File

@ -220,30 +220,39 @@ proc run(
## Start the JSON-RPC APIs
let rpcFlags = getRpcFlags(config.rpcApi)
proc setupRpcServer(
rpcServer: RpcHttpServer | RpcWebSocketServer
) {.raises: [CatchableError].} =
rpcServer.installDiscoveryApiHandlers(d)
if node.stateNetwork.isSome():
rpcServer.installDebugApiHandlers(node.stateNetwork)
rpcServer.installPortalApiHandlers(
node.stateNetwork.value.portalProtocol, "state"
)
if node.historyNetwork.isSome():
rpcServer.installEthApiHandlers(
node.historyNetwork.value, node.beaconLightClient, node.stateNetwork
)
rpcServer.installPortalApiHandlers(
node.historyNetwork.value.portalProtocol, "history"
)
rpcServer.installPortalDebugApiHandlers(
node.historyNetwork.value.portalProtocol, "history"
)
if node.beaconNetwork.isSome():
rpcServer.installPortalApiHandlers(
node.beaconNetwork.value.portalProtocol, "beacon"
)
for rpcFlag in rpcFlags:
case rpcFlag
of RpcFlag.eth:
rpcServer.installEthApiHandlers(
node.historyNetwork, node.beaconLightClient, node.stateNetwork
)
of RpcFlag.debug:
rpcServer.installDebugApiHandlers(node.stateNetwork)
of RpcFlag.portal:
if node.historyNetwork.isSome():
rpcServer.installPortalApiHandlers(
node.historyNetwork.value.portalProtocol, "history"
)
if node.beaconNetwork.isSome():
rpcServer.installPortalApiHandlers(
node.beaconNetwork.value.portalProtocol, "beacon"
)
if node.stateNetwork.isSome():
rpcServer.installPortalApiHandlers(
node.stateNetwork.value.portalProtocol, "state"
)
of RpcFlag.portal_debug:
if node.historyNetwork.isSome():
rpcServer.installPortalDebugApiHandlers(
node.historyNetwork.value.portalProtocol, "history"
)
of RpcFlag.discovery:
rpcServer.installDiscoveryApiHandlers(d)
rpcServer.start()

View File

@ -5,6 +5,6 @@
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import json_rpc/rpcclient, ./rpc_calls/[rpc_web3_calls, rpc_eth_calls]
import json_rpc/rpcclient, ./rpc_calls/rpc_eth_calls
export rpcclient, rpc_web3_calls, rpc_eth_calls
export rpcclient, rpc_eth_calls

View File

@ -18,6 +18,7 @@ import
export eth_api_types
createRpcSigsFromNim(RpcClient):
proc web3_clientVersion(): string
proc eth_chainId(): Quantity
proc eth_getBlockByHash(data: BlockHash, fullTransactions: bool): Opt[BlockObject]
proc eth_getBlockByNumber(

View File

@ -1,13 +0,0 @@
# fluffy
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import json_rpc/rpcclient
createRpcSigsFromNim(RpcClient):
proc web3_clientVersion(): string

View File

@ -17,6 +17,11 @@ import
../common/common_utils,
../network/state/state_endpoints
template getOrRaise(stateNetwork: Opt[StateNetwork]): StateNetwork =
let sn = stateNetwork.valueOr:
raise newException(ValueError, "state sub-network not enabled")
sn
proc installDebugApiHandlers*(rpcServer: RpcServer, stateNetwork: Opt[StateNetwork]) =
rpcServer.rpc("debug_getBalanceByStateRoot") do(
data: web3Types.Address, stateRoot: web3types.Hash256
@ -27,15 +32,14 @@ proc installDebugApiHandlers*(rpcServer: RpcServer, stateNetwork: Opt[StateNetwo
## stateRoot: the state root used to search the state trie.
## Returns integer of the current balance in wei.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
let balance = (
await sn.getBalanceByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress
)
).valueOr:
raise newException(ValueError, "Unable to get balance")
let
sn = stateNetwork.getOrRaise()
balance = (
await sn.getBalanceByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress
)
).valueOr:
raise newException(ValueError, "Unable to get balance")
return balance
@ -48,15 +52,15 @@ proc installDebugApiHandlers*(rpcServer: RpcServer, stateNetwork: Opt[StateNetwo
## stateRoot: the state root used to search the state trie.
## Returns integer of the number of transactions send from this address.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
let
sn = stateNetwork.getOrRaise()
nonce = (
await sn.getTransactionCountByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress
)
).valueOr:
raise newException(ValueError, "Unable to get transaction count")
let nonce = (
await sn.getTransactionCountByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress
)
).valueOr:
raise newException(ValueError, "Unable to get transaction count")
return nonce.Quantity
rpcServer.rpc("debug_getStorageAtByStateRoot") do(
@ -69,15 +73,15 @@ proc installDebugApiHandlers*(rpcServer: RpcServer, stateNetwork: Opt[StateNetwo
## stateRoot: the state root used to search the state trie.
## Returns: the value at this storage position.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
let
sn = stateNetwork.getOrRaise()
slotValue = (
await sn.getStorageAtByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress, slot
)
).valueOr:
raise newException(ValueError, "Unable to get storage slot")
let slotValue = (
await sn.getStorageAtByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress, slot
)
).valueOr:
raise newException(ValueError, "Unable to get storage slot")
return FixedBytes[32](slotValue.toBytesBE())
rpcServer.rpc("debug_getCodeByStateRoot") do(
@ -89,15 +93,14 @@ proc installDebugApiHandlers*(rpcServer: RpcServer, stateNetwork: Opt[StateNetwo
## stateRoot: the state root used to search the state trie.
## Returns the code from the given address.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
let bytecode = (
await sn.getCodeByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress
)
).valueOr:
raise newException(ValueError, "Unable to get code")
let
sn = stateNetwork.getOrRaise()
bytecode = (
await sn.getCodeByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress
)
).valueOr:
raise newException(ValueError, "Unable to get code")
return bytecode.asSeq()
@ -112,15 +115,14 @@ proc installDebugApiHandlers*(rpcServer: RpcServer, stateNetwork: Opt[StateNetwo
## stateRoot: the state root used to search the state trie.
## Returns: the proof response containing the account, account proof and storage proof
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
let proofs = (
await sn.getProofsByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress, slots
)
).valueOr:
raise newException(ValueError, "Unable to get proofs")
let
sn = stateNetwork.getOrRaise()
proofs = (
await sn.getProofsByStateRoot(
KeccakHash.fromBytes(stateRoot.bytes()), data.EthAddress, slots
)
).valueOr:
raise newException(ValueError, "Unable to get proofs")
var storageProof = newSeqOfCap[StorageProof](slots.len)
for i, slot in slots:

View File

@ -116,9 +116,24 @@ func init*(
blockObject
template getOrRaise(historyNetwork: Opt[HistoryNetwork]): HistoryNetwork =
let hn = historyNetwork.valueOr:
raise newException(ValueError, "history sub-network not enabled")
hn
template getOrRaise(beaconLightClient: Opt[LightClient]): LightClient =
let sn = beaconLightClient.valueOr:
raise newException(ValueError, "beacon sub-network not enabled")
sn
template getOrRaise(stateNetwork: Opt[StateNetwork]): StateNetwork =
let sn = stateNetwork.valueOr:
raise newException(ValueError, "state sub-network not enabled")
sn
proc installEthApiHandlers*(
rpcServer: RpcServer,
historyNetwork: HistoryNetwork,
historyNetwork: Opt[HistoryNetwork],
beaconLightClient: Opt[LightClient],
stateNetwork: Opt[StateNetwork],
) =
@ -141,8 +156,9 @@ proc installEthApiHandlers*(
##
## Returns BlockObject or nil when no block was found.
let
hn = historyNetwork.getOrRaise()
blockHash = data.toHash()
(header, body) = (await historyNetwork.getBlock(blockHash)).valueOr:
(header, body) = (await hn.getBlock(blockHash)).valueOr:
return Opt.none(BlockObject)
return Opt.some(BlockObject.init(header, body, fullTransactions))
@ -150,6 +166,8 @@ proc installEthApiHandlers*(
rpcServer.rpc("eth_getBlockByNumber") do(
quantityTag: RtBlockIdentifier, fullTransactions: bool
) -> Opt[BlockObject]:
let hn = historyNetwork.getOrRaise()
if quantityTag.kind == bidAlias:
let tag = quantityTag.alias.toLowerAscii
case tag
@ -162,28 +180,26 @@ proc installEthApiHandlers*(
of "earliest":
raise newException(ValueError, "Earliest tag not yet implemented")
of "safe":
if beaconLightClient.isNone():
raise newException(ValueError, "Safe tag not yet implemented")
let blc = beaconLightClient.getOrRaise()
withForkyStore(beaconLightClient.value().store[]):
withForkyStore(blc.store[]):
when lcDataFork > LightClientDataFork.Altair:
let
blockHash = forkyStore.optimistic_header.execution.block_hash
(header, body) = (await historyNetwork.getBlock(blockHash)).valueOr:
(header, body) = (await hn.getBlock(blockHash)).valueOr:
return Opt.none(BlockObject)
return Opt.some(BlockObject.init(header, body, fullTransactions))
else:
raise newException(ValueError, "Not available before Capella - not synced?")
of "finalized":
if beaconLightClient.isNone():
raise newException(ValueError, "Finalized tag not yet implemented")
let blc = beaconLightClient.getOrRaise()
withForkyStore(beaconLightClient.value().store[]):
withForkyStore(blc.store[]):
when lcDataFork > LightClientDataFork.Altair:
let
blockHash = forkyStore.finalized_header.execution.block_hash
(header, body) = (await historyNetwork.getBlock(blockHash)).valueOr:
(header, body) = (await hn.getBlock(blockHash)).valueOr:
return Opt.none(BlockObject)
return Opt.some(BlockObject.init(header, body, fullTransactions))
@ -196,7 +212,7 @@ proc installEthApiHandlers*(
else:
let
blockNumber = quantityTag.number.uint64
(header, body) = (await historyNetwork.getBlock(blockNumber)).valueOr:
(header, body) = (await hn.getBlock(blockNumber)).valueOr:
return Opt.none(BlockObject)
return Opt.some(BlockObject.init(header, body, fullTransactions))
@ -210,8 +226,9 @@ proc installEthApiHandlers*(
## data: hash of a block
## Returns integer of the number of transactions in this block.
let
hn = historyNetwork.getOrRaise()
blockHash = data.toHash()
(_, body) = (await historyNetwork.getBlock(blockHash)).valueOr:
(_, body) = (await hn.getBlock(blockHash)).valueOr:
raise newException(ValueError, "Could not find block with requested hash")
var txCount: uint = 0
@ -236,19 +253,20 @@ proc installEthApiHandlers*(
"Unsupported query: Only `blockHash` queries are currently supported",
)
let hash = ethHash filterOptions.blockHash.unsafeGet()
let header = (await historyNetwork.getVerifiedBlockHeader(hash)).valueOr:
raise newException(ValueError, "Could not find header with requested hash")
let
hn = historyNetwork.getOrRaise()
hash = ethHash filterOptions.blockHash.unsafeGet()
header = (await hn.getVerifiedBlockHeader(hash)).valueOr:
raise newException(ValueError, "Could not find header with requested hash")
if headerBloomFilter(header, filterOptions.address, filterOptions.topics):
# TODO: These queries could be done concurrently, investigate if there
# are no assumptions about usage of concurrent queries on portal
# wire protocol level
let
body = (await historyNetwork.getBlockBody(hash, header)).valueOr:
body = (await hn.getBlockBody(hash, header)).valueOr:
raise newException(ValueError, "Could not find block body for requested hash")
receipts = (await historyNetwork.getReceipts(hash, header)).valueOr:
receipts = (await hn.getReceipts(hash, header)).valueOr:
raise newException(ValueError, "Could not find receipts for requested hash")
logs = deriveLogs(header, body.transactions, receipts)
@ -268,14 +286,16 @@ proc installEthApiHandlers*(
## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
## Returns integer of the current balance in wei.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
if quantityTag.kind == bidAlias:
# TODO: Implement
raise newException(ValueError, "tag not yet implemented")
# This endpoint requires history network to be enabled in order to look up
# the state root by block number in the call to getBalance
discard historyNetwork.getOrRaise()
let
sn = stateNetwork.getOrRaise()
blockNumber = quantityTag.number.uint64
balance = (await sn.getBalance(blockNumber, data.EthAddress)).valueOr:
raise newException(ValueError, "Unable to get balance")
@ -291,14 +311,16 @@ proc installEthApiHandlers*(
## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
## Returns integer of the number of transactions send from this address.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
if quantityTag.kind == bidAlias:
# TODO: Implement
raise newException(ValueError, "tag not yet implemented")
# This endpoint requires history network to be enabled in order to look up
# the state root by block number in the call to getTransactionCount
discard historyNetwork.getOrRaise()
let
sn = stateNetwork.getOrRaise()
blockNumber = quantityTag.number.uint64
nonce = (await sn.getTransactionCount(blockNumber, data.EthAddress)).valueOr:
raise newException(ValueError, "Unable to get transaction count")
@ -314,14 +336,16 @@ proc installEthApiHandlers*(
## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
## Returns: the value at this storage position.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
if quantityTag.kind == bidAlias:
# TODO: Implement
raise newException(ValueError, "tag not yet implemented")
# This endpoint requires history network to be enabled in order to look up
# the state root by block number in the call to getStorageAt
discard historyNetwork.getOrRaise()
let
sn = stateNetwork.getOrRaise()
blockNumber = quantityTag.number.uint64
slotValue = (await sn.getStorageAt(blockNumber, data.EthAddress, slot)).valueOr:
raise newException(ValueError, "Unable to get storage slot")
@ -336,14 +360,16 @@ proc installEthApiHandlers*(
## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
## Returns the code from the given address.
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
if quantityTag.kind == bidAlias:
# TODO: Implement
raise newException(ValueError, "tag not yet implemented")
# This endpoint requires history network to be enabled in order to look up
# the state root by block number in the call to getCode
discard historyNetwork.getOrRaise()
let
sn = stateNetwork.getOrRaise()
blockNumber = quantityTag.number.uint64
bytecode = (await sn.getCode(blockNumber, data.EthAddress)).valueOr:
raise newException(ValueError, "Unable to get code")
@ -361,14 +387,16 @@ proc installEthApiHandlers*(
## quantityTag: integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
## Returns: the proof response containing the account, account proof and storage proof
let sn = stateNetwork.valueOr:
raise newException(ValueError, "State sub-network not enabled")
if quantityTag.kind == bidAlias:
# TODO: Implement
raise newException(ValueError, "tag not yet implemented")
# This endpoint requires history network to be enabled in order to look up
# the state root by block number in the call to getProof
discard historyNetwork.getOrRaise()
let
sn = stateNetwork.getOrRaise()
blockNumber = quantityTag.number.uint64
proofs = (await sn.getProofs(blockNumber, data.EthAddress, slots)).valueOr:
raise newException(ValueError, "Unable to get proofs")

View File

@ -338,6 +338,7 @@ for NUM_NODE in $(seq 0 $(( NUM_NODES - 1 ))); do
--rpc \
--rpc-address="127.0.0.1" \
--rpc-port="$(( BASE_RPC_PORT + NUM_NODE ))" \
--rpc-api=eth,debug,portal,portal_debug,discovery \
--metrics \
--metrics-address="127.0.0.1" \
--metrics-port="$(( BASE_METRICS_PORT + NUM_NODE ))" \

View File

@ -96,11 +96,11 @@ proc runBackfillCollectBlockDataLoop(
let
blockId = blockId(currentBlockNumber)
blockObject = (await web3Client.getBlockByNumber(blockId, false)).valueOr:
error "Failed to get block", error
error "Failed to get block", error = error
await sleepAsync(1.seconds)
continue
stateDiffs = (await web3Client.getStateDiffsByBlockNumber(blockId)).valueOr:
error "Failed to get state diffs", error
error "Failed to get state diffs", error = error
await sleepAsync(1.seconds)
continue
@ -109,7 +109,7 @@ proc runBackfillCollectBlockDataLoop(
let uncleBlock = (
await web3Client.getUncleByBlockNumberAndIndex(blockId, i.Quantity)
).valueOr:
error "Failed to get uncle block", error
error "Failed to get uncle block", error = error
await sleepAsync(1.seconds)
continue
uncleBlocks.add(uncleBlock)