Simplify verified proxy (#2754)

Reuse helpers from nimbus/web3/eth to simplify verifying proxy
implementation.
This commit is contained in:
Jacek Sieka 2024-10-21 05:10:41 +02:00 committed by GitHub
parent 503dcd40c4
commit 693ad315b3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 118 additions and 300 deletions

View File

@ -339,9 +339,9 @@ proc getNextPayload(cl: CLMocker): bool =
cl.latestShouldOverrideBuilder = x.shouldOverrideBuilder
cl.latestExecutionRequests = x.executionRequests
let beaconRoot = cl.latestPayloadAttributes.parentBeaconblockRoot
let parentBeaconblockRoot = cl.latestPayloadAttributes.parentBeaconblockRoot
let requestsHash = calcRequestsHash(x.executionRequests)
let header = blockHeader(cl.latestPayloadBuilt, beaconRoot = beaconRoot, requestsHash)
let header = blockHeader(cl.latestPayloadBuilt, parentBeaconblockRoot = parentBeaconblockRoot, requestsHash)
let blockHash = header.blockHash
if blockHash != cl.latestPayloadBuilt.blockHash:
error "CLMocker: getNextPayload blockHash mismatch",

View File

@ -12,34 +12,38 @@
import
./web3_eth_conv,
web3/execution_types,
../utils/utils,
eth/common/eth_types_rlp
eth/common/eth_types_rlp,
eth/trie/ordered_trie
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
func wdRoot(list: openArray[WithdrawalV1]): Hash32
{.gcsafe, raises:[].} =
{.noSideEffect.}:
calcWithdrawalsRoot(ethWithdrawals list)
template append(w: var RlpWriter, t: TypedTransaction) =
w.appendRawBytes(distinctBase t)
func wdRoot(x: Opt[seq[WithdrawalV1]]): Opt[Hash32]
{.gcsafe, raises:[].} =
{.noSideEffect.}:
template append(w: var RlpWriter, t: WithdrawalV1) =
w.append blocks.Withdrawal(
index: distinctBase(t.index),
validatorIndex: distinctBase(t.validatorIndex),
address: t.address,
amount: distinctBase(t.amount),
)
func wdRoot(list: openArray[WithdrawalV1]): Hash32 =
orderedTrieRoot(list)
func wdRoot(x: Opt[seq[WithdrawalV1]]): Opt[Hash32] =
if x.isNone: Opt.none(Hash32)
else: Opt.some(wdRoot x.get)
func txRoot(list: openArray[Web3Tx]): Hash32
{.gcsafe, raises:[RlpError].} =
{.noSideEffect.}:
calcTxRoot(ethTxs(list))
func txRoot(list: openArray[Web3Tx]): Hash32 =
orderedTrieRoot(list)
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
{.push gcsafe, raises:[].}
func executionPayload*(blk: Block): ExecutionPayload =
ExecutionPayload(
parentHash : blk.header.parentHash,
@ -81,9 +85,9 @@ func executionPayloadV1V2*(blk: Block): ExecutionPayloadV1OrV2 =
)
func blockHeader*(p: ExecutionPayload,
beaconRoot: Opt[Hash32],
parentBeaconBlockRoot: Opt[Hash32],
requestsHash: Opt[Hash32]):
Header {.gcsafe, raises:[RlpError].} =
Header =
Header(
parentHash : p.parentHash,
ommersHash : EMPTY_UNCLE_HASH,
@ -104,7 +108,7 @@ func blockHeader*(p: ExecutionPayload,
withdrawalsRoot: wdRoot p.withdrawals,
blobGasUsed : u64(p.blobGasUsed),
excessBlobGas : u64(p.excessBlobGas),
parentBeaconBlockRoot: beaconRoot,
parentBeaconBlockRoot: parentBeaconBlockRoot,
requestsHash : requestsHash,
)
@ -117,11 +121,11 @@ func blockBody*(p: ExecutionPayload):
)
func ethBlock*(p: ExecutionPayload,
beaconRoot: Opt[Hash32],
parentBeaconBlockRoot: Opt[Hash32],
requestsHash: Opt[Hash32]):
Block {.gcsafe, raises:[RlpError].} =
Block(
header : blockHeader(p, beaconRoot, requestsHash),
header : blockHeader(p, parentBeaconBlockRoot, requestsHash),
uncles : @[],
transactions: ethTxs p.transactions,
withdrawals : ethWithdrawals p.withdrawals,

View File

@ -75,7 +75,7 @@ func ethBlob*(x: Web3ExtraData): seq[byte] =
func ethWithdrawal*(x: WithdrawalV1): common.Withdrawal =
result.index = x.index.uint64
result.validatorIndex = x.validatorIndex.uint64
result.address = x.address.Address
result.address = x.address
result.amount = x.amount.uint64
func ethWithdrawals*(list: openArray[WithdrawalV1]):

View File

@ -131,8 +131,7 @@ proc toWdList(list: openArray[Withdrawal]): seq[WithdrawalObject] =
proc populateTransactionObject*(tx: Transaction,
optionalHeader: Opt[Header] = Opt.none(Header),
txIndex: Opt[uint64] = Opt.none(uint64)): TransactionObject
{.gcsafe, raises: [ValidationError].} =
txIndex: Opt[uint64] = Opt.none(uint64)): TransactionObject =
result = TransactionObject()
result.`type` = Opt.some Quantity(tx.txType)
if optionalHeader.isSome:
@ -166,7 +165,7 @@ proc populateTransactionObject*(tx: Transaction,
result.blobVersionedHashes = Opt.some(tx.versionedHashes)
proc populateBlockObject*(header: Header, chain: CoreDbRef, fullTx: bool, isUncle = false): BlockObject
{.gcsafe, raises: [CatchableError].} =
{.gcsafe, raises: [RlpError].} =
let blockHash = header.blockHash
result = BlockObject()

View File

@ -7,51 +7,42 @@
{.push raises: [].}
import eth/common/hashes, web3/primitives, stew/keyed_queue, results, ./rpc/rpc_utils
import eth/common/hashes, web3/eth_api_types, minilru, results
## Cache for payloads received through block gossip and validated by the
## consensus light client.
## The payloads are stored in order of arrival. When the cache is full, the
## oldest payload is deleted first.
type BlockCache* = ref object
max: int
blocks: KeyedQueue[Hash32, ExecutionData]
proc `==`(x, y: Quantity): bool {.borrow, noSideEffect.}
blocks: LruCache[Hash32, BlockObject]
proc new*(T: type BlockCache, max: uint32): T =
let maxAsInt = int(max)
return
BlockCache(max: maxAsInt, blocks: KeyedQueue[Hash32, ExecutionData].init(maxAsInt))
BlockCache(blocks: LruCache[Hash32, BlockObject].init(maxAsInt))
func len*(self: BlockCache): int =
return len(self.blocks)
len(self.blocks)
func isEmpty*(self: BlockCache): bool =
return len(self.blocks) == 0
len(self.blocks) == 0
proc add*(self: BlockCache, payload: ExecutionData) =
if self.blocks.hasKey(payload.blockHash):
return
proc add*(self: BlockCache, payload: BlockObject) =
# Only add if it didn't exist before - the implementation of `latest` relies
# on this..
if payload.hash notin self.blocks:
self.blocks.put(payload.hash, payload)
if len(self.blocks) >= self.max:
discard self.blocks.shift()
proc latest*(self: BlockCache): Opt[BlockObject] =
for b in self.blocks.values:
return Opt.some(b)
Opt.none(BlockObject)
discard self.blocks.append(payload.blockHash, payload)
proc getByNumber*(self: BlockCache, number: Quantity): Opt[BlockObject] =
for b in self.blocks.values:
if b.number == number:
return Opt.some(b)
proc latest*(self: BlockCache): results.Opt[ExecutionData] =
let latestPair = ?self.blocks.last()
return Opt.some(latestPair.data)
Opt.none(BlockObject)
proc getByNumber*(self: BlockCache, number: Quantity): Opt[ExecutionData] =
var payloadResult: Opt[ExecutionData]
for payload in self.blocks.prevValues:
if payload.blockNumber == number:
payloadResult = Opt.some(payload)
break
return payloadResult
proc getPayloadByHash*(self: BlockCache, hash: Hash32): Opt[ExecutionData] =
return self.blocks.eq(hash)
proc getPayloadByHash*(self: BlockCache, hash: Hash32): Opt[BlockObject] =
self.blocks.get(hash)

View File

@ -8,24 +8,25 @@
{.push raises: [].}
import
std/[json, os, strutils],
std/[os, strutils],
chronicles,
chronos,
confutils,
eth/common/keys,
eth/common/[keys, eth_types_rlp],
json_rpc/rpcproxy,
beacon_chain/el/[el_manager, engine_api_conversions],
beacon_chain/gossip_processing/optimistic_processor,
beacon_chain/networking/network_metadata,
beacon_chain/networking/topic_params,
beacon_chain/spec/beaconstate,
beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
beacon_chain/[light_client, nimbus_binary_common, version],
../nimbus/rpc/cors,
"."/rpc/[rpc_eth_api, rpc_utils],
../nimbus/rpc/[cors, server_api_helpers],
../nimbus/beacon/payload_conv,
./rpc/rpc_eth_api,
./nimbus_verified_proxy_conf,
./block_cache
from beacon_chain/gossip_processing/block_processor import newExecutionPayload
from beacon_chain/gossip_processing/eth2_processor import toValidationResult
type OnHeaderCallback* = proc(s: cstring, t: int) {.cdecl, raises: [], gcsafe.}
@ -130,7 +131,7 @@ proc run*(
optimisticHandler = proc(
signedBlock: ForkedSignedBeaconBlock
): Future[void] {.async: (raises: [CancelledError]).} =
) {.async: (raises: [CancelledError]).} =
notice "New LC optimistic block",
opt = signedBlock.toBlockId(), wallSlot = getBeaconTime().slotOrZero
withBlck(signedBlock):
@ -139,10 +140,16 @@ proc run*(
template payload(): auto =
forkyBlck.message.body
blockCache.add(asExecutionData(payload.asEngineExecutionPayload()))
else:
discard
return
try:
# TODO parentBeaconBlockRoot / requestsHash
let blk = ethBlock(
executionPayload(payload.asEngineExecutionPayload()),
parentBeaconBlockRoot = Opt.none(Hash32),
requestsHash = Opt.none(Hash32),
)
blockCache.add(populateBlockObject(blk.header.rlpHash, blk, true))
except RlpError as exc:
debug "Invalid block received", err = exc.msg
optimisticProcessor = initOptimisticProcessor(getBeaconTime, optimisticHandler)

View File

@ -9,22 +9,14 @@
import
std/strutils,
stint,
stew/byteutils,
results,
chronicles,
json_rpc/[rpcproxy, rpcserver, rpcclient],
eth/common/accounts,
web3/[primitives, eth_api_types, eth_api],
beacon_chain/el/el_manager,
beacon_chain/networking/network_metadata,
beacon_chain/spec/forks,
./rpc_utils,
../validate_proof,
../block_cache
export forks
logScope:
topics = "verified_proxy"
@ -66,44 +58,37 @@ template checkPreconditions(proxy: VerifiedRpcProxy) =
template rpcClient(lcProxy: VerifiedRpcProxy): RpcClient =
lcProxy.proxy.getClient()
proc getPayloadByTag(
proc getBlockByTag(
proxy: VerifiedRpcProxy, quantityTag: BlockTag
): results.Opt[ExecutionData] {.raises: [ValueError].} =
): results.Opt[BlockObject] {.raises: [ValueError].} =
checkPreconditions(proxy)
let tagResult = parseQuantityTag(quantityTag)
if tagResult.isErr:
raise newException(ValueError, tagResult.error)
let tag = tagResult.get()
let tag = parseQuantityTag(quantityTag).valueOr:
raise newException(ValueError, error)
case tag.kind
of LatestBlock:
# this will always return some block, as we always checkPreconditions
return proxy.blockCache.latest
proxy.blockCache.latest
of BlockNumber:
return proxy.blockCache.getByNumber(tag.blockNumber)
proxy.blockCache.getByNumber(tag.blockNumber)
proc getPayloadByTagOrThrow(
proc getBlockByTagOrThrow(
proxy: VerifiedRpcProxy, quantityTag: BlockTag
): ExecutionData {.raises: [ValueError].} =
let tagResult = getPayloadByTag(proxy, quantityTag)
if tagResult.isErr:
): BlockObject {.raises: [ValueError].} =
getBlockByTag(proxy, quantityTag).valueOr:
raise newException(ValueError, "No block stored for given tag " & $quantityTag)
return tagResult.get()
proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) =
lcProxy.proxy.rpc("eth_chainId") do() -> Quantity:
return lcProxy.chainId
lcProxy.chainId
lcProxy.proxy.rpc("eth_blockNumber") do() -> Quantity:
## Returns the number of the most recent block.
checkPreconditions(lcProxy)
let latest = lcProxy.blockCache.latest.valueOr:
raise (ref ValueError)(msg: "Syncing")
return lcProxy.blockCache.latest.get.blockNumber
latest.number
lcProxy.proxy.rpc("eth_getBalance") do(
address: Address, quantityTag: BlockTag
@ -113,89 +98,76 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) =
# can mean different blocks and ultimatly piece received piece of state
# must by validated against correct state root
let
executionPayload = lcProxy.getPayloadByTagOrThrow(quantityTag)
blockNumber = executionPayload.blockNumber.uint64
blk = lcProxy.getBlockByTagOrThrow(quantityTag)
blockNumber = blk.number.uint64
info "Forwarding eth_getBalance call", blockNumber
let proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber))
let
proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber))
account = getAccountFromProof(
blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash,
proof.storageHash, proof.accountProof,
).valueOr:
raise newException(ValueError, error)
let accountResult = getAccountFromProof(
executionPayload.stateRoot, proof.address, proof.balance, proof.nonce,
proof.codeHash, proof.storageHash, proof.accountProof,
)
if accountResult.isOk():
return accountResult.get.balance
else:
raise newException(ValueError, accountResult.error)
account.balance
lcProxy.proxy.rpc("eth_getStorageAt") do(
address: Address, slot: UInt256, quantityTag: BlockTag
) -> UInt256:
let
executionPayload = lcProxy.getPayloadByTagOrThrow(quantityTag)
blockNumber = executionPayload.blockNumber.uint64
blk = lcProxy.getBlockByTagOrThrow(quantityTag)
blockNumber = blk.number.uint64
info "Forwarding eth_getStorageAt", blockNumber
let proof =
await lcProxy.rpcClient.eth_getProof(address, @[slot], blockId(blockNumber))
let dataResult = getStorageData(executionPayload.stateRoot, slot, proof)
if dataResult.isOk():
let slotValue = dataResult.get()
return slotValue
else:
raise newException(ValueError, dataResult.error)
getStorageData(blk.stateRoot, slot, proof).valueOr:
raise newException(ValueError, error)
lcProxy.proxy.rpc("eth_getTransactionCount") do(
address: Address, quantityTag: BlockTag
) -> Quantity:
let
executionPayload = lcProxy.getPayloadByTagOrThrow(quantityTag)
blockNumber = executionPayload.blockNumber.uint64
blk = lcProxy.getBlockByTagOrThrow(quantityTag)
blockNumber = blk.number.uint64
info "Forwarding eth_getTransactionCount", blockNumber
let proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber))
let
proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber))
let accountResult = getAccountFromProof(
executionPayload.stateRoot, proof.address, proof.balance, proof.nonce,
proof.codeHash, proof.storageHash, proof.accountProof,
)
account = getAccountFromProof(
blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash,
proof.storageHash, proof.accountProof,
).valueOr:
raise newException(ValueError, error)
if accountResult.isOk():
return Quantity(accountResult.get.nonce)
else:
raise newException(ValueError, accountResult.error)
Quantity(account.nonce)
lcProxy.proxy.rpc("eth_getCode") do(
address: Address, quantityTag: BlockTag
) -> seq[byte]:
let
executionPayload = lcProxy.getPayloadByTagOrThrow(quantityTag)
blockNumber = executionPayload.blockNumber.uint64
blk = lcProxy.getBlockByTagOrThrow(quantityTag)
blockNumber = blk.number.uint64
info "Forwarding eth_getCode", blockNumber
let
proof = await lcProxy.rpcClient.eth_getProof(address, @[], blockId(blockNumber))
accountResult = getAccountFromProof(
executionPayload.stateRoot, proof.address, proof.balance, proof.nonce,
proof.codeHash, proof.storageHash, proof.accountProof,
)
if accountResult.isErr():
raise newException(ValueError, accountResult.error)
let account = accountResult.get()
account = getAccountFromProof(
blk.stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash,
proof.storageHash, proof.accountProof,
).valueOr:
raise newException(ValueError, error)
if account.codeHash == EMPTY_CODE_HASH:
# account does not have any code, return empty hex data
return @[]
info "Forwarding eth_getCode", blockNumber
let code = await lcProxy.rpcClient.eth_getCode(address, blockId(blockNumber))
if isValidCode(account, code):
@ -218,22 +190,12 @@ proc installEthApiHandlers*(lcProxy: VerifiedRpcProxy) =
lcProxy.proxy.rpc("eth_getBlockByNumber") do(
quantityTag: BlockTag, fullTransactions: bool
) -> Opt[BlockObject]:
let executionPayload = lcProxy.getPayloadByTag(quantityTag)
if executionPayload.isErr:
return Opt.none(BlockObject)
return Opt.some(asBlockObject(executionPayload.get()))
lcProxy.getBlockByTag(quantityTag)
lcProxy.proxy.rpc("eth_getBlockByHash") do(
blockHash: Hash32, fullTransactions: bool
) -> Opt[BlockObject]:
let executionPayload = lcProxy.blockCache.getPayloadByHash(blockHash)
if executionPayload.isErr:
return Opt.none(BlockObject)
return Opt.some(asBlockObject(executionPayload.get()))
lcProxy.blockCache.getPayloadByHash(blockHash)
proc new*(
T: type VerifiedRpcProxy, proxy: RpcProxy, blockCache: BlockCache, chainId: Quantity
@ -269,7 +231,7 @@ template awaitWithRetries*[T](
var errorMsg = reqType & " failed " & $retries & " times"
if f.failed:
errorMsg &= ". Last error: " & f.error.msg
raise newException(DataProviderFailure, errorMsg)
raise newException(ValueError, errorMsg)
await sleepAsync(chronos.milliseconds(retryDelayMs))
retryDelayMs *= 2

View File

@ -1,145 +0,0 @@
# nimbus_verified_proxy
# Copyright (c) 2022-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
eth/common/[base_rlp, headers_rlp, blocks, hashes],
stint,
web3/eth_api_types,
web3/engine_api_types,
../../nimbus/db/core_db
export eth_api_types, engine_api_types
type ExecutionData* = object
parentHash*: Hash32
feeRecipient*: Address
stateRoot*: Hash32
receiptsRoot*: Hash32
logsBloom*: FixedBytes[256]
prevRandao*: FixedBytes[32]
blockNumber*: Quantity
gasLimit*: Quantity
gasUsed*: Quantity
timestamp*: Quantity
extraData*: DynamicBytes[0, 32]
baseFeePerGas*: UInt256
blockHash*: Hash32
transactions*: seq[TypedTransaction]
withdrawals*: seq[WithdrawalV1]
proc asExecutionData*(payload: SomeExecutionPayload): ExecutionData =
when payload is ExecutionPayloadV1:
return ExecutionData(
parentHash: payload.parentHash,
feeRecipient: payload.feeRecipient,
stateRoot: payload.stateRoot,
receiptsRoot: payload.receiptsRoot,
logsBloom: payload.logsBloom,
prevRandao: payload.prevRandao,
blockNumber: payload.blockNumber,
gasLimit: payload.gasLimit,
gasUsed: payload.gasUsed,
timestamp: payload.timestamp,
extraData: payload.extraData,
baseFeePerGas: payload.baseFeePerGas,
blockHash: payload.blockHash,
transactions: payload.transactions,
withdrawals: @[],
)
else:
# TODO: Deal with different payload types
return ExecutionData(
parentHash: payload.parentHash,
feeRecipient: payload.feeRecipient,
stateRoot: payload.stateRoot,
receiptsRoot: payload.receiptsRoot,
logsBloom: payload.logsBloom,
prevRandao: payload.prevRandao,
blockNumber: payload.blockNumber,
gasLimit: payload.gasLimit,
gasUsed: payload.gasUsed,
timestamp: payload.timestamp,
extraData: payload.extraData,
baseFeePerGas: payload.baseFeePerGas,
blockHash: payload.blockHash,
transactions: payload.transactions,
withdrawals: payload.withdrawals,
)
proc calculateTransactionData(
items: openArray[TypedTransaction]
): (Hash32, seq[TxOrHash], uint64) =
## returns tuple composed of
## - root of transactions trie
## - list of transactions hashes
## - total size of transactions in block
var tr = newCoreDbRef(DefaultDbMemory).ctx.getGeneric()
var txHashes: seq[TxOrHash]
var txSize: uint64
for i, t in items:
let tx = distinctBase(t)
txSize = txSize + uint64(len(tx))
tr.merge(rlp.encode(uint64 i), tx).expect "merge data"
txHashes.add(txOrHash keccak256(tx))
let rootHash = tr.state(updateOk = true).expect "hash"
(rootHash, txHashes, txSize)
func blockHeaderSize(payload: ExecutionData, txRoot: Hash32): uint64 =
let header = Header(
parentHash: payload.parentHash,
ommersHash: EMPTY_UNCLE_HASH,
coinbase: payload.feeRecipient,
stateRoot: payload.stateRoot,
transactionsRoot: txRoot,
receiptsRoot: payload.receiptsRoot,
logsBloom: payload.logsBloom,
difficulty: default(DifficultyInt),
number: distinctBase(payload.blockNumber),
gasLimit: distinctBase(payload.gasLimit),
gasUsed: distinctBase(payload.gasUsed),
timestamp: payload.timestamp.EthTime,
extraData: payload.extraData.data,
mixHash: payload.prevRandao,
nonce: default(Bytes8),
baseFeePerGas: Opt.some payload.baseFeePerGas,
)
return uint64(len(rlp.encode(header)))
proc asBlockObject*(p: ExecutionData): BlockObject {.raises: [ValueError].} =
# TODO: currently we always calculate txHashes as BlockObject does not have
# option of returning full transactions. It needs fixing at nim-web3 library
# level
let (txRoot, txHashes, txSize) = calculateTransactionData(p.transactions)
let headerSize = blockHeaderSize(p, txRoot)
let blockSize = txSize + headerSize
BlockObject(
number: p.blockNumber,
hash: p.blockHash,
parentHash: p.parentHash,
sha3Uncles: EMPTY_UNCLE_HASH,
logsBloom: p.logsBloom,
transactionsRoot: txRoot,
stateRoot: p.stateRoot,
receiptsRoot: p.receiptsRoot,
miner: p.feeRecipient,
difficulty: UInt256.zero,
extraData: fromHex(DynamicBytes[0, 4096], p.extraData.toHex),
gasLimit: p.gasLimit,
gasUsed: p.gasUsed,
timestamp: p.timestamp,
nonce: Opt.some(default(Bytes8)),
size: Quantity(blockSize),
# TODO: It does not matter what we put here after merge blocks.
# Other projects like `helios` return `0`, data providers like alchemy return
# transition difficulty. For now retruning `0` as this is a bit easier to do.
totalDifficulty: UInt256.zero,
transactions: txHashes,
uncles: @[],
baseFeePerGas: Opt.some(p.baseFeePerGas),
)