nimbus-eth1/hive_integration/nodocker/engine/clmock.nim

624 lines
22 KiB
Nim

import
std/[times, tables],
chronicles,
nimcrypto/sysrand,
stew/[byteutils, endians2],
eth/common, chronos,
json_rpc/rpcclient,
../../../nimbus/beacon/execution_types,
../../../nimbus/beacon/web3_eth_conv,
../../../nimbus/beacon/payload_conv,
../../../nimbus/[constants],
../../../nimbus/common as nimbus_common,
./client_pool,
./engine_env,
./engine_client
import web3/engine_api_types except Hash256 # conflict with the one from eth/common
# Consensus Layer Client Mock used to sync the Execution Clients once the TTD has been reached
type
CLMocker* = ref object
com: CommonRef
# Number of required slots before a block which was set as Head moves to `safe` and `finalized` respectively
slotsToSafe* : int
slotsToFinalized*: int
# Wait time before attempting to get the payload
payloadProductionClientDelay: int
# Block production related
blockTimestampIncrement*: Option[int]
# Block Production State
clients : ClientPool
nextBlockProducer* : EngineEnv
nextFeeRecipient* : EthAddress
nextPayloadID* : PayloadID
currentPayloadNumber* : uint64
# Chain History
headerHistory : Table[uint64, common.BlockHeader]
# PoS Chain History Information
prevRandaoHistory* : Table[uint64, common.Hash256]
executedPayloadHistory* : Table[uint64, ExecutionPayload]
headHashHistory : seq[BlockHash]
# Latest broadcasted data using the PoS Engine API
latestHeadNumber* : uint64
latestHeader* : common.BlockHeader
latestPayloadBuilt* : ExecutionPayload
latestBlockValue* : Option[UInt256]
latestBlobsBundle* : Option[BlobsBundleV1]
latestPayloadAttributes*: PayloadAttributes
latestExecutedPayload* : ExecutionPayload
latestForkchoice* : ForkchoiceStateV1
# Merge related
firstPoSBlockNumber : Option[uint64]
ttdReached* : bool
transitionPayloadTimestamp: Option[int]
safeSlotsToImportOptimistically: int
chainTotalDifficulty : UInt256
# Shanghai related
nextWithdrawals* : Option[seq[WithdrawalV1]]
BlockProcessCallbacks* = object
onPayloadProducerSelected* : proc(): bool {.gcsafe.}
onRequestNextPayload* : proc(): bool {.gcsafe.}
onGetPayload* : proc(): bool {.gcsafe.}
onNewPayloadBroadcast* : proc(): bool {.gcsafe.}
onForkchoiceBroadcast* : proc(): bool {.gcsafe.}
onSafeBlockChange * : proc(): bool {.gcsafe.}
onFinalizedBlockChange* : proc(): bool {.gcsafe.}
GetPayloadResponse = object
executionPayload: ExecutionPayload
blockValue: Option[UInt256]
blobsBundle: Option[BlobsBundleV1]
func latestPayloadNumber*(h: Table[uint64, ExecutionPayload]): uint64 =
result = 0'u64
for n, _ in h:
if n > result:
result = n
func latestWithdrawalsIndex*(h: Table[uint64, ExecutionPayload]): uint64 =
result = 0'u64
for n, p in h:
if p.withdrawals.isNone:
continue
let wds = p.withdrawals.get
for w in wds:
if w.index.uint64 > result:
result = w.index.uint64
func client(cl: CLMocker): RpcClient =
cl.clients.first.client
proc init(cl: CLMocker, clients: ClientPool, com: CommonRef) =
cl.clients = clients
cl.com = com
cl.slotsToSafe = 1
cl.slotsToFinalized = 2
cl.payloadProductionClientDelay = 1
cl.headerHistory[0] = com.genesisHeader()
proc newClMocker*(clients: ClientPool, com: CommonRef): CLMocker =
new result
result.init(clients, com)
proc waitForTTD*(cl: CLMocker): Future[bool] {.async.} =
let ttd = cl.com.ttd()
doAssert(ttd.isSome)
let (header, waitRes) = await cl.client.waitForTTD(ttd.get)
if not waitRes:
error "CLMocker: timeout while waiting for TTD"
return false
cl.latestHeader = header
cl.headerHistory[header.blockNumber.truncate(uint64)] = header
cl.ttdReached = true
let headerHash = BlockHash(common.blockHash(cl.latestHeader).data)
if cl.slotsToSafe == 0:
cl.latestForkchoice.safeBlockHash = headerHash
if cl.slotsToFinalized == 0:
cl.latestForkchoice.finalizedBlockHash = headerHash
# Reset transition values
cl.latestHeadNumber = cl.latestHeader.blockNumber.truncate(uint64)
cl.headHashHistory = @[]
cl.firstPoSBlockNumber = none(uint64)
# Prepare initial forkchoice, to be sent to the transition payload producer
cl.latestForkchoice = ForkchoiceStateV1()
cl.latestForkchoice.headBlockHash = headerHash
let res = cl.client.forkchoiceUpdatedV1(cl.latestForkchoice)
if res.isErr:
error "waitForTTD: forkchoiceUpdated error", msg=res.error
return false
let s = res.get()
if s.payloadStatus.status != PayloadExecutionStatus.valid:
error "waitForTTD: forkchoiceUpdated response unexpected",
expect = PayloadExecutionStatus.valid,
get = s.payloadStatus.status
return false
return true
# Check whether a block number is a PoS block
proc isBlockPoS*(cl: CLMocker, bn: common.BlockNumber): bool =
if cl.firstPoSBlockNumber.isNone:
return false
let number = cl.firstPoSBlockNumber.get()
let bn = bn.truncate(uint64)
if number > bn:
return false
return true
# Return the per-block timestamp value increment
func getTimestampIncrement(cl: CLMocker): int =
cl.blockTimestampIncrement.get(1)
# Returns the timestamp value to be included in the next payload attributes
func getNextBlockTimestamp(cl: CLMocker): int64 =
if cl.firstPoSBlockNumber.isNone and cl.transitionPayloadTimestamp.isSome:
# We are producing the transition payload and there's a value specified
# for this specific payload
return cl.transitionPayloadTimestamp.get
return cl.latestHeader.timestamp.toUnix + cl.getTimestampIncrement().int64
func setNextWithdrawals(cl: CLMocker, nextWithdrawals: Option[seq[WithdrawalV1]]) =
cl.nextWithdrawals = nextWithdrawals
func timestampToBeaconRoot(timestamp: Quantity): FixedBytes[32] =
# Generates a deterministic hash from the timestamp
let h = keccakHash(timestamp.uint64.toBytesBE)
FixedBytes[32](h.data)
func isShanghai(cl: CLMocker, timestamp: Quantity): bool =
let ts = fromUnix(timestamp.int64)
cl.com.isShanghaiOrLater(ts)
func isCancun(cl: CLMocker, timestamp: Quantity): bool =
let ts = fromUnix(timestamp.int64)
cl.com.isCancunOrLater(ts)
func V1(attr: Option[PayloadAttributes]): Option[PayloadAttributesV1] =
if attr.isNone:
return none(PayloadAttributesV1)
some(attr.get.V1)
when false:
func V2(attr: Option[PayloadAttributes]): Option[PayloadAttributesV2] =
if attr.isNone:
return none(PayloadAttributesV2)
some(attr.get.V2)
func V3(attr: Option[PayloadAttributes]): Option[PayloadAttributesV3] =
if attr.isNone:
return none(PayloadAttributesV3)
some(attr.get.V3)
proc fcu(cl: CLMocker, version: Version,
update: ForkchoiceStateV1,
attr: Option[PayloadAttributes]):
Result[ForkchoiceUpdatedResponse, string] =
let client = cl.nextBlockProducer.client
case version
of Version.V1: client.forkchoiceUpdatedV1(update, attr.V1)
of Version.V2: client.forkchoiceUpdatedV2(update, attr)
of Version.V3: client.forkchoiceUpdatedV3(update, attr)
# Picks the next payload producer from the set of clients registered
proc pickNextPayloadProducer(cl: CLMocker): bool =
doAssert cl.clients.len != 0
for i in 0 ..< cl.clients.len:
# Get a client to generate the payload
let id = (cl.latestHeadNumber.int + i) mod cl.clients.len
cl.nextBlockProducer = cl.clients[id]
# Get latest header. Number and hash must coincide with our view of the chain,
# and only then we can build on top of this client's chain
var latestHeader: common.BlockHeader
let res = cl.nextBlockProducer.client.latestHeader(latestHeader)
if res.isErr:
error "CLMocker: Could not get latest block header while selecting client for payload production",
msg=res.error
return false
let lastBlockHash = latestHeader.blockHash
if cl.latestHeader.blockHash != lastBlockHash or
cl.latestHeadNumber != latestHeader.blockNumber.truncate(uint64):
# Selected client latest block hash does not match canonical chain, try again
cl.nextBlockProducer = nil
continue
else:
break
doAssert cl.nextBlockProducer != nil
return true
proc requestNextPayload(cl: CLMocker): bool =
# Generate a random value for the PrevRandao field
var nextPrevRandao: common.Hash256
doAssert randomBytes(nextPrevRandao.data) == 32
let timestamp = Quantity cl.getNextBlockTimestamp.uint64
cl.latestPayloadAttributes = PayloadAttributes(
timestamp: timestamp,
prevRandao: FixedBytes[32] nextPrevRandao.data,
suggestedFeeRecipient: Address cl.nextFeeRecipient,
)
if cl.isShanghai(timestamp):
cl.latestPayloadAttributes.withdrawals = cl.nextWithdrawals
if cl.isCancun(timestamp):
# Write a deterministic hash based on the block number
let beaconRoot = timestampToBeaconRoot(timestamp)
cl.latestPayloadAttributes.parentBeaconBlockRoot = some(beaconRoot)
# Save random value
let number = cl.latestHeader.blockNumber.truncate(uint64) + 1
cl.prevRandaoHistory[number] = nextPrevRandao
let version = cl.latestPayloadAttributes.version
let res = cl.fcu(version, cl.latestForkchoice, some(cl.latestPayloadAttributes))
if res.isErr:
error "CLMocker: Could not send forkchoiceUpdated", version=version, msg=res.error
return false
let s = res.get()
if s.payloadStatus.status != PayloadExecutionStatus.valid:
error "CLMocker: Unexpected forkchoiceUpdated Response from Payload builder",
status=s.payloadStatus.status
return false
if s.payloadStatus.latestValidHash.isNone or s.payloadStatus.latestValidHash.get != cl.latestForkchoice.headBlockHash:
error "CLMocker: Unexpected forkchoiceUpdated LatestValidHash Response from Payload builder",
latest=s.payloadStatus.latestValidHash,
head=cl.latestForkchoice.headBlockHash
return false
doAssert s.payLoadID.isSome
cl.nextPayloadID = s.payloadID.get()
return true
proc getPayload(cl: CLMocker, payloadId: PayloadID): Result[GetPayloadResponse, string] =
let ts = cl.latestPayloadAttributes.timestamp
let client = cl.nextBlockProducer.client
if cl.isCancun(ts):
let res = client.getPayloadV3(payloadId)
if res.isErr:
return err(res.error)
let x = res.get
return ok(GetPayloadResponse(
executionPayload: executionPayload(x.executionPayload),
blockValue: some(x.blockValue),
blobsBundle: some(x.blobsBundle)
))
if cl.isShanghai(ts):
let res = client.getPayloadV2(payloadId)
if res.isErr:
return err(res.error)
let x = res.get
return ok(GetPayloadResponse(
executionPayload: executionPayload(x.executionPayload),
blockValue: some(x.blockValue)
))
let res = client.getPayloadV1(payloadId)
if res.isErr:
return err(res.error)
return ok(GetPayloadResponse(
executionPayload: executionPayload(res.get),
))
proc getNextPayload(cl: CLMocker): bool =
let res = cl.getPayload(cl.nextPayloadID)
if res.isErr:
error "CLMocker: Could not getPayload",
payloadID=toHex(cl.nextPayloadID)
return false
let x = res.get()
cl.latestPayloadBuilt = x.executionPayload
cl.latestBlockValue = x.blockValue
cl.latestBlobsBundle = x.blobsBundle
let beaconRoot = ethHash cl.latestPayloadAttributes.parentBeaconblockRoot
let header = blockHeader(cl.latestPayloadBuilt, beaconRoot)
let blockHash = w3Hash header.blockHash
if blockHash != cl.latestPayloadBuilt.blockHash:
error "CLMocker: getNextPayload blockHash mismatch",
expected=cl.latestPayloadBuilt.blockHash,
get=blockHash.toHex
return false
if cl.latestPayloadBuilt.timestamp != cl.latestPayloadAttributes.timestamp:
error "CLMocker: Incorrect Timestamp on payload built",
expect=cl.latestPayloadBuilt.timestamp.uint64,
get=cl.latestPayloadAttributes.timestamp.uint64
return false
if cl.latestPayloadBuilt.feeRecipient != cl.latestPayloadAttributes.suggestedFeeRecipient:
error "CLMocker: Incorrect SuggestedFeeRecipient on payload built",
expect=cl.latestPayloadBuilt.feeRecipient,
get=cl.latestPayloadAttributes.suggestedFeeRecipient
return false
if cl.latestPayloadBuilt.prevRandao != cl.latestPayloadAttributes.prevRandao:
error "CLMocker: Incorrect PrevRandao on payload built",
expect=cl.latestPayloadBuilt.prevRandao,
get=cl.latestPayloadAttributes.prevRandao
return false
if cl.latestPayloadBuilt.parentHash != BlockHash cl.latestHeader.blockHash.data:
error "CLMocker: Incorrect ParentHash on payload built",
expect=cl.latestPayloadBuilt.parentHash,
get=cl.latestHeader.blockHash
return false
if cl.latestPayloadBuilt.blockNumber.uint64.toBlockNumber != cl.latestHeader.blockNumber + 1.toBlockNumber:
error "CLMocker: Incorrect Number on payload built",
expect=cl.latestPayloadBuilt.blockNumber.uint64,
get=cl.latestHeader.blockNumber+1.toBlockNumber
return false
return true
func versionedHashes(bb: BlobsBundleV1): seq[Web3Hash] =
doAssert(bb.commitments.len > 0)
result = newSeqOfCap[BlockHash](bb.commitments.len)
for com in bb.commitments:
var h = keccakHash(com.bytes)
h.data[0] = VERSIONED_HASH_VERSION_KZG
result.add w3Hash h
proc broadcastNewPayload(cl: CLMocker, payload: ExecutionPayload): Result[PayloadStatusV1, string] =
var versionedHashes: seq[Web3Hash]
if cl.latestBlobsBundle.isSome:
# Broadcast the blob bundle to all clients
versionedHashes = versionedHashes(cl.latestBlobsBundle.get)
case payload.version
of Version.V1: return cl.client.newPayloadV1(payload.V1)
of Version.V2: return cl.client.newPayloadV2(payload.V2)
of Version.V3: return cl.client.newPayloadV3(payload.V3,
versionedHashes,
cl.latestPayloadAttributes.parentBeaconBlockRoot.get)
proc broadcastNextNewPayload(cl: CLMocker): bool =
let res = cl.broadcastNewPayload(cl.latestPayloadBuilt)
if res.isErr:
error "CLMocker: broadcastNewPayload Error", msg=res.error
return false
let s = res.get()
if s.status == PayloadExecutionStatus.valid:
# The client is synced and the payload was immediately validated
# https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md:
# - If validation succeeds, the response MUST contain {status: VALID, latestValidHash: payload.blockHash}
let blockHash = cl.latestPayloadBuilt.blockHash
if s.latestValidHash.isNone:
error "CLMocker: NewPayload returned VALID status with nil LatestValidHash",
expected=blockHash.toHex
return false
let latestValidHash = s.latestValidHash.get()
if latestValidHash != BlockHash(blockHash):
error "CLMocker: NewPayload returned VALID status with incorrect LatestValidHash",
get=latestValidHash.toHex, expected=blockHash.toHex
return false
elif s.status == PayloadExecutionStatus.accepted:
# The client is not synced but the payload was accepted
# https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md:
# - {status: ACCEPTED, latestValidHash: null, validationError: null} if the following conditions are met:
# the blockHash of the payload is valid
# the payload doesn't extend the canonical chain
# the payload hasn't been fully validated.
let nullHash = BlockHash common.Hash256().data
let latestValidHash = s.latestValidHash.get(nullHash)
if s.latestValidHash.isSome and latestValidHash != nullHash:
error "CLMocker: NewPayload returned ACCEPTED status with incorrect LatestValidHash",
hash=latestValidHash.toHex
return false
else:
error "CLMocker: broadcastNewPayload Response",
status=s.status
return false
cl.latestExecutedPayload = cl.latestPayloadBuilt
let number = uint64 cl.latestPayloadBuilt.blockNumber
cl.executedPayloadHistory[number] = cl.latestPayloadBuilt
return true
proc broadcastForkchoiceUpdated(cl: CLMocker,
update: ForkchoiceStateV1): Result[ForkchoiceUpdatedResponse, string] =
let version = cl.latestExecutedPayload.version
cl.fcu(version, update, none(PayloadAttributes))
proc broadcastLatestForkchoice(cl: CLMocker): bool =
let res = cl.broadcastForkchoiceUpdated(cl.latestForkchoice)
if res.isErr:
error "CLMocker: broadcastForkchoiceUpdated Error", msg=res.error
return false
let s = res.get()
if s.payloadStatus.status != PayloadExecutionStatus.valid:
error "CLMocker: broadcastForkchoiceUpdated Response",
status=s.payloadStatus.status
return false
if s.payloadStatus.latestValidHash.get != cl.latestForkchoice.headBlockHash:
error "CLMocker: Incorrect LatestValidHash from ForkchoiceUpdated",
get=s.payloadStatus.latestValidHash.get.toHex,
expect=cl.latestForkchoice.headBlockHash.toHex
if s.payloadStatus.validationError.isSome:
error "CLMocker: Expected empty validationError",
msg=s.payloadStatus.validationError.get
if s.payloadID.isSome:
error "CLMocker: Expected empty PayloadID",
msg=s.payloadID.get.toHex
return true
proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe.} =
doAssert(cl.ttdReached)
cl.currentPayloadNumber = cl.latestHeader.blockNumber.truncate(uint64) + 1'u64
if not cl.pickNextPayloadProducer():
return false
# Check if next withdrawals necessary, test can override this value on
# `OnPayloadProducerSelected` callback
if cl.nextWithdrawals.isNone:
var nw: seq[WithdrawalV1]
cl.setNextWithdrawals(some(nw))
if cb.onPayloadProducerSelected != nil:
if not cb.onPayloadProducerSelected():
return false
if not cl.requestNextPayload():
return false
cl.setNextWithdrawals(none(seq[WithdrawalV1]))
if cb.onRequestNextPayload != nil:
if not cb.onRequestNextPayload():
return false
# Give the client a delay between getting the payload ID and actually retrieving the payload
#time.Sleep(PayloadProductionClientDelay)
if not cl.getNextPayload():
return false
if cb.onGetPayload != nil:
if not cb.onGetPayload():
return false
if not cl.broadcastNextNewPayload():
return false
if cb.onNewPayloadBroadcast != nil:
if not cb.onNewPayloadBroadcast():
return false
# Broadcast forkchoice updated with new HeadBlock to all clients
let previousForkchoice = cl.latestForkchoice
cl.headHashHistory.add cl.latestPayloadBuilt.blockHash
cl.latestForkchoice = ForkchoiceStateV1()
cl.latestForkchoice.headBlockHash = cl.latestPayloadBuilt.blockHash
let hhLen = cl.headHashHistory.len
if hhLen > cl.slotsToSafe:
cl.latestForkchoice.safeBlockHash = cl.headHashHistory[hhLen - cl.slotsToSafe - 1]
if hhLen > cl.slotsToFinalized:
cl.latestForkchoice.finalizedBlockHash = cl.headHashHistory[hhLen - cl.slotsToFinalized - 1]
if not cl.broadcastLatestForkchoice():
return false
if cb.onForkchoiceBroadcast != nil:
if not cb.onForkchoiceBroadcast():
return false
# Broadcast forkchoice updated with new SafeBlock to all clients
if cb.onSafeBlockChange != nil and cl.latestForkchoice.safeBlockHash != previousForkchoice.safeBlockHash:
if not cb.onSafeBlockChange():
return false
# Broadcast forkchoice updated with new FinalizedBlock to all clients
if cb.onFinalizedBlockChange != nil and cl.latestForkchoice.finalizedBlockHash != previousForkchoice.finalizedBlockHash:
if not cb.onFinalizedBlockChange():
return false
# Broadcast forkchoice updated with new FinalizedBlock to all clients
# Save the number of the first PoS block
if cl.firstPoSBlockNumber.isNone:
let number = cl.latestHeader.blockNumber.truncate(uint64) + 1
cl.firstPoSBlockNumber = some(number)
# Save the header of the latest block in the PoS chain
cl.latestHeadNumber = cl.latestHeadNumber + 1
# Check if any of the clients accepted the new payload
var newHeader: common.BlockHeader
let res = cl.client.headerByNumber(cl.latestHeadNumber, newHeader)
if res.isErr:
error "CLMock ProduceSingleBlock", msg=res.error
return false
let newHash = BlockHash newHeader.blockHash.data
if newHash != cl.latestPayloadBuilt.blockHash:
error "CLMocker: None of the clients accepted the newly constructed payload",
hash=newHash.toHex
return false
# Check that the new finalized header has the correct properties
# ommersHash == 0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347
if newHeader.ommersHash != EMPTY_UNCLE_HASH:
error "CLMocker: Client produced a new header with incorrect ommersHash", ommersHash = newHeader.ommersHash
return false
# difficulty == 0
if newHeader.difficulty != 0.u256:
error "CLMocker: Client produced a new header with incorrect difficulty", difficulty = newHeader.difficulty
return false
# mixHash == prevRandao
if newHeader.mixDigest != cl.prevRandaoHistory[cl.latestHeadNumber]:
error "CLMocker: Client produced a new header with incorrect mixHash",
get = newHeader.mixDigest.data.toHex,
expect = cl.prevRandaoHistory[cl.latestHeadNumber].data.toHex
return false
# nonce == 0x0000000000000000
if newHeader.nonce != default(BlockNonce):
error "CLMocker: Client produced a new header with incorrect nonce",
nonce = newHeader.nonce.toHex
return false
if newHeader.extraData.len > 32:
error "CLMocker: Client produced a new header with incorrect extraData (len > 32)",
len = newHeader.extraData.len
return false
cl.latestHeader = newHeader
cl.headerHistory[cl.latestHeadNumber] = cl.latestHeader
return true
# Loop produce PoS blocks by using the Engine API
proc produceBlocks*(cl: CLMocker, blockCount: int, cb: BlockProcessCallbacks): bool {.gcsafe.} =
# Produce requested amount of blocks
for i in 0..<blockCount:
if not cl.produceSingleBlock(cb):
return false
return true
proc posBlockNumber*(cl: CLMocker): uint64 =
cl.firstPoSBlockNumber.get(0'u64)