reimplement engine API rpc kiln spec v2

nimbus/db/db_chain.nim

@@ -119,6 +119,10 @@ proc getBlockHash*(self: BaseChainDB, n: BlockNumber): Hash256 {.inline.} =
   if not self.getHash(blockNumberToHashKey(n), result):
     raise newException(BlockNotFound, "No block hash for number " & $n)
 
+proc getCurrentBlockHash*(self: BaseChainDB): Hash256 =
+  if not self.getHash(blockNumberToHashKey(self.currentBlock), result):
+    result = Hash256()
+
 proc getBlockHeader*(self: BaseChainDB; n: BlockNumber, output: var BlockHeader): bool =
   ## Returns the block header with the given number in the canonical chain.
   var blockHash: Hash256
@@ -133,6 +137,15 @@ proc getBlockHeader*(self: BaseChainDB; n: BlockNumber): BlockHeader =
 proc getScore*(self: BaseChainDB; blockHash: Hash256): Uint256 =
   rlp.decode(self.db.get(blockHashToScoreKey(blockHash).toOpenArray), Uint256)
 
+proc getTd*(self: BaseChainDB; blockHash: Hash256, td: var Uint256): bool =
+  let bytes = self.db.get(blockHashToScoreKey(blockHash).toOpenArray)
+  if bytes.len == 0: return false
+  try:
+    td = rlp.decode(bytes, Uint256)
+  except RlpError:
+    return false
+  return true
+
 proc getAncestorsHashes*(self: BaseChainDB, limit: Uint256, header: BlockHeader): seq[Hash256] =
   var ancestorCount = min(header.blockNumber, limit).truncate(int)
   var h = header
@@ -332,6 +345,28 @@ iterator getReceipts*(self: BaseChainDB; receiptRoot: Hash256): Receipt =
       break
     inc receiptIdx
 
+proc readTerminalHash*(self: BaseChainDB; h: var Hash256): bool =
+  let bytes = self.db.get(terminalHashKey().toOpenArray)
+  if bytes.len == 0:
+    return false
+  try:
+    h = rlp.decode(bytes, Hash256)
+  except RlpError:
+    return false
+
+  true
+
+proc writeTerminalHash*(self: BaseChainDB; h: Hash256) =
+  self.db.put(terminalHashKey().toOpenArray, rlp.encode(h))
+
+proc currentTerminalHeader*(self: BaseChainDB; header: var BlockHeader): bool =
+  var terminalHash: Hash256
+  if not self.readTerminalHash(terminalHash):
+    return false
+  if not self.getBlockHeader(terminalHash, header):
+    return false
+  true
+
 proc persistHeaderToDb*(self: BaseChainDB; header: BlockHeader): seq[BlockHeader] =
   let isGenesis = header.parentHash == GENESIS_PARENT_HASH
   let headerHash = header.blockHash
@@ -352,10 +387,19 @@ proc persistHeaderToDb*(self: BaseChainDB; header: BlockHeader): seq[BlockHeader
   except CanonicalHeadNotFound:
     return self.setAsCanonicalChainHead(headerHash)
 
+  let ttd = self.ttd()
+  if headScore < ttd and score >= ttd:
+    self.writeTerminalHash(headerHash)
+
   if score > headScore:
     self.totalDifficulty = score
     result = self.setAsCanonicalChainHead(headerHash)
 
+proc persistHeaderToDbWithoutSetHead*(self: BaseChainDB; header: BlockHeader) =
+  let headerHash = header.blockHash
+  self.addBlockNumberToHashLookup(header)
+  self.db.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header))
+
 proc persistUncles*(self: BaseChainDB, uncles: openarray[BlockHeader]): Hash256 =
   ## Persists the list of uncles to the database.
   ## Returns the uncles hash.

nimbus/db/storage_types.nim

@@ -11,6 +11,8 @@ type
     slotHashToSlot
     contractHash
     cliqueSnapshot
+    transitionStatus
+    terminalHash
 
   DbKey* = object
     # The first byte stores the key type. The rest are key-specific values
@@ -58,6 +60,15 @@ proc cliqueSnapshotKey*(h: Hash256): DbKey {.inline.} =
   result.data[1 .. 32] = h.data
   result.dataEndPos = uint8 32
 
+proc transitionStatusKey*(): DbKey =
+  # ETH-2 Transition Status
+  result.data[0] = byte ord(transitionStatus)
+  result.dataEndPos = uint8 1
+
+proc terminalHashKey*(): DbKey =
+  result.data[0] = byte ord(terminalHash)
+  result.dataEndPos = uint8 1
+
 template toOpenArray*(k: DbKey): openarray[byte] =
   k.data.toOpenArray(0, int(k.dataEndPos))
 

nimbus/merge/merger.nim

@@ -0,0 +1,72 @@
+import
+  chronicles,
+  eth/[rlp, trie/db],
+  ../db/[storage_types, db_chain]
+
+type
+  # transitionStatus describes the status of eth1/2 transition. This switch
+  # between modes is a one-way action which is triggered by corresponding
+  # consensus-layer message.
+  TransitionStatus = object
+    leftPoW   : bool # The flag is set when the first NewHead message received
+    enteredPoS: bool # The flag is set when the first FinalisedBlock message received
+
+  # Merger is an internal help structure used to track the eth1/2 transition status.
+  # It's a common structure can be used in both full node and light client.
+  Merger* = object
+    db    : TrieDatabaseRef
+    status: TransitionStatus
+
+proc write(db: TrieDatabaseRef, status: TransitionStatus) =
+  db.put(transitionStatusKey().toOpenArray(), rlp.encode(status))
+
+proc read(db: TrieDatabaseRef, status: var TransitionStatus) =
+  var bytes = db.get(transitionStatusKey().toOpenArray())
+  if bytes.len > 0:
+    try:
+      status = rlp.decode(bytes, typeof status)
+    except:
+      error "Failed to decode transition status"
+
+proc init*(m: var Merger, db: TrieDatabaseRef) =
+  m.db = db
+  db.read(m.status)
+
+proc init*(m: var Merger, db: BaseChainDB) =
+  init(m, db.db)
+
+proc initMerger*(db: BaseChainDB): Merger =
+  result.init(db)
+
+proc initMerger*(db: TrieDatabaseRef): Merger =
+  result.init(db)
+
+# ReachTTD is called whenever the first NewHead message received
+# from the consensus-layer.
+proc reachTTD*(m: var Merger) =
+  if m.status.leftPoW:
+    return
+
+  m.status = TransitionStatus(leftPoW: true)
+  m.db.write(m.status)
+
+  info "Left PoW stage"
+
+# FinalizePoS is called whenever the first FinalisedBlock message received
+# from the consensus-layer.
+proc finalizePoS*(m: var Merger) =
+  if m.status.enteredPoS:
+    return
+
+  m.status = TransitionStatus(leftPoW: true, enteredPoS: true)
+  m.db.write(m.status)
+
+  info "Entered PoS stage"
+
+# TTDReached reports whether the chain has left the PoW stage.
+proc ttdReached*(m: Merger): bool =
+   m.status.leftPoW
+
+# PoSFinalized reports whether the chain has entered the PoS stage.
+proc posFinalized*(m: Merger): bool =
+  m.status.enteredPoS

nimbus/merge/mergetypes.nim

@@ -0,0 +1,80 @@
+import
+  web3/engine_api_types,
+  ../db/db_chain,
+  ./merger
+
+import eth/common/eth_types except BlockHeader
+
+export merger
+
+type
+  EthBlockHeader* = eth_types.BlockHeader
+
+const
+  # maxTrackedPayloads is the maximum number of prepared payloads the execution
+  # engine tracks before evicting old ones. Ideally we should only ever track the
+  # latest one; but have a slight wiggle room for non-ideal conditions.
+  MaxTrackedPayloads = 10
+
+  # maxTrackedHeaders is the maximum number of executed payloads the execution
+  # engine tracks before evicting old ones. Ideally we should only ever track the
+  # latest one; but have a slight wiggle room for non-ideal conditions.
+  MaxTrackedHeaders = 10
+
+type
+  QueueItem[T] = object
+    used: bool
+    data: T
+
+  SimpleQueue[M: static[int]; T] = object
+    list: array[M, QueueItem[T]]
+
+  PayloadItem = object
+    id: PayloadId
+    payload: ExecutionPayloadV1
+
+  HeaderItem = object
+    hash: Hash256
+    header: EthBlockHeader
+
+  EngineAPI* = ref object
+    merger*: Merger
+    payloadQueue: SimpleQueue[MaxTrackedPayloads, PayloadItem]
+    headerQueue: SimpleQueue[MaxTrackedHeaders, HeaderItem]
+
+template shiftRight[M, T](x: var SimpleQueue[M, T]) =
+  x.list[1..^1] = x.list[0..^2]
+
+proc put[M, T](x: var SimpleQueue[M, T], val: T) =
+  x.shiftRight()
+  x.list[0] = QueueItem[T](used: true, data: val)
+
+iterator items[M, T](x: SimpleQueue[M, T]): T =
+  for z in x.list:
+    if z.used:
+      yield z.data
+
+proc new*(_: type EngineAPI, db: BaseChainDB): EngineAPI =
+  new result
+  if not db.isNil:
+    result.merger.init(db)
+
+proc put*(api: EngineAPI, hash: Hash256, header: EthBlockHeader) =
+  api.headerQueue.put(HeaderItem(hash: hash, header: header))
+
+proc get*(api: EngineAPI, hash: Hash256, header: var EthBlockHeader): bool =
+  for x in api.headerQueue:
+    if x.hash == hash:
+      header = x.header
+      return true
+  false
+
+proc put*(api: EngineAPI, id: PayloadId, payload: ExecutionPayloadV1) =
+  api.payloadQueue.put(PayloadItem(id: id, payload: payload))
+
+proc get*(api: EngineAPI, id: PayloadId, payload: var ExecutionPayloadV1): bool =
+  for x in api.payloadQueue:
+    if x.id == id:
+      payload = x.payload
+      return true
+  false

nimbus/merge/mergeutils.nim

@@ -0,0 +1,96 @@
+import
+  std/[typetraits, times, strutils],
+  nimcrypto/[hash, sha2],
+  web3/engine_api_types,
+  eth/[trie, rlp, common, trie/db],
+  stew/[objects, results, byteutils],
+  ../constants,
+  ./mergetypes
+
+import eth/common/eth_types except BlockHeader
+
+proc computePayloadId*(headBlockHash: Hash256, params: PayloadAttributesV1): PayloadID =
+  var dest: Hash256
+  var ctx: sha256
+  ctx.init()
+  ctx.update(headBlockHash.data)
+  ctx.update(toBytesBE distinctBase params.timestamp)
+  ctx.update(distinctBase params.prevRandao)
+  ctx.update(distinctBase params.suggestedFeeRecipient)
+  ctx.finish dest.data
+  ctx.clear()
+  (distinctBase result)[0..7] = dest.data[0..7]
+
+template unsafeQuantityToInt64(q: Quantity): int64 =
+  int64 q
+
+template asEthHash*(hash: engine_api_types.BlockHash): Hash256 =
+  Hash256(data: distinctBase(hash))
+
+proc calcRootHashRlp*(items: openArray[seq[byte]]): Hash256 =
+  var tr = initHexaryTrie(newMemoryDB())
+  for i, t in items:
+    tr.put(rlp.encode(i), t)
+  return tr.rootHash()
+
+proc toBlockHeader*(payload: ExecutionPayloadV1): eth_types.BlockHeader =
+  let transactions = seq[seq[byte]](payload.transactions)
+  let txRoot = calcRootHashRlp(transactions)
+
+  EthBlockHeader(
+    parentHash    : payload.parentHash.asEthHash,
+    ommersHash    : EMPTY_UNCLE_HASH,
+    coinbase      : EthAddress payload.feeRecipient,
+    stateRoot     : payload.stateRoot.asEthHash,
+    txRoot        : txRoot,
+    receiptRoot   : payload.receiptsRoot.asEthHash,
+    bloom         : distinctBase(payload.logsBloom),
+    difficulty    : default(DifficultyInt),
+    blockNumber   : payload.blockNumber.distinctBase.u256,
+    gasLimit      : payload.gasLimit.unsafeQuantityToInt64,
+    gasUsed       : payload.gasUsed.unsafeQuantityToInt64,
+    timestamp     : fromUnix payload.timestamp.unsafeQuantityToInt64,
+    extraData     : bytes payload.extraData,
+    mixDigest     : payload.prevRandao.asEthHash, # EIP-4399 redefine `mixDigest` -> `prevRandao`
+    nonce         : default(BlockNonce),
+    fee           : some payload.baseFeePerGas
+  )
+
+template toHex*(x: Hash256): string =
+  toHex(x.data)
+
+template validHash*(x: Hash256): Option[BlockHash] =
+  some(BlockHash(x.data))
+
+proc validate*(header: eth_types.BlockHeader, gotHash: Hash256): Result[void, string] =
+  let wantHash = header.blockHash
+  if wantHash != gotHash:
+    return err("blockhash mismatch, want $1, got $2" % [wantHash.toHex, gotHash.toHex])
+
+  return ok()
+
+proc simpleFCU*(status: PayloadExecutionStatus): ForkchoiceUpdatedResponse =
+  ForkchoiceUpdatedResponse(payloadStatus: PayloadStatusV1(status: status))
+
+proc simpleFCU*(status: PayloadExecutionStatus, msg: string): ForkchoiceUpdatedResponse =
+  ForkchoiceUpdatedResponse(payloadStatus: PayloadStatusV1(status: status, validationError: some(msg)))
+
+proc validFCU*(id: Option[PayloadId], validHash: Hash256): ForkchoiceUpdatedResponse =
+  ForkchoiceUpdatedResponse(
+    payloadStatus: PayloadStatusV1(
+      status: PayloadExecutionStatus.valid,
+      latestValidHash: some(BlockHash validHash.data)
+    ),
+    payloadId: id
+  )
+
+proc invalidStatus*(validHash: Hash256, msg: string): PayloadStatusV1 =
+  PayloadStatusV1(
+    status: PayloadExecutionStatus.invalid,
+    latestValidHash: some(BlockHash validHash.data),
+    validationError: some(msg)
+  )
+
+proc toBlockBody*(payload: ExecutionPayloadV1): BlockBody =
+  # TODO the transactions from the payload have to be converted here
+  discard payload.transactions

nimbus/nimbus.nim

@@ -206,18 +206,12 @@ proc localServices(nimbus: NimbusNode, conf: NimbusConf,
 
     # TODO: There should be a better place to initialize this
     nimbus.chainRef.clique.authorize(conf.engineSigner, signFunc)
-
+    var initialState = EngineStopped
-    let initialSealingEngineState =
+    if chainDB.totalDifficulty > chainDB.ttd:
-      if conf.networkParams.config.terminalTotalDifficulty.isSome and
+       initialState = EnginePostMerge
-         conf.networkParams.config.terminalTotalDifficulty.get.isZero:
-        nimbus.chainRef.ttdReachedAt = some(BlockNumber.zero)
-        EnginePostMerge
-      else:
-        EngineStopped
     nimbus.sealingEngine = SealingEngineRef.new(
-      # TODO: Implement the initial state correctly
       nimbus.chainRef, nimbus.ctx, conf.engineSigner,
-      nimbus.txPool, initialSealingEngineState
+      nimbus.txPool, initialState
     )
     nimbus.sealingEngine.start()
 

nimbus/p2p/chain/chain_desc.nim

@@ -63,11 +63,6 @@ type
       ## For non-PoA networks (when `db.config.poaEngine` is `false`),
       ## this descriptor is ignored.
 
-    ttdReachedAt*: Option[BlockNumber]
-      ## The first block which difficulty was above the terminal
-      ## total difficulty. In networks with TTD=0, this would be
-      ## the very first block.
-
 {.push raises: [Defect].}
 
 # ------------------------------------------------------------------------------
@@ -81,8 +76,7 @@ func toNextFork(n: BlockNumber): uint64 =
     result = n.truncate(uint64)
 
 func isBlockAfterTtd*(c: Chain, blockHeader: BlockHeader): bool =
-  # TODO: This should be fork aware
+  c.db.totalDifficulty + blockHeader.difficulty > c.db.ttd
-  c.ttdReachedAt.isSome and blockHeader.blockNumber > c.ttdReachedAt.get
   
 func getNextFork(c: ChainConfig, fork: ChainFork): uint64 =
   let next: array[ChainFork, uint64] = [

nimbus/rpc/engine_api.nim

@@ -8,136 +8,278 @@
 # those terms.
 
 import
-  std/[typetraits, times],
+  std/[typetraits, times, strutils],
-  stew/[objects, results],
+  stew/[objects, results, byteutils],
   json_rpc/[rpcserver, errors],
-  web3/[conversions, engine_api_types],
+  web3/[conversions, engine_api_types], chronicles,
   eth/[trie, rlp, common, trie/db],
   ".."/db/db_chain,
   ".."/p2p/chain/[chain_desc, persist_blocks],
-  ".."/[sealer, utils, constants]
+  ".."/[sealer, utils, constants],
+  ".."/merge/[mergetypes, mergeutils]
 
 import eth/common/eth_types except BlockHeader
-type EthBlockHeader = eth_types.BlockHeader
-
-# TODO move this to stew/objects
-template newClone*[T: not ref](x: T): ref T =
-  # TODO not nil in return type: https://github.com/nim-lang/Nim/issues/14146
-  # TODO use only when x is a function call that returns a new instance!
-  let res = new typeof(x) # TODO safe to do noinit here?
-  res[] = x
-  res
-
-template asEthHash*(hash: engine_api_types.BlockHash): Hash256 =
-  Hash256(data: distinctBase(hash))
-
-template unsafeQuantityToInt64(q: Quantity): int64 =
-  int64 q
-
-proc calcRootHashRlp*(items: openArray[seq[byte]]): Hash256 =
-  var tr = initHexaryTrie(newMemoryDB())
-  for i, t in items:
-    tr.put(rlp.encode(i), t)
-  return tr.rootHash()
-
-proc toBlockHeader(payload: ExecutionPayloadV1): eth_types.BlockHeader =
-  discard payload.prevRandao # TODO: What should this be used for?
-
-  let transactions = seq[seq[byte]](payload.transactions)
-  let txRoot = calcRootHashRlp(transactions)
-
-  EthBlockHeader(
-    parentHash    : payload.parentHash.asEthHash,
-    ommersHash    : EMPTY_UNCLE_HASH,
-    coinbase      : EthAddress payload.feeRecipient,
-    stateRoot     : payload.stateRoot.asEthHash,
-    txRoot        : txRoot,
-    receiptRoot   : payload.receiptsRoot.asEthHash,
-    bloom         : distinctBase(payload.logsBloom),
-    difficulty    : default(DifficultyInt),
-    blockNumber   : payload.blockNumber.distinctBase.u256,
-    gasLimit      : payload.gasLimit.unsafeQuantityToInt64,
-    gasUsed       : payload.gasUsed.unsafeQuantityToInt64,
-    timestamp     : fromUnix payload.timestamp.unsafeQuantityToInt64,
-    extraData     : distinctBase payload.extraData,
-    mixDigest     : default(Hash256),
-    nonce         : default(BlockNonce),
-    fee           : some payload.baseFeePerGas
-  )
-
-proc toBlockBody(payload: ExecutionPayloadV1): BlockBody =
-  # TODO the transactions from the payload have to be converted here
-  discard payload.transactions
 
 proc setupEngineAPI*(
     sealingEngine: SealingEngineRef,
     server: RpcServer) =
 
-  var payloadsInstance = newClone(newSeq[ExecutionPayloadV1]())
+  # TODO: put it somewhere else singleton
-  template payloads: auto = payloadsInstance[]
+  let api = EngineAPI.new(sealingEngine.chain.db)
 
-  # https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.5/src/engine/specification.md#engine_getpayloadv1
+  # https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.7/src/engine/specification.md#engine_newpayloadv1
-  server.rpc("engine_getPayloadV1") do(payloadIdBytes: FixedBytes[8]) -> ExecutionPayloadV1:
+  # cannot use `params` as param name. see https:#github.com/status-im/nim-json-rpc/issues/128
-    let payloadId = uint64.fromBytesBE(distinctBase payloadIdBytes)
+  server.rpc("engine_newPayloadV1") do(payload: ExecutionPayloadV1) -> PayloadStatusV1:
-    if payloadId > payloads.len.uint64:
+    trace "Engine API request received",
-      raise (ref InvalidRequest)(code: engineApiUnknownPayload, msg: "Unknown payload")
+      meth = "newPayloadV1", number = $(distinctBase payload.blockNumber), hash = payload.blockHash.toHex
-    return payloads[int payloadId]
 
-  # https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.5/src/engine/specification.md#engine_executepayloadv1
+    var header = toBlockHeader(payload)
-  #[server.rpc("engine_executePayloadV1") do(payload: ExecutionPayloadV1) -> ExecutePayloadResponse:
+    let blockHash = payload.blockHash.asEthHash
-    # TODO
+    var res = header.validate(blockHash)
-    if payload.transactions.len > 0:
+    if res.isErr:
-      # Give us a break, a block with transcations? instructions to execute?
+      return PayloadStatusV1(status: PayloadExecutionStatus.invalid_block_hash, validationError: some(res.error))
-      # Nah, we are syncing!
-      return ExecutePayloadResponse(status: PayloadExecutionStatus.syncing)
 
-    # TODO check whether we are syncing
+    let db = sealingEngine.chain.db
 
+    # If we already have the block locally, ignore the entire execution and just
+    # return a fake success.
+    if db.getBlockHeader(blockHash, header):
+      warn "Ignoring already known beacon payload",
+        number = header.blockNumber, hash = blockHash.data.toHex
+      return PayloadStatusV1(status: PayloadExecutionStatus.valid, latestValidHash: validHash(blockHash))
+
+    # If the parent is missing, we - in theory - could trigger a sync, but that
+    # would also entail a reorg. That is problematic if multiple sibling blocks
+    # are being fed to us, and even moreso, if some semi-distant uncle shortens
+    # our live chain. As such, payload execution will not permit reorgs and thus
+    # will not trigger a sync cycle. That is fine though, if we get a fork choice
+    # update after legit payload executions.
+    var parent: eth_types.BlockHeader
+    if not db.getBlockHeader(header.parentHash, parent):
+      # Stash the block away for a potential forced forckchoice update to it
+      # at a later time.
+      api.put(blockHash, header)
+
+      # Although we don't want to trigger a sync, if there is one already in
+      # progress, try to extend if with the current payload request to relieve
+      # some strain from the forkchoice update.
+      #if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil {
+      #  log.Debug("Payload accepted for sync extension", "number", params.Number, "hash", params.BlockHash)
+      #  return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil
+
+      # Either no beacon sync was started yet, or it rejected the delivered
+      # payload as non-integratable on top of the existing sync. We'll just
+      # have to rely on the beacon client to forcefully update the head with
+      # a forkchoice update request.
+      warn "Ignoring payload with missing parent",
+        number = header.blockNumber, hash = blockHash.data.toHex, parent = header.parentHash.data.toHex
+      return PayloadStatusV1(status: PayloadExecutionStatus.accepted)
+
+    # We have an existing parent, do some sanity checks to avoid the beacon client
+    # triggering too early
     let
-      headers = [payload.toBlockHeader]
+      td  = db.getScore(header.parentHash)
-      bodies = [payload.toBlockBody]
+      ttd = db.ttd()
 
-    if rlpHash(headers[0]) != payload.blockHash.asEthHash:
+    if td < ttd:
-      return ExecutePayloadResponse(status: PayloadExecutionStatus.invalid,
+      warn "Ignoring pre-merge payload",
-                                    validationError: some "payload root doesn't match its contents")
+        number = header.blockNumber, hash = blockHash.data.toHex, td, ttd
+      return PayloadStatusV1(status: PayloadExecutionStatus.invalid_terminal_block)
 
-    if sealingEngine.chain.persistBlocks(headers, bodies) != ValidationResult.OK:
+    if header.timestamp <= parent.timestamp:
-      # TODO Provide validationError and latestValidHash
+      warn "Invalid timestamp",
-      return ExecutePayloadResponse(status: PayloadExecutionStatus.invalid)
+        parent = header.timestamp, header = header.timestamp
+      return invalidStatus(db.getCurrentBlockHash(), "Invalid timestamp")
 
-    return ExecutePayloadResponse(status: PayloadExecutionStatus.valid,
+    trace "Inserting block without sethead",
-                                  latestValidHash: some payload.blockHash)
+      hash = blockHash.data.toHex, number = header.blockNumber
+    db.persistHeaderToDbWithoutSetHead(header)
 
-  # https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.5/src/engine/specification.md#engine_forkchoiceupdatedv1
+    # We've accepted a valid payload from the beacon client. Mark the local
+    # chain transitions to notify other subsystems (e.g. downloader) of the
+    # behavioral change.
+    if not api.merger.ttdReached():
+      api.merger.reachTTD()
+      # TODO: cancel downloader
+
+    return PayloadStatusV1(status: PayloadExecutionStatus.valid, latestValidHash: validHash(blockHash))
+
+  # https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.7/src/engine/specification.md#engine_getpayloadv1
+  server.rpc("engine_getPayloadV1") do(payloadId: PayloadID) -> ExecutionPayloadV1:
+    trace "Engine API request received",
+      meth = "GetPayload", id = payloadId.toHex
+
+    var payload: ExecutionPayloadV1
+    if not api.get(payloadId, payload):
+      raise (ref InvalidRequest)(code: engineApiUnknownPayload, msg: "Unknown payload")
+      #raise newException(ValueError, "Unknown payload")
+
+    return payload
+
+  # https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.7/src/engine/specification.md#engine_exchangeTransitionConfigurationV1
+  server.rpc("engine_exchangeTransitionConfigurationV1") do(conf: TransitionConfigurationV1) -> TransitionConfigurationV1:
+    trace "Engine API request received",
+      meth = "exchangeTransitionConfigurationV1",
+      ttd = conf.terminalTotalDifficulty,
+      number = uint64(conf.terminalBlockNumber),
+      blockHash = conf.terminalBlockHash.toHex
+
+    let db = sealingEngine.chain.db
+    let ttd = db.ttd()
+
+    if conf.terminalTotalDifficulty != ttd:
+      raise newException(ValueError, "invalid ttd: EL $1 CL $2" % [$ttd, $conf.terminalTotalDifficulty])
+
+    var header: EthBlockHeader
+    let terminalBlockNumber = uint64(conf.terminalBlockNumber)
+    let terminalBlockHash = conf.terminalBlockHash.asEthHash
+    if db.currentTerminalHeader(header):
+      let headerHash = header.blockHash
+
+      if terminalBlockNumber != 0'u64 and terminalBlockNumber != header.blockNumber.truncate(uint64):
+        raise newException(ValueError, "invalid terminal block number, got $1 want $2" % [$terminalBlockNumber, $header.blockNumber])
+
+      if terminalBlockHash != Hash256() and terminalBlockHash != headerHash:
+        raise newException(ValueError, "invalid terminal block hash, got $1 want $2" % [terminalBlockHash.toHex, headerHash.data.toHex])
+
+      return TransitionConfigurationV1(
+        terminalTotalDifficulty: ttd,
+        terminalBlockHash      : BlockHash headerHash.data,
+        terminalBlockNumber    : Quantity header.blockNumber.truncate(uint64)
+      )
+
+    if terminalBlockNumber != 0:
+      raise newException(ValueError, "invalid terminal block number: $1" % [$terminalBlockNumber])
+
+    if terminalBlockHash != Hash256():
+      raise newException(ValueError, "invalid terminal block hash, no terminal header set")
+
+    return TransitionConfigurationV1(terminalTotalDifficulty: ttd)
+
+  # ForkchoiceUpdatedV1 has several responsibilities:
+  # If the method is called with an empty head block:
+  #     we return success, which can be used to check if the catalyst mode is enabled
+  # If the total difficulty was not reached:
+  #     we return INVALID
+  # If the finalizedBlockHash is set:
+  #     we check if we have the finalizedBlockHash in our db, if not we start a sync
+  # We try to set our blockchain to the headBlock
+  # If there are payloadAttributes:
+  #     we try to assemble a block with the payloadAttributes and return its payloadID
+  # https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.7/src/engine/specification.md#engine_forkchoiceupdatedv1
   server.rpc("engine_forkchoiceUpdatedV1") do(
       update: ForkchoiceStateV1,
       payloadAttributes: Option[PayloadAttributesV1]) -> ForkchoiceUpdatedResponse:
     let
       db = sealingEngine.chain.db
-      newHead = update.headBlockHash.asEthHash
+      blockHash = update.headBlockHash.asEthHash
 
-    # TODO Use the finalized block information to prune any alterantive
+    if blockHash == Hash256():
-    #      histories that are no longer relevant
+      warn "Forkchoice requested update to zero hash"
-    discard update.finalizedBlockHash
+      return simpleFCU(PayloadExecutionStatus.invalid)
 
-    # TODO Check whether we are syncing
+    # Check whether we have the block yet in our database or not. If not, we'll
+    # need to either trigger a sync, or to reject this forkchoice update for a
+    # reason.
+    var header: EthBlockHeader
+    if not db.getBlockHeader(blockHash, header):
+      # If the head hash is unknown (was not given to us in a newPayload request),
+      # we cannot resolve the header, so not much to do. This could be extended in
+      # the future to resolve from the `eth` network, but it's an unexpected case
+      # that should be fixed, not papered over.
+      if not api.get(blockHash, header):
+        warn "Forkchoice requested unknown head",
+          hash = blockHash.data.toHex
+        return simpleFCU(PayloadExecutionStatus.syncing)
 
-    if not db.setHead(newHead):
+      # Header advertised via a past newPayload request. Start syncing to it.
-      return ForkchoiceUpdatedResponse(status: ForkchoiceUpdatedStatus.syncing)
+      # Before we do however, make sure any legacy sync in switched off so we
+      # don't accidentally have 2 cycles running.
+      if not api.merger.ttdReached():
+        api.merger.reachTTD()
+        # TODO: cancel downloader
 
+      info "Forkchoice requested sync to new head",
+        number = header.blockNumber,
+        hash = blockHash.data.toHex
+
+      return simpleFCU(PayloadExecutionStatus.syncing)
+
+    # Block is known locally, just sanity check that the beacon client does not
+    # attempt to push us back to before the merge.
+    let blockNumber = header.blockNumber.truncate(uint64)
+    if header.difficulty > 0.u256 or blockNumber ==  0'u64:
+      var
+        td, ptd: DifficultyInt
+        ttd = db.ttd()
+
+      if not db.getTd(blockHash, td) or (blockNumber > 0'u64 and not db.getTd(header.parentHash, ptd)):
+        error "TDs unavailable for TTD check",
+          number = blockNumber,
+          hash = blockHash.data.toHex,
+          td = td,
+          parent = header.parentHash.data.toHex,
+          ptd = ptd
+        return simpleFCU(PayloadExecutionStatus.invalid, "TDs unavailable for TDD check")
+
+      if td < ttd or (blockNumber > 0'u64 and ptd > ttd):
+        error "Refusing beacon update to pre-merge",
+          number = blockNumber,
+          hash = blockHash.data.toHex,
+          diff = header.difficulty
+
+        return simpleFCU(PayloadExecutionStatus.invalid_terminal_block)
+
+    # If the head block is already in our canonical chain, the beacon client is
+    # probably resyncing. Ignore the update.
+    var canonHash: Hash256
+    if db.getBlockHash(header.blockNumber, canonHash) and canonHash == blockHash:
+      # TODO should this be possible?
+      # If we allow these types of reorgs, we will do lots and lots of reorgs during sync
+      warn "Reorg to previous block"
+      if not db.setHead(blockHash):
+        return simpleFCU(PayloadExecutionStatus.invalid)
+    elif not db.setHead(blockHash):
+      return simpleFCU(PayloadExecutionStatus.invalid)
+
+    # If the beacon client also advertised a finalized block, mark the local
+    # chain final and completely in PoS mode.
+    let finalizedBlockHash = update.finalizedBlockHash.asEthHash
+    if finalizedBlockHash != Hash256():
+      if not api.merger.posFinalized:
+        api.merger.finalizePoS()
+
+      # TODO: If the finalized block is not in our canonical tree, somethings wrong
+      var finalBlock: EthBlockHeader
+      if not db.getBlockHeader(finalizedBlockHash, finalBlock):
+        warn "Final block not available in database",
+          hash = finalizedBlockHash.data.toHex
+        return simpleFCU(PayloadExecutionStatus.invalid, "final block not available")
+      elif not db.getBlockHash(finalBlock.blockNumber, canonHash) or canonHash != finalizedBlockHash:
+        warn "Final block not in canonical chain",
+          number = finalBlock.blockNumber,
+          hash = finalizedBlockHash.data.toHex
+        return simpleFCU(PayloadExecutionStatus.invalid, "final block not canonical")
+
+    # If payload generation was requested, create a new block to be potentially
+    # sealed by the beacon client. The payload will be requested later, and we
+    # might replace it arbitrarilly many times in between.
     if payloadAttributes.isSome:
-      let payloadId = uint64 payloads.len
+      info "Creating new payload for sealing"
-
+      let payloadAttrs = payloadAttributes.get()
       var payload: ExecutionPayloadV1
-      let generatePayloadRes = sealingEngine.generateExecutionPayload(
+      let res = sealingEngine.generateExecutionPayload(payloadAttrs, payload)
-        payloadAttributes.get,
-        payload)
-      if generatePayloadRes.isErr:
-        raise newException(CatchableError, generatePayloadRes.error)
 
-      payloads.add payload
+      if res.isErr:
+        error "Failed to create sealing payload", err = res.error
+        return simpleFCU(PayloadExecutionStatus.invalid, res.error)
 
-      return ForkchoiceUpdatedResponse(status: ForkchoiceUpdatedStatus.success,
+      let id = computePayloadId(blockHash, payloadAttrs)
-                                       payloadId: some payloadId.toBytesBE.PayloadID)
+      api.put(id, payload)
-    else:
+
-      return ForkchoiceUpdatedResponse(status: ForkchoiceUpdatedStatus.success)]#
+      info "Created payload for sealing",
+        id = id.toHex
+
+      return validFCU(some(id), blockHash)
+
+    return validFCU(none(PayloadId), blockHash)

nimbus/sealer.nim

@@ -10,18 +10,23 @@
 
 import
   std/[times, tables, typetraits],
-  pkg/[chronos, stew/results, chronicles, eth/common, eth/keys],
+  pkg/[chronos, stew/results, chronicles, eth/common, eth/keys, eth/rlp],
-  "."/[config, db/db_chain, p2p/chain, constants, utils/header],
+  "."/[config,
+    db/db_chain,
+    p2p/chain,
+    constants,
+    utils/header],
   "."/p2p/clique/[clique_defs,
     clique_desc,
     clique_cfg,
     clique_sealer],
   ./p2p/[gaslimit, validate],
   "."/[chain_config, utils, context],
-  "."/utils/tx_pool
+  "."/utils/tx_pool,
+  "."/merge/mergetypes
 
 from web3/ethtypes as web3types import nil
-from web3/engine_api_types import ExecutionPayloadV1, PayloadAttributesV1
+from web3/engine_api_types import PayloadAttributesV1, ExecutionPayloadV1
 
 type
   EngineState* = enum
@@ -43,7 +48,7 @@ type
     signer: EthAddress
     txPool: TxPoolRef
 
-template asEthHash*(hash: Web3BlockHash): Hash256 =
+template asEthHash(hash: Web3BlockHash): Hash256 =
   Hash256(data: distinctBase(hash))
 
 proc validateSealer*(conf: NimbusConf, ctx: EthContext, chain: Chain): Result[void, string] =
@@ -135,8 +140,11 @@ proc generateBlock(engine: SealingEngineRef,
     header: res.get()
   )
 
-  if not engine.chain.isBlockAfterTtd(outBlock.header):
+  if engine.chain.isBlockAfterTtd(outBlock.header):
-    # TODO Post merge, Clique should not be executing
+    # Stop the block generator if we reach TTD
+    engine.state = EnginePostMerge
+  else:
+    # Post merge, Clique should not be executing
     let sealRes = engine.chain.clique.seal(outBlock)
     if sealRes.isErr:
       return err("error sealing block header: " & $sealRes.error)
@@ -209,6 +217,10 @@ proc sealingLoop(engine: SealingEngineRef): Future[void] {.async.} =
       error "sealing engine generateBlock error", msg=blkRes.error
       break
 
+    # if TTD reached during block generation, stop the sealer
+    if engine.state != EngineRunning:
+      break
+
     let res = engine.chain.persistBlocks([blk.header], [
       BlockBody(transactions: blk.txs, uncles: blk.uncles)
     ])
@@ -243,24 +255,26 @@ proc generateExecutionPayload*(engine: SealingEngineRef,
     BlockBody(transactions: blk.txs, uncles: blk.uncles)
   ])
 
+  if blk.header.extraData.len > 32:
+    return err "extraData length should not exceed 32 bytes"
+
   payloadRes.parentHash = Web3BlockHash blk.header.parentHash.data
   payloadRes.feeRecipient = Web3Address blk.header.coinbase
   payloadRes.stateRoot = Web3BlockHash blk.header.stateRoot.data
   payloadRes.receiptsRoot = Web3BlockHash blk.header.receiptRoot.data
   payloadRes.logsBloom = Web3Bloom blk.header.bloom
-  # TODO Check the extra data length here
-  # payloadres.extraData = web3types.DynamicBytes[256] blk.header.extraData
   payloadRes.prevRandao  = web3types.FixedBytes[32](payloadAttrs.prevRandao)
   payloadRes.blockNumber = Web3Quantity blk.header.blockNumber.truncate(uint64)
   payloadRes.gasLimit = Web3Quantity blk.header.gasLimit
   payloadRes.gasUsed = Web3Quantity blk.header.gasUsed
   payloadRes.timestamp = payloadAttrs.timestamp
-  # TODO
+  payloadres.extraData = web3types.DynamicBytes[0, 32] blk.header.extraData
-  # res.extraData
   payloadRes.baseFeePerGas = blk.header.fee.get(UInt256.zero)
   payloadRes.blockHash = Web3BlockHash rlpHash(blk.header).data
-  # TODO
+
-  # res.transactions*: seq[TypedTransaction]
+  for tx in blk.txs:
+    let txData = rlp.encode(tx)
+    payloadRes.transactions.add web3types.TypedTransaction(txData)
 
   return ok()
 

tests/all_tests.nim

@@ -43,4 +43,6 @@ cliBuilder:
           ./test_pow,
           ./test_configuration,
           ./test_keyed_queue_rlp,
-          ./test_txpool
+          ./test_txpool,
+          ./test_merge
+

tests/merge/params.json

@@ -0,0 +1,37 @@
+{
+	"config": {
+		"chainId":1,
+		"homesteadBlock":0,
+		"eip150Block":0,
+		"eip155Block":0,
+		"eip158Block":0,
+		"byzantiumBlock":0,
+		"constantinopleBlock":0,
+		"petersburgBlock":0,
+		"istanbulBlock":0,
+		"muirGlacierBlock":0,
+		"berlinBlock":0,
+		"londonBlock":0,
+		"clique": {
+			"period": 5,
+			"epoch": 30000
+		},
+		"terminalTotalDifficulty":0
+	},
+	"genesis": {
+    "nonce":"0x42",
+    "timestamp":"0x0",
+    "extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "gasLimit":"0x1C9C380",
+    "difficulty":"0x400000000",
+    "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+    "coinbase":"0x0000000000000000000000000000000000000000",
+    "alloc":{
+      "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b":{"balance":"0x6d6172697573766477000000"}
+    },
+    "number":"0x0",
+    "gasUsed":"0x0",
+    "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+    "baseFeePerGas":"0x7"
+  }
+}

tests/merge/steps.json

@@ -0,0 +1,142 @@
+[
+  {
+    "name": "Prepare a payload",
+    "method":"engine_forkchoiceUpdatedV1",
+    "params":[
+      {
+        "headBlockHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a",
+        "safeBlockHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a",
+        "finalizedBlockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"
+      },
+      {
+        "timestamp":"0x5",
+        "prevRandao":"0x0000000000000000000000000000000000000000000000000000000000000000",
+        "suggestedFeeRecipient":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"
+      }
+    ],
+    "expect": {
+      "payloadStatus":{
+        "status":"VALID",
+        "latestValidHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a",
+        "validationError":null
+      },
+      "payloadId":"0xa247243752eb10b4"
+    }
+  },
+  {
+    "name": "Get the payload",
+    "method":"engine_getPayloadV1",
+    "params":[
+      "0xa247243752eb10b4"
+    ],
+    "expect": {
+      "parentHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a",
+      "feeRecipient":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+      "stateRoot":"0xca3149fa9e37db08d1cd49c9061db1002ef1cd58db2210f2115c8c989b2bdf45",
+      "receiptsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+      "logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+      "prevRandao":"0x0000000000000000000000000000000000000000000000000000000000000000",
+      "blockNumber":"0x1",
+      "gasLimit":"0x1c95111",
+      "gasUsed":"0x0",
+      "timestamp":"0x5",
+      "extraData":"0x64616f2d686172642d666f726b",
+      "baseFeePerGas":"0x7",
+      "blockHash":"0x29671a05d0e18797905296bba15941c96edefc2aefe2240253cd33cf3eda80c0",
+      "transactions":[]
+    }
+  },
+  {
+    "name": "Execute the payload",
+    "method":"engine_newPayloadV1",
+    "params":[
+      {
+        "parentHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a",
+        "feeRecipient":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+        "stateRoot":"0xca3149fa9e37db08d1cd49c9061db1002ef1cd58db2210f2115c8c989b2bdf45",
+        "receiptsRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+        "logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "prevRandao":"0x0000000000000000000000000000000000000000000000000000000000000000",
+        "blockNumber":"0x1",
+        "gasLimit":"0x1c9c380",
+        "gasUsed":"0x0",
+        "timestamp":"0x5",
+        "extraData":"0x",
+        "baseFeePerGas":"0x7",
+        "blockHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858",
+        "transactions":[]
+       }
+    ],
+    "expect": {
+      "status":"VALID",
+      "latestValidHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858",
+      "validationError":null
+    }
+  },
+  {
+    "name": "Update the forkchoice",
+    "method":"engine_forkchoiceUpdatedV1",
+    "params":[
+      {
+        "headBlockHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858",
+        "safeBlockHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858",
+        "finalizedBlockHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a"
+      },
+      null
+    ],
+    "expect":{
+      "payloadStatus":{
+        "status":"VALID",
+        "latestValidHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858",
+        "validationError":null
+      },
+      "payloadId":null
+    }
+  },
+  {
+    "name": "Invalid payload length",
+    "method":"engine_getPayloadV1",
+    "params":[
+      "0x01"
+    ],
+    "error":{
+      "code":-32602,
+      "message":"invalid argument 0: invalid payload id \"0x01\": hex string has length 2, want 16 for PayloadID"
+    }
+  },
+  {
+    "name": "Unknown paylod",
+    "method":"engine_getPayloadV1",
+    "params":[
+      "0x0000000000000000"
+    ],
+    "error":{
+      "code":-32001,
+      "message":"Unknown payload"
+    }
+  },
+  {
+    "name": "Invalid head",
+    "method":"engine_forkchoiceUpdatedV1",
+    "params":[
+      {
+        "headBlockHash":"0x0000000000000000000000000000000000000000000000000000000000000001",
+        "safeBlockHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a",
+        "finalizedBlockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"
+      },
+      {
+        "timestamp":"0x5",
+        "prevRandao":"0x0000000000000000000000000000000000000000000000000000000000000000",
+        "suggestedFeeRecipient":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"
+      }
+    ],
+    "expect":{
+      "payloadStatus":{
+        "status":"SYNCING",
+        "latestValidHash":null,
+        "validationError":null
+      },
+      "payloadId":null
+    }
+  }
+]

tests/test_merge.nim

@@ -0,0 +1,176 @@
+# Nimbus
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import
+  std/[json, os, sets, strformat, strutils, typetraits],
+  unittest2, nimcrypto, eth/common as eth_common,
+  json_rpc/[rpcserver, rpcclient], web3/[conversions, engine_api_types],
+  eth/[trie/db, p2p/private/p2p_types],
+  ../nimbus/sync/protocol_eth65,
+  ../nimbus/rpc/[common, p2p, hexstrings, rpc_types, rpc_utils, engine_api],
+  ../nimbus/db/[db_chain],
+  ../nimbus/[chain_config, config, context, genesis, sealer],
+  ../nimbus/utils/[tx_pool],
+  ../nimbus/p2p/[chain],
+  ../nimbus/merge/mergetypes,
+  ./test_helpers
+
+const
+  baseDir = "tests" / "merge"
+  paramsFile = baseDir / "params.json"
+  stepsFile = baseDir / "steps.json"
+
+type
+  Step = ref object
+    name: string
+    meth: string
+    params: JSonNode
+    expect: JsonNode
+    error : bool
+
+  Steps = ref object
+    list: seq[Step]
+
+proc parseStep(s: Step, node: JsonNode) =
+  for k, v in node:
+    case k
+    of "name": s.name = v.getStr()
+    of "method": s.meth = v.getStr()
+    of "params": s.params = v
+    of "expect": s.expect = v
+    of "error": s.error = true
+    else:
+      doAssert(false, "unknown key: " & k)
+
+proc parseSteps(node: JsonNode): Steps =
+  let ss = Steps(list: @[])
+  for n in node:
+    let s = Step()
+    s.parseStep(n)
+    ss.list.add s
+  ss
+
+proc forkChoiceUpdate(step: Step, client: RpcClient, testStatusIMPL: var TestStatus) =
+  let arg = step.params[1]
+  if arg.kind == JNull:
+    step.params.elems.setLen(1)
+
+  let res = waitFor client.call(step.meth, step.params)
+  check toLowerAscii($res) == toLowerAscii($step.expect)
+
+proc getPayload(step: Step, client: RpcClient, testStatusIMPL: var TestStatus) =
+  try:
+    let res = waitFor client.call(step.meth, step.params)
+    check toLowerAscii($res) == toLowerAscii($step.expect)
+  except:
+    check step.error == true
+
+proc newPayload(step: Step, client: RpcClient, testStatusIMPL: var TestStatus) =
+  let res = waitFor client.call(step.meth, step.params)
+  check toLowerAscii($res) == toLowerAscii($step.expect)
+
+proc runTest(steps: Steps) =
+  let
+    conf = makeConfig(@["--custom-network:" & paramsFile])
+    ctx  = newEthContext()
+    ethNode = setupEthNode(conf, ctx, eth)
+    chainDB = newBaseChainDB(
+      newMemoryDb(),
+      conf.pruneMode == PruneMode.Full,
+      conf.networkId,
+      conf.networkParams
+    )
+    chainRef = newChain(chainDB)
+
+  initializeEmptyDb(chainDB)
+
+  var
+    rpcServer = newRpcSocketServer(["localhost:" & $conf.rpcPort])
+    client = newRpcSocketClient()
+    txPool = TxPoolRef.new(chainDB, conf.engineSigner)
+    sealingEngine = SealingEngineRef.new(
+      chainRef, ctx, conf.engineSigner,
+      txPool, EnginePostMerge
+    )
+
+  setupEthRpc(ethNode, ctx, chainDB, txPool, rpcServer)
+  setupEngineAPI(sealingEngine, rpcServer)
+
+  sealingEngine.start()
+  rpcServer.start()
+  waitFor client.connect("localhost", Port(conf.rpcPort))
+
+  suite "Engine API tests":
+    for i, step in steps.list:
+      test $i & " " & step.name:
+        case step.meth
+        of "engine_forkchoiceUpdatedV1":
+          forkChoiceUpdate(step, client, testStatusIMPL)
+        of "engine_getPayloadV1":
+          getPayload(step, client, testStatusIMPL)
+        of "engine_newPayloadV1":
+          newPayload(step, client, testStatusIMPL)
+        else:
+          doAssert(false, "unknown method: " & step.meth)
+
+  waitFor client.close()
+  waitFor sealingEngine.stop()
+  rpcServer.stop()
+  waitFor rpcServer.closeWait()
+
+proc testEngineAPI() =
+  let node = parseJSON(readFile(stepsFile))
+  let steps = parseSteps(node)
+  runTest(steps)
+
+proc toId(x: int): PayloadId =
+  var id: distinctBase PayloadId
+  id[^1] = x.byte
+  PayloadId(id)
+
+proc `==`(a, b: Quantity): bool =
+  uint64(a) == uint64(b)
+
+proc main() =
+  var db = newBaseChainDB(newMemoryDB())
+  var api = EngineAPI.new()
+  let
+    id1 = toId(1)
+    id2 = toId(2)
+    ep1 = ExecutionPayloadV1(gasLimit: Quantity 100)
+    ep2 = ExecutionPayloadV1(gasLimit: Quantity 101)
+    hdr1 = EthBlockHeader(gasLimit: 100)
+    hdr2 = EthBlockHeader(gasLimit: 101)
+    hash1 = hdr1.blockHash
+    hash2 = hdr2.blockHash
+
+  suite "Test engine api support":
+    test "test payload queue":
+      api.put(id1, ep1)
+      api.put(id2, ep2)
+      var eep1, eep2: ExecutionPayloadV1
+      check api.get(id1, eep1)
+      check api.get(id2, eep2)
+      check eep1.gasLimit == ep1.gasLimit
+      check eep2.gasLimit == ep2.gasLimit
+
+    test "test header queue":
+      api.put(hash1, hdr1)
+      api.put(hash2, hdr2)
+      var eh1, eh2: EthBlockHeader
+      check api.get(hash1, eh1)
+      check api.get(hash2, eh2)
+      check eh1.gasLimit == hdr1.gasLimit
+      check eh2.gasLimit == hdr2.gasLimit
+
+proc mergeMain*() =
+  testEngineAPI()
+
+when isMainModule:
+  mergeMain()