Add Engine API generic tests

hive_integration/nodocker/engine/auths_tests.nim

@@ -77,7 +77,7 @@ proc specExecute(ws: BaseSpec): bool =
     ws  = AuthSpec(ws)
     env = TestEnv.new("", true)
 
-  env.engine.setRealTTD(0)
+  env.engine.setRealTTD()
   result = ws.exec(env)
   env.close()
 

hive_integration/nodocker/engine/base_spec.nim

@@ -23,9 +23,9 @@ proc configureCLMock*(s: BaseSpec, cl: CLMocker) =
 
   cl.blockTimestampIncrement = some(s.getBlockTimeIncrements())
 
-func getMainFork*(s: BaseSpec): string =
+func getMainFork*(s: BaseSpec): EngineFork =
   let mainFork = s.mainFork
-  if mainFork == "":
+  if mainFork == ForkNone:
     return ForkParis
   return mainFork
 
@@ -44,27 +44,32 @@ func getForkTime*(s: BaseSpec): uint64 =
     forkTime = s.getBlockTime(s.forkHeight.uint64)
   return forkTime
 
-func getForkConfig*(s: BaseSpec): ChainConfig =
+method getForkConfig*(s: BaseSpec): ChainConfig {.base.} =
   let
     forkTime = s.getForkTime()
     previousForkTime = s.previousForkTime
     mainFork = s.getMainFork()
-    forkConfig = getChainConfig(mainFork)
+    forkConfig = getChainConfig($mainFork)
     genesisTimestamp = s.getGenesisTimestamp()
 
   doAssert(previousForkTime <= forkTime,
     "previous fork time cannot be greater than fork time")
 
   if mainFork == ForkParis:
-    let cond = forkTime > genesisTimestamp or previousForkTime != 0
-    doAssert(not cond, "Cannot configure a fork before Paris, skip test")
+    # Cannot configure a fork before Paris, skip test
+    if forkTime > genesisTimestamp or previousForkTime != 0:
+      debugEcho "forkTime: ", forkTime
+      debugEcho "genesisTime: ", genesisTimestamp
+      return nil
   elif mainFork == ForkShanghai:
-    doAssert(previousForkTime == 0, "Cannot configure a fork before Shanghai")
+    # Cannot configure a fork before Shanghai
+    if previousForkTime != 0:
+      return nil
     forkConfig.shanghaiTime = some(forkTime.EthTime)
   elif mainFork == ForkCancun:
     forkConfig.shanghaiTime = some(previousForkTime.EthTime)
     forkConfig.cancunTime = some(forkTime.EthTime)
   else:
-    doAssert(false, "unknown fork: " & mainFork)
+    doAssert(false, "unknown fork: " & $mainFork)
 
   return forkConfig

hive_integration/nodocker/engine/cancun/customizer.nim

@@ -1,6 +1,5 @@
 import
   std/[options, strutils, typetraits, random],
-  nimcrypto/sysrand,
   stew/byteutils,
   ./blobs,
   ../types,
@@ -268,8 +267,7 @@ method getVersionedHashes(cust: ExtraVersionedHash,
   for i, h in baseVersionedHashes:
     v[i] = h
 
-  var extraHash: common.Hash256
-  doAssert randomBytes(extraHash.data) == 32
+  var extraHash = common.Hash256.randomBytes()
   extraHash.data[0] = VERSIONED_HASH_VERSION_KZG
   v[^1] = extraHash
   some(v)
@@ -395,6 +393,11 @@ proc customizePayload*(cust: CustomPayloadData, data: ExecutableData): Executabl
     header: customHeader,
   )
 
+  if cust.transactions.isSome:
+    blk.txs = cust.transactions.get
+  else:
+    blk.txs = ethTxs data.basePayload.transactions
+
   if cust.removeWithdrawals:
     blk.withdrawals = none(seq[Withdrawal])
   elif cust.withdrawals.isSome:
@@ -573,8 +576,7 @@ proc generateInvalidPayload*(sender: TxSender, data: ExecutableData, payloadFiel
   of InvalidPrevRandao:
     # This option potentially requires a transaction that uses the PREVRANDAO opcode.
     # Otherwise the payload will still be valid.
-    var randomHash: common.Hash256
-    doAssert randomBytes(randomHash.data) == 32
+    let randomHash = common.Hash256.randomBytes()
     customPayloadMod = CustomPayloadData(
       prevRandao: some(randomHash),
     )
@@ -645,7 +647,8 @@ proc generateInvalidPayload*(sender: TxSender, data: ExecutableData, payloadFiel
 
     case payloadField
     of InvalidTransactionSignature:
-      custTx.signature = some(baseTx.R - 1.u256)
+      var sig = CustSig(R: baseTx.R - 1.u256)
+      custTx.signature = some(sig)
     of InvalidTransactionNonce:
       custTx.nonce = some(baseTx.nonce - 1)
     of InvalidTransactionGas:
@@ -661,7 +664,8 @@ proc generateInvalidPayload*(sender: TxSender, data: ExecutableData, payloadFiel
       custTx.chainId = some(ChainId(baseTx.chainId.uint64 + 1))
     else: discard
 
-    let modifiedTx = sender.customizeTransaction(baseTx, custTx)
+    let acc = sender.getNextAccount()
+    let modifiedTx = sender.customizeTransaction(acc, baseTx, custTx)
     customPayloadMod = CustomPayloadData(
       transactions: some(@[modifiedTx]),
     )

hive_integration/nodocker/engine/cancun/step_newpayloads.nim

@@ -210,7 +210,7 @@ method execute*(step: NewPayloads, ctx: CancunTestContext): bool =
             forkchoiceState   = env.clMock.latestForkchoice
             expectedError     = step.fcUOnPayloadRequest.getExpectedError()
             expectedStatus    = PayloadExecutionStatus.valid
-            timestamp         = env.clMock.latestHeader.timestamp.uint64            
+            timestamp         = env.clMock.latestHeader.timestamp.uint64
 
           payloadAttributes = step.fcUOnPayloadRequest.getPayloadAttributes(payloadAttributes)
           let version = step.fcUOnPayloadRequest.forkchoiceUpdatedVersion(timestamp, some(payloadAttributes.timestamp.uint64))
@@ -295,7 +295,7 @@ method execute*(step: NewPayloads, ctx: CancunTestContext): bool =
           # Send a custom new payload
           payload = step.newPayloadCustomizer.customizePayload(payload)
           let
-            version = step.newPayloadCustomizer.newPayloadVersion(payload.basePayload.timestamp.uint64)
+            version = step.newPayloadCustomizer.newPayloadVersion(payload.timestamp.uint64)
 
           if step.newPayloadCustomizer.getExpectInvalidStatus():
             expectedStatus = PayloadExecutionStatus.invalid
@@ -305,7 +305,7 @@ method execute*(step: NewPayloads, ctx: CancunTestContext): bool =
             r.expectErrorCode(expectedError, step.expectationDescription)
           else:
             r.expectNoError(step.expectationDescription)
-            r.expectNPStatus(expectedStatus)
+            r.expectStatus(expectedStatus)
 
         if step.fcUOnHeadSet != nil:
           step.fcUOnHeadSet.setEngineAPIVersionResolver(env.engine.com)

hive_integration/nodocker/engine/cancun/step_sendmodpayload.nim

@@ -28,7 +28,7 @@ method execute*(step: SendModifiedLatestPayload, ctx: CancunTestContext): bool =
   step.newPayloadCustomizer.setEngineAPIVersionResolver(env.engine.com)
 
   payload = step.newPayloadCustomizer.customizePayload(payload)
-  let version = step.newPayloadCustomizer.newPayloadVersion(payload.basePayload.timestamp.uint64)
+  let version = step.newPayloadCustomizer.newPayloadVersion(payload.timestamp.uint64)
 
   if step.newPayloadCustomizer.getExpectInvalidStatus():
     expectedStatus = PayloadExecutionStatus.invalid
@@ -41,7 +41,7 @@ method execute*(step: SendModifiedLatestPayload, ctx: CancunTestContext): bool =
   if expectedError != 0:
     r.expectErrorCode(expectedError)
   else:
-    r.expectNPStatus(expectedStatus)
+    r.expectStatus(expectedStatus)
 
   return true
 

hive_integration/nodocker/engine/cancun_tests.nim

@@ -44,7 +44,7 @@ proc specExecute(ws: BaseSpec): bool =
 
   getGenesis(conf.networkParams)
   let env  = TestEnv.new(conf)
-  env.engine.setRealTTD(0)
+  env.engine.setRealTTD()
   env.setupCLMock()
   ws.configureCLMock(env.clMock)
 
@@ -1827,7 +1827,7 @@ func init() {
   }
   onlyBlobTxsSpec := test.BaseSpec{
     mainFork:            Cancun,
-    TestTransactionType: helper.BlobTxOnly,
+    TestTransactionType: BlobTxOnly,
   }
 
   # Payload Attributes
@@ -1865,30 +1865,30 @@ func init() {
   }
 
   # Invalid Payload Tests
-  for _, invalidField := range []helper.InvalidPayloadBlockField{
-    helper.InvalidParentBeaconBlockRoot,
-    helper.InvalidBlobGasUsed,
-    helper.InvalidBlobCountGasUsed,
-    helper.InvalidExcessBlobGas,
-    helper.InvalidVersionedHashes,
-    helper.InvalidVersionedHashesVersion,
-    helper.IncompleteVersionedHashes,
-    helper.ExtraVersionedHashes,
+  for _, invalidField := range []InvalidPayloadBlockField{
+    InvalidParentBeaconBlockRoot,
+    InvalidBlobGasUsed,
+    InvalidBlobCountGasUsed,
+    InvalidExcessBlobGas,
+    InvalidVersionedHashes,
+    InvalidVersionedHashesVersion,
+    IncompleteVersionedHashes,
+    ExtraVersionedHashes,
   } {
     for _, syncing := range []bool{false, true} {
       # Invalidity of payload can be detected even when syncing because the
       # blob gas only depends on the transactions contained.
-      invalidDetectedOnSync := (invalidField == helper.InvalidBlobGasUsed ||
-        invalidField == helper.InvalidBlobCountGasUsed ||
-        invalidField == helper.InvalidVersionedHashes ||
-        invalidField == helper.InvalidVersionedHashesVersion ||
-        invalidField == helper.IncompleteVersionedHashes ||
-        invalidField == helper.ExtraVersionedHashes)
+      invalidDetectedOnSync := (invalidField == InvalidBlobGasUsed ||
+        invalidField == InvalidBlobCountGasUsed ||
+        invalidField == InvalidVersionedHashes ||
+        invalidField == InvalidVersionedHashesVersion ||
+        invalidField == IncompleteVersionedHashes ||
+        invalidField == ExtraVersionedHashes)
 
-      nilLatestValidHash := (invalidField == helper.InvalidVersionedHashes ||
-        invalidField == helper.InvalidVersionedHashesVersion ||
-        invalidField == helper.IncompleteVersionedHashes ||
-        invalidField == helper.ExtraVersionedHashes)
+      nilLatestValidHash := (invalidField == InvalidVersionedHashes ||
+        invalidField == InvalidVersionedHashesVersion ||
+        invalidField == IncompleteVersionedHashes ||
+        invalidField == ExtraVersionedHashes)
 
       Tests = append(Tests, suite_engine.InvalidPayloadTestCase{
         BaseSpec:              onlyBlobTxsSpec,
@@ -1909,7 +1909,7 @@ func init() {
 
   Tests = append(Tests, suite_engine.PayloadBuildAfterInvalidPayloadTest{
     BaseSpec:     onlyBlobTxsSpec,
-    InvalidField: helper.InvalidParentBeaconBlockRoot,
+    InvalidField: InvalidParentBeaconBlockRoot,
   })
 
   # Suggested Fee Recipient Tests (New Transaction Type)

hive_integration/nodocker/engine/client_pool.nim

@@ -21,3 +21,12 @@ func `[]`*(pool: ClientPool, idx: int): EngineEnv =
 iterator items*(pool: ClientPool): EngineEnv =
   for x in pool.clients:
     yield x
+
+proc remove*(pool: ClientPool, client: EngineEnv) =
+  var index = -1
+  for i, x in pool.clients:
+    if x == client:
+      index = i
+      break
+  if index != -1:
+    pool.clients.delete(index)

hive_integration/nodocker/engine/clmock.nim

@@ -1,7 +1,6 @@
 import
   std/[tables],
   chronicles,
-  nimcrypto/sysrand,
   stew/[byteutils],
   eth/common, chronos,
   json_rpc/rpcclient,
@@ -131,6 +130,9 @@ proc newClMocker*(eng: EngineEnv, com: CommonRef): CLMocker =
 proc addEngine*(cl: CLMocker, eng: EngineEnv) =
   cl.clients.add eng
 
+proc removeEngine*(cl: CLMocker, eng: EngineEnv) =
+  cl.clients.remove eng
+
 proc waitForTTD*(cl: CLMocker): Future[bool] {.async.} =
   let ttd = cl.com.ttd()
   doAssert(ttd.isSome)
@@ -232,13 +234,13 @@ proc pickNextPayloadProducer(cl: CLMocker): bool =
 
     # Get latest header. Number and hash must coincide with our view of the chain,
     # and only then we can build on top of this client's chain
-    var latestHeader: common.BlockHeader
-    let res = cl.nextBlockProducer.client.latestHeader(latestHeader)
+    let res = cl.nextBlockProducer.client.latestHeader()
     if res.isErr:
       error "CLMocker: Could not get latest block header while selecting client for payload production",
         msg=res.error
       return false
 
+    let latestHeader = res.get
     let lastBlockHash = latestHeader.blockHash
     if cl.latestHeader.blockHash != lastBlockHash or
        cl.latestHeadNumber != latestHeader.blockNumber.truncate(uint64):
@@ -253,9 +255,7 @@ proc pickNextPayloadProducer(cl: CLMocker): bool =
 
 proc generatePayloadAttributes(cl: CLMocker) =
   # Generate a random value for the PrevRandao field
-  var nextPrevRandao: common.Hash256
-  doAssert randomBytes(nextPrevRandao.data) == 32
-
+  let nextPrevRandao = common.Hash256.randomBytes()
   let timestamp = Quantity cl.getNextBlockTimestamp.uint64
   cl.latestPayloadAttributes = PayloadAttributes(
     timestamp:             timestamp,
@@ -370,91 +370,97 @@ func versionedHashes(payload: ExecutionPayload): seq[Web3Hash] =
     for vs in tx.versionedHashes:
       result.add w3Hash vs
 
-proc broadcastNewPayload(cl: CLMocker, payload: ExecutionPayload): Result[PayloadStatusV1, string] =
+proc broadcastNewPayload(cl: CLMocker,
+                         eng: EngineEnv,
+                         payload: ExecutionPayload): Result[PayloadStatusV1, string] =
   case payload.version
-  of Version.V1: return cl.client.newPayloadV1(payload.V1)
-  of Version.V2: return cl.client.newPayloadV2(payload.V2)
-  of Version.V3: return cl.client.newPayloadV3(payload.V3,
+  of Version.V1: return eng.client.newPayloadV1(payload.V1)
+  of Version.V2: return eng.client.newPayloadV2(payload.V2)
+  of Version.V3: return eng.client.newPayloadV3(payload.V3,
     versionedHashes(payload),
     cl.latestPayloadAttributes.parentBeaconBlockRoot.get)
 
 proc broadcastNextNewPayload(cl: CLMocker): bool =
-  let res = cl.broadcastNewPayload(cl.latestPayloadBuilt)
-  if res.isErr:
-    error "CLMocker: broadcastNewPayload Error", msg=res.error
-    return false
-
-  let s = res.get()
-  if s.status == PayloadExecutionStatus.valid:
-    # The client is synced and the payload was immediately validated
-    # https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md:
-    # - If validation succeeds, the response MUST contain {status: VALID, latestValidHash: payload.blockHash}
-    let blockHash = cl.latestPayloadBuilt.blockHash
-    if s.latestValidHash.isNone:
-      error "CLMocker: NewPayload returned VALID status with nil LatestValidHash",
-        expected=blockHash.toHex
+  for eng in cl.clients:
+    let res = cl.broadcastNewPayload(eng, cl.latestPayloadBuilt)
+    if res.isErr:
+      error "CLMocker: broadcastNewPayload Error", msg=res.error
       return false
 
-    let latestValidHash = s.latestValidHash.get()
-    if latestValidHash != BlockHash(blockHash):
-      error "CLMocker: NewPayload returned VALID status with incorrect LatestValidHash",
-        get=latestValidHash.toHex, expected=blockHash.toHex
-      return false
+    let s = res.get()
+    if s.status == PayloadExecutionStatus.valid:
+      # The client is synced and the payload was immediately validated
+      # https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md:
+      # - If validation succeeds, the response MUST contain {status: VALID, latestValidHash: payload.blockHash}
+      let blockHash = cl.latestPayloadBuilt.blockHash
+      if s.latestValidHash.isNone:
+        error "CLMocker: NewPayload returned VALID status with nil LatestValidHash",
+          expected=blockHash.toHex
+        return false
 
-  elif s.status == PayloadExecutionStatus.accepted:
-    # The client is not synced but the payload was accepted
-    # https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md:
-    # - {status: ACCEPTED, latestValidHash: null, validationError: null} if the following conditions are met:
-    # the blockHash of the payload is valid
-    # the payload doesn't extend the canonical chain
-    # the payload hasn't been fully validated.
-    let nullHash = BlockHash common.Hash256().data
-    let latestValidHash = s.latestValidHash.get(nullHash)
-    if s.latestValidHash.isSome and latestValidHash != nullHash:
-      error "CLMocker: NewPayload returned ACCEPTED status with incorrect LatestValidHash",
-        hash=latestValidHash.toHex
-      return false
+      let latestValidHash = s.latestValidHash.get()
+      if latestValidHash != BlockHash(blockHash):
+        error "CLMocker: NewPayload returned VALID status with incorrect LatestValidHash",
+          get=latestValidHash.toHex, expected=blockHash.toHex
+        return false
 
-  else:
-    error "CLMocker: broadcastNewPayload Response",
-      status=s.status
-    return false
+    elif s.status == PayloadExecutionStatus.accepted:
+      # The client is not synced but the payload was accepted
+      # https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md:
+      # - {status: ACCEPTED, latestValidHash: null, validationError: null} if the following conditions are met:
+      # the blockHash of the payload is valid
+      # the payload doesn't extend the canonical chain
+      # the payload hasn't been fully validated.
+      let nullHash = BlockHash common.Hash256().data
+      let latestValidHash = s.latestValidHash.get(nullHash)
+      if s.latestValidHash.isSome and latestValidHash != nullHash:
+        error "CLMocker: NewPayload returned ACCEPTED status with incorrect LatestValidHash",
+          hash=latestValidHash.toHex
+        return false
+
+    else:
+      error "CLMocker: broadcastNewPayload Response",
+        status=s.status
+      return false
 
   cl.latestExecutedPayload = cl.latestPayloadBuilt
   let number = uint64 cl.latestPayloadBuilt.blockNumber
   cl.executedPayloadHistory[number] = cl.latestPayloadBuilt
   return true
 
-proc broadcastForkchoiceUpdated(cl: CLMocker,
+proc broadcastForkchoiceUpdated(cl: CLMocker, eng: EngineEnv,
       update: ForkchoiceStateV1): Result[ForkchoiceUpdatedResponse, string] =
   let version = cl.latestExecutedPayload.version
-  let client = cl.nextBlockProducer.client
-  client.forkchoiceUpdated(version, update, none(PayloadAttributes))
+  eng.client.forkchoiceUpdated(version, update, none(PayloadAttributes))
 
 proc broadcastLatestForkchoice(cl: CLMocker): bool =
-  let res = cl.broadcastForkchoiceUpdated(cl.latestForkchoice)
-  if res.isErr:
-    error "CLMocker: broadcastForkchoiceUpdated Error", msg=res.error
-    return false
+  for eng in cl.clients:
+    let res = cl.broadcastForkchoiceUpdated(eng, cl.latestForkchoice)
+    if res.isErr:
+      error "CLMocker: broadcastForkchoiceUpdated Error", msg=res.error
+      return false
 
-  let s = res.get()
-  if s.payloadStatus.status != PayloadExecutionStatus.valid:
-    error "CLMocker: broadcastForkchoiceUpdated Response",
-      status=s.payloadStatus.status
-    return false
+    let s = res.get()
+    if s.payloadStatus.status != PayloadExecutionStatus.valid:
+      error "CLMocker: broadcastForkchoiceUpdated Response",
+        status=s.payloadStatus.status
+      return false
 
-  if s.payloadStatus.latestValidHash.get != cl.latestForkchoice.headBlockHash:
-    error "CLMocker: Incorrect LatestValidHash from ForkchoiceUpdated",
-      get=s.payloadStatus.latestValidHash.get.toHex,
-      expect=cl.latestForkchoice.headBlockHash.toHex
+    if s.payloadStatus.latestValidHash.get != cl.latestForkchoice.headBlockHash:
+      error "CLMocker: Incorrect LatestValidHash from ForkchoiceUpdated",
+        get=s.payloadStatus.latestValidHash.get.toHex,
+        expect=cl.latestForkchoice.headBlockHash.toHex
+      return false
 
-  if s.payloadStatus.validationError.isSome:
-    error "CLMocker: Expected empty validationError",
-      msg=s.payloadStatus.validationError.get
+    if s.payloadStatus.validationError.isSome:
+      error "CLMocker: Expected empty validationError",
+        msg=s.payloadStatus.validationError.get
+      return false
 
-  if s.payloadID.isSome:
-    error "CLMocker: Expected empty PayloadID",
-      msg=s.payloadID.get.toHex
+    if s.payloadID.isSome:
+      error "CLMocker: Expected empty PayloadID",
+        msg=s.payloadID.get.toHex
+      return false
 
   return true
 
@@ -528,7 +534,7 @@ proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe
     let period = chronos.seconds(cl.payloadProductionClientDelay)
     waitFor sleepAsync(period)
 
-  if not cl.getNextPayload():    
+  if not cl.getNextPayload():
     return false
 
   if cb.onGetPayload != nil:
@@ -590,13 +596,13 @@ proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe
   cl.latestHeadNumber = cl.latestHeadNumber + 1
 
   # Check if any of the clients accepted the new payload
-  var newHeader: common.BlockHeader
-  let res = cl.client.headerByNumber(cl.latestHeadNumber, newHeader)
+  let res = cl.client.headerByNumber(cl.latestHeadNumber)
   if res.isErr:
     error "CLMock ProduceSingleBlock", msg=res.error
     return false
 
-  let newHash = BlockHash newHeader.blockHash.data
+  let newHeader = res.get
+  let newHash = w3Hash newHeader.blockHash
   if newHash != cl.latestPayloadBuilt.blockHash:
     error "CLMocker: None of the clients accepted the newly constructed payload",
       hash=newHash.toHex

hive_integration/nodocker/engine/engine/bad_hash.nim

@@ -0,0 +1,188 @@
+import
+  std/strutils,
+  chronicles,
+  eth/common/eth_types_rlp,
+  ./engine_spec,
+  ../cancun/customizer
+
+# Corrupt the hash of a valid payload, client should reject the payload.
+# All possible scenarios:
+#    (fcU)
+# ┌────────┐        ┌────────────────────────┐
+# │  HEAD  │◄───────┤ Bad Hash (!Sync,!Side) │
+# └────┬───┘        └────────────────────────┘
+#    │
+#    │
+# ┌────▼───┐        ┌────────────────────────┐
+# │ HEAD-1 │◄───────┤ Bad Hash (!Sync, Side) │
+# └────┬───┘        └────────────────────────┘
+#    │
+#
+#
+#   (fcU)
+# ********************  ┌───────────────────────┐
+# *  (Unknown) HEAD  *◄─┤ Bad Hash (Sync,!Side) │
+# ********************  └───────────────────────┘
+#    │
+#    │
+# ┌────▼───┐            ┌───────────────────────┐
+# │ HEAD-1 │◄───────────┤ Bad Hash (Sync, Side) │
+# └────┬───┘            └───────────────────────┘
+#    │
+#
+
+type
+  BadHashOnNewPayload* = ref object of EngineSpec
+    syncing*:   bool
+    sidechain*: bool
+
+  Shadow = ref object
+    payload: ExecutableData
+
+method withMainFork(cs: BadHashOnNewPayload, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: BadHashOnNewPayload): string =
+  "Bad Hash on NewPayload (syncing=$1, sidechain=$1)" % [$cs.syncing, $cs.sidechain]
+
+method execute(cs: BadHashOnNewPayload, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  var shadow = Shadow()
+
+  var pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Run test after the new payload has been obtained
+    onGetPayload: proc(): bool =
+      # Alter hash on the payload and send it to client, should produce an error
+      shadow.payload = env.clMock.latestExecutableData
+      var invalidHash = ethHash shadow.payload.blockHash
+      invalidHash.data[^1] = byte(255 - invalidHash.data[^1])
+      shadow.payload.blockHash = w3Hash invalidHash
+
+      if not cs.syncing and cs.sidechain:
+        # We alter the payload by setting the parent to a known past block in the
+        # canonical chain, which makes this payload a side chain payload, and also an invalid block hash
+        # (because we did not update the block hash appropriately)
+        shadow.payload.parentHash = w3Hash env.clMock.latestHeader.parentHash
+      elif cs.syncing:
+        # We need to send an fcU to put the client in syncing state.
+        let
+          randomHeadBlock = Web3Hash.randomBytes()
+          latestHash = w3Hash env.clMock.latestHeader.blockHash
+          fcU = ForkchoiceStateV1(
+            headblockHash:      randomHeadBlock,
+            safeblockHash:      latestHash,
+            finalizedblockHash: latestHash,
+          )
+          version = env.engine.version(env.clMock.latestHeader.timestamp)
+          r = env.engine.client.forkchoiceUpdated(version, fcU)
+
+        r.expectPayloadStatus(PayloadExecutionStatus.syncing)
+
+        if cs.sidechain:
+          # syncing and sidechain, the caonincal head is an unknown payload to us,
+          # but this specific bad hash payload is in theory part of a side chain.
+          # Therefore the parent we use is the head hash.
+          shadow.payload.parentHash = latestHash
+        else:
+          # The invalid bad-hash payload points to the unknown head, but we know it is
+          # indeed canonical because the head was set using forkchoiceUpdated.
+          shadow.payload.parentHash = randomHeadBlock
+
+      # Execution specification::
+      # - (status: INVALID_BLOCK_HASH, latestValidHash: null, validationError: null) if the blockHash validation has failed
+      # Starting from Shanghai, INVALID should be returned instead (https:#githucs.com/ethereum/execution-apis/pull/338)
+      let
+        version = env.engine.version(shadow.payload.timestamp)
+        r = env.engine.client.newPayload(version, shadow.payload)
+
+      if version >= Version.V2:
+        r.expectStatus(PayloadExecutionStatus.invalid)
+      else:
+        r.expectStatusEither([PayloadExecutionStatus.invalidBlockHash, PayloadExecutionStatus.invalid])
+
+      r.expectLatestValidHash()
+      return true
+  ))
+  testCond pbRes
+
+  # Lastly, attempt to build on top of the invalid payload
+  pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Run test after the new payload has been obtained
+    onGetPayload: proc(): bool =
+      var customizer = CustomPayloadData(
+        parentHash: some(ethHash shadow.payload.blockHash),
+      )
+      shadow.payload = customizer.customizePayload(env.clMock.latestExecutableData)
+
+      # Response status can be ACCEPTED (since parent payload could have been thrown out by the client)
+      # or INVALID (client still has the payload and can verify that this payload is incorrectly building on top of it),
+      # but a VALID response is incorrect.
+      let
+        version = env.engine.version(shadow.payload.timestamp)
+        r = env.engine.client.newPayload(version, shadow.payload)
+      r.expectStatusEither([PayloadExecutionStatus.accepted, PayloadExecutionStatus.invalid, PayloadExecutionStatus.syncing])
+      return true
+  ))
+
+  testCond pbRes
+  return true
+
+type
+  ParentHashOnNewPayload* = ref object of EngineSpec
+    syncing*: bool
+
+method withMainFork(cs: ParentHashOnNewPayload, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ParentHashOnNewPayload): string =
+  var name = "parentHash==blockHash on NewPayload"
+  if cs.syncing:
+    name.add " (syncing)"
+  return name
+
+# Copy the parentHash into the blockHash, client should reject the payload
+# (from Kintsugi Incident Report: https:#notes.ethereum.org/@ExXcnR0-SJGthjz1dwkA1A/BkkdHWXTY)
+method execute(cs: ParentHashOnNewPayload, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  let pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Run test after the new payload has been obtained
+    onGetPayload: proc(): bool =
+      # Alter hash on the payload and send it to client, should produce an error
+      var payload = env.clMock.latestExecutableData
+      if cs.syncing:
+        # Parent hash is unknown but also (incorrectly) set as the block hash
+        payload.parentHash = Web3Hash.randomBytes()
+
+      payload.blockHash = payload.parentHash
+      # Execution specification::
+      # - (status: INVALID_BLOCK_HASH, latestValidHash: null, validationError: null) if the blockHash validation has failed
+      # Starting from Shanghai, INVALID should be returned instead (https:#githucs.com/ethereum/execution-apis/pull/338)
+      let
+        version = env.engine.version(payload.timestamp)
+        r = env.engine.client.newPayload(version, payload)
+
+      if version >= Version.V2:
+        r.expectStatus(PayloadExecutionStatus.invalid)
+      else:
+        r.expectStatusEither([PayloadExecutionStatus.invalid, PayloadExecutionStatus.invalidBlockHash])
+      r.expectLatestValidHash()
+      return true
+  ))
+  testCond pbRes
+  return true

hive_integration/nodocker/engine/engine/fork_id.nim

@@ -0,0 +1,68 @@
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  ForkIDSpec* = ref object of EngineSpec
+    produceBlocksBeforePeering: int
+
+method withMainFork(cs: ForkIDSpec, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ForkIDSpec): string =
+  name = "Fork ID: Genesis at %d, %s at %d", cs.GetGenesistimestamp(), cs.mainFork, cs.ForkTime)
+  if cs.previousForkTime != 0 (
+    name += ", %s at %d", cs.mainFork.PreviousFork(), cs.previousForkTime)
+  )
+  if cs.produceBlocksBeforePeering > 0 (
+    name += ", Produce %d blocks before peering", cs.produceBlocksBeforePeering)
+  )
+  return name
+)
+
+method execute(cs: ForkIDSpec, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test if required
+  env.clMock.produceBlocks(cs.produceBlocksBeforePeering, BlockProcessCallbacks())
+
+  # Get client index's enode
+  engine = t.Engine
+  conn, err = devp2p.PeerEngineClient(engine, t.CLMock)
+  if err != nil (
+    fatal "Error peering engine client: %v", err)
+  )
+  defer conn.Close()
+  info "Connected to client, remote public key: %s", conn.RemoteKey())
+
+  # Sleep
+  await sleepAsync(1 * time.Second)
+
+  # Timeout value for all requests
+  timeout = 20 * time.Second
+
+  # Send a ping request to verify that we are not immediately disconnected
+  pingReq = &devp2p.Ping()
+  if size, err = conn.Write(pingReq); err != nil (
+    fatal "Could not write to connection: %v", err)
+  else:
+    info "Wrote %d bytes to conn", size)
+  )
+
+  # Finally wait for the pong response
+  msg, err = conn.WaitForResponse(timeout, 0)
+  if err != nil (
+    fatal "Error waiting for response: %v", err)
+  )
+  switch msg = msg.(type) (
+  case *devp2p.Pong:
+    info "Received pong response: %v", msg)
+  default:
+    fatal "Unexpected message type: %v", err)
+  )
+
+)

hive_integration/nodocker/engine/engine/forkchoice.nim

@@ -0,0 +1,159 @@
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  ForkchoiceStateField = enum
+    HeadblockHash      = "Head"
+    SafeblockHash      = "Safe"
+    FinalizedblockHash = "Finalized"
+
+type
+  InconsistentForkchoiceTest* = ref object of EngineSpec
+		field*: ForkchoiceStateField
+
+method withMainFork(cs: InconsistentForkchoiceTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: InconsistentForkchoiceTest): string =
+	return "Inconsistent %s in ForkchoiceState", cs.Field)
+)
+
+# Send an inconsistent ForkchoiceState with a known payload that belongs to a side chain as head, safe or finalized.
+method execute(cs: InconsistentForkchoiceTest, env: TestEnv): bool =
+	# Wait until TTD is reached by this client
+	let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+	shadow.canon = make([]*ExecutableData, 0)
+	shadow.alt = make([]*ExecutableData, 0)
+	# Produce blocks before starting the test
+	env.clMock.produceBlocks(3, BlockProcessCallbacks(
+		onGetPayload: proc(): bool =
+			# Generate and send an alternative side chain
+			customData = CustomPayloadData()
+			customData.ExtraData = &([]byte(0x01))
+			if len(shadow.alt) > 0 (
+				customData.parentHash = &shadow.alt[len(shadow.alt)-1].blockHash
+			)
+			alternativePayload, err = customData.CustomizePayload(env.clMock.latestPayloadBuilt)
+			if err != nil (
+				fatal "Unable to construct alternative payload: %v", t.TestName, err)
+			)
+			shadow.alt = append(shadow.alt, alternativePayload)
+			latestCanonicalPayload = env.clMock.latestPayloadBuilt
+			shadow.canon = append(shadow.canon, &latestCanonicalPayload)
+
+			# Send the alternative payload
+			r = env.engine.client.newPayload(alternativePayload)
+			r.expectStatusEither(PayloadExecutionStatus.valid, test.Accepted)
+		),
+	))
+	# Send the invalid ForkchoiceStates
+	inconsistentFcU = ForkchoiceStateV1(
+		headblockHash:      shadow.canon[len(shadow.alt)-1].blockHash,
+		safeblockHash:      shadow.canon[len(shadow.alt)-2].blockHash,
+		finalizedblockHash: shadow.canon[len(shadow.alt)-3].blockHash,
+	)
+	switch cs.Field (
+	case HeadblockHash:
+		inconsistentFcU.headblockHash = shadow.alt[len(shadow.alt)-1].blockHash
+	case SafeblockHash:
+		inconsistentFcU.safeblockHash = shadow.alt[len(shadow.canon)-2].blockHash
+	case FinalizedblockHash:
+		inconsistentFcU.finalizedblockHash = shadow.alt[len(shadow.canon)-3].blockHash
+	)
+	r = env.engine.client.forkchoiceUpdated(inconsistentFcU, nil, env.clMock.latestPayloadBuilt.timestamp)
+	r.expectError()
+
+	# Return to the canonical chain
+	r = env.engine.client.forkchoiceUpdated(env.clMock.latestForkchoice, nil, env.clMock.latestPayloadBuilt.timestamp)
+	r.expectPayloadStatus(PayloadExecutionStatus.valid)
+)
+
+type
+  ForkchoiceUpdatedUnknownblockHashTest* = ref object of EngineSpec
+		field: ForkchoiceStateField
+
+method withMainFork(cs: ForkchoiceUpdatedUnknownblockHashTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ForkchoiceUpdatedUnknownblockHashTest): string =
+	return "Unknown %sblockHash", cs.Field)
+)
+
+# Send an inconsistent ForkchoiceState with a known payload that belongs to a side chain as head, safe or finalized.
+method execute(cs: ForkchoiceUpdatedUnknownblockHashTest, env: TestEnv): bool =
+	# Wait until TTD is reached by this client
+	let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+	# Produce blocks before starting the test
+	env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+	# Generate a random block hash
+	randomblockHash = common.Hash256()
+	randomBytes(randomblockHash[:])
+
+	if cs.Field == HeadblockHash (
+
+		forkchoiceStateUnknownHeadHash = ForkchoiceStateV1(
+			headblockHash:      randomblockHash,
+			safeblockHash:      env.clMock.latestForkchoice.safeblockHash,
+			finalizedblockHash: env.clMock.latestForkchoice.finalizedblockHash,
+		)
+
+		t.Logf("INFO (%v) forkchoiceStateUnknownHeadHash: %v\n", t.TestName, forkchoiceStateUnknownHeadHash)
+
+		# Execution specification::
+		# - (payloadStatus: (status: SYNCING, latestValidHash: null, validationError: null), payloadId: null)
+		#   if forkchoiceState.headblockHash references an unknown payload or a payload that can't be validated
+		#   because requisite data for the validation is missing
+		r = env.engine.client.forkchoiceUpdated(forkchoiceStateUnknownHeadHash, nil, env.clMock.latestExecutedPayload.timestamp)
+		r.expectPayloadStatus(PayloadExecutionStatus.syncing)
+
+		payloadAttributes = env.clMock.latestPayloadAttributes
+		payloadAttributes.timestamp += 1
+
+		# Test again using PayloadAttributes, should also return SYNCING and no PayloadID
+		r = env.engine.client.forkchoiceUpdated(forkchoiceStateUnknownHeadHash,
+			&payloadAttributes, env.clMock.latestExecutedPayload.timestamp)
+		r.expectPayloadStatus(PayloadExecutionStatus.syncing)
+		r.ExpectPayloadID(nil)
+	else:
+		env.clMock.produceSingleBlock(BlockProcessCallbacks(
+			# Run test after a new payload has been broadcast
+			onNewPayloadBroadcast: proc(): bool =
+
+				forkchoiceStateRandomHash = ForkchoiceStateV1(
+					headblockHash:      env.clMock.latestExecutedPayload.blockHash,
+					safeblockHash:      env.clMock.latestForkchoice.safeblockHash,
+					finalizedblockHash: env.clMock.latestForkchoice.finalizedblockHash,
+				)
+
+				if cs.Field == SafeblockHash (
+					forkchoiceStateRandomHash.safeblockHash = randomblockHash
+				elif cs.Field == FinalizedblockHash (
+					forkchoiceStateRandomHash.finalizedblockHash = randomblockHash
+				)
+
+				r = env.engine.client.forkchoiceUpdated(forkchoiceStateRandomHash, nil, env.clMock.latestExecutedPayload.timestamp)
+				r.expectError()
+
+				payloadAttributes = env.clMock.latestPayloadAttributes
+				payloadAttributes.Random = common.Hash256()
+				payloadAttributes.SuggestedFeeRecipient = common.Address()
+
+				# Test again using PayloadAttributes, should also return INVALID and no PayloadID
+				r = env.engine.client.forkchoiceUpdated(forkchoiceStateRandomHash,
+					&payloadAttributes, env.clMock.latestExecutedPayload.timestamp)
+				r.expectError()
+
+			),
+		))
+	)
+)

hive_integration/nodocker/engine/engine/invalid_ancestor.nim

@@ -0,0 +1,427 @@
+import
+  std/strutils,
+  chronicles,
+  eth/common/eth_types_rlp,
+  ./engine_spec,
+  ../cancun/customizer,
+  ../../../../nimbus/utils/utils
+
+# Attempt to re-org to a chain which at some point contains an unknown payload which is also invalid.
+# Then reveal the invalid payload and expect that the client rejects it and rejects forkchoice updated calls to this chain.
+# The invalidIndex parameter determines how many payloads apart is the common ancestor from the block that invalidates the chain,
+# with a value of 1 meaning that the immediate payload after the common ancestor will be invalid.
+
+type
+  InvalidMissingAncestorReOrgTest* = ref object of EngineSpec
+    sidechainLength*  : int
+    invalidIndex*     : int
+    invalidField*     : InvalidPayloadBlockField
+    emptyTransactions*: bool
+
+  Shadow = ref object
+    payloads: seq[ExecutableData]
+    n: int
+    cAHeight: int
+
+method withMainFork(cs: InvalidMissingAncestorReOrgTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: InvalidMissingAncestorReOrgTest): string =
+  var desc = "Invalid Missing Ancestor ReOrg Invalid" & $cs.invalidField
+
+  if cs.emptyTransactions:
+   desc.add ", Empty Txs"
+
+  desc.add ", Invalid P" & $cs.invalidIndex & "'"
+  desc
+
+method execute(cs: InvalidMissingAncestorReOrgTest, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  # Save the common ancestor
+  let cA = env.clMock.latestExecutableData
+
+  # Slice to save the side B chain
+  var shadow = Shadow()
+
+  # Append the common ancestor
+  shadow.payloads.add(cA)
+
+  # Produce blocks but at the same time create an side chain which contains an invalid payload at some point (INV_P)
+  # CommonAncestor◄─▲── P1 ◄─ P2 ◄─ P3 ◄─ ... ◄─ Pn
+  #                 │
+  #                 └── P1' ◄─ P2' ◄─ ... ◄─ INV_P ◄─ ... ◄─ Pn'
+  var pbRes = env.clMock.produceBlocks(cs.sidechainLength, BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      # Function to send at least one transaction each block produced.
+      # Empty Txs Payload with invalid stateRoot discovered an issue in geth sync, hence this is customizable.
+      if not cs.emptyTransactions:
+        # Send the transaction to the globals.PrevRandaoContractAddr
+        let eng = env.clMock.nextBlockProducer
+        let ok = env.sendNextTx(eng, BaseTx(
+            recipient:  some(prevRandaoContractAddr),
+            amount:     1.u256,
+            txType:     cs.txType,
+            gasLimit:   75000,
+        ))
+
+        testCond ok:
+          fatal "Error trying to send transaction"
+      return true
+    ,
+    onGetPayload: proc(): bool =
+      # Insert extraData to ensure we deviate from the main payload, which contains empty extradata
+      let customizer = CustomPayloadData(
+        parentHash: some(ethHash shadow.payloads[^1].blockHash),
+        extraData:  some(@[0x01.byte]),
+      )
+
+      var sidePayload = customizer.customizePayload(env.clMock.latestExecutableData)
+      if shadow.payloads.len == cs.invalidIndex:
+        sidePayload = env.generateInvalidPayload(sidePayload, cs.invalidField)
+      shadow.payloads.add sidePayload
+      return true
+  ))
+  testCond pbRes
+
+  pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Note: We perform the test in the middle of payload creation by the CL Mock, in order to be able to
+    # re-org back into this chain and use the new payload without issues.
+    onGetPayload: proc(): bool =
+      # Now let's send the side chain to the client using newPayload/sync
+      for i in 1..cs.sidechainLength:
+        # Send the payload
+        var payloadValidStr = "VALID"
+        if i == cs.invalidIndex:
+          payloadValidStr = "INVALID"
+        elif i > cs.invalidIndex:
+          payloadValidStr = "VALID with INVALID ancestor"
+
+        info "Invalid chain payload",
+          index=i,
+          payloadValidStr,
+          blockHash=shadow.payloads[i].blockHash.short,
+          number=shadow.payloads[i].blockNumber.uint64
+
+        let version = env.engine.version(shadow.payloads[i].timestamp)
+        let r = env.engine.client.newPayload(version, shadow.payloads[i])
+        let fcState = ForkchoiceStateV1(
+          headblockHash: shadow.payloads[i].blockHash,
+        )
+        let p = env.engine.client.forkchoiceUpdated(version, fcState)
+
+        if i == cs.invalidIndex:
+          # If this is the first payload after the common ancestor, and this is the payload we invalidated,
+          # then we have all the information to determine that this payload is invalid.
+          r.expectStatus(PayloadExecutionStatus.invalid)
+          r.expectLatestValidHash(shadow.payloads[i-1].blockHash)
+        elif i > cs.invalidIndex:
+          # We have already sent the invalid payload, but the client could've discarded it.
+          # In reality the CL will not get to this point because it will have already received the `INVALID`
+          # response from the previous payload.
+          # The node might save the parent as invalid, thus returning INVALID
+          r.expectStatusEither([PayloadExecutionStatus.accepted, PayloadExecutionStatus.syncing, PayloadExecutionStatus.invalid])
+          let status = r.get.status
+          if status in [PayloadExecutionStatus.accepted, PayloadExecutionStatus.syncing]:
+            r.expectLatestValidHash()
+          elif status == PayloadExecutionStatus.invalid:
+            r.expectLatestValidHash(shadow.payloads[cs.invalidIndex-1].blockHash)
+        else:
+          # This is one of the payloads before the invalid one, therefore is valid.
+          r.expectStatus(PayloadExecutionStatus.valid)
+          p.expectPayloadStatus(PayloadExecutionStatus.valid)
+          p.expectLatestValidHash(shadow.payloads[i].blockHash)
+
+      # Resend the latest correct fcU
+      let version = env.engine.version(env.clMock.latestPayloadBuilt.timestamp)
+      let r = env.engine.client.forkchoiceUpdated(version, env.clMock.latestForkchoice)
+      r.expectNoError()
+      # After this point, the CL Mock will send the next payload of the canonical chain
+      return true
+  ))
+
+  testCond pbRes
+  return true
+
+# Attempt to re-org to a chain which at some point contains an unknown payload which is also invalid.
+# Then reveal the invalid payload and expect that the client rejects it and rejects forkchoice updated calls to this chain.
+type
+  InvalidMissingAncestorReOrgSyncTest* = ref object of EngineSpec
+    # Index of the payload to invalidate, starting with 0 being the common ancestor.
+    # Value must be greater than 0.
+    invalidIndex*: int
+    # Field of the payload to invalidate (see helper module)
+    invalidField*: InvalidPayloadBlockField
+    # Whether to create payloads with empty transactions or not:
+    # Used to test scenarios where the stateRoot is invalidated but its invalidation
+    # goes unnoticed by the client because of the lack of transactions.
+    emptyTransactions*: bool
+    # Height of the common ancestor in the proof-of-stake chain.
+    # Value of 0 means the common ancestor is the terminal proof-of-work block.
+    commonAncestorHeight*: Option[int]
+    # Amount of payloads to produce between the common ancestor and the head of the
+    # proof-of-stake chain.
+    deviatingPayloadCount*: Option[int]
+    # Whether the syncing client must re-org from a canonical chain.
+    # If set to true, the client is driven through a valid canonical chain first,
+    # and then the client is prompted to re-org to the invalid chain.
+    # If set to false, the client is prompted to sync from the genesis
+    # or start chain (if specified).
+    reOrgFromCanonical*: bool
+
+
+method withMainFork(cs: InvalidMissingAncestorReOrgSyncTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: InvalidMissingAncestorReOrgSyncTest): string =
+  var name = "Invalid Missing Ancestor ReOrg"
+  name.add ", Invalid " & $cs.invalidField
+
+  if cs.emptyTransactions:
+    name.add ", Empty Txs"
+
+  name.add ", Invalid P" & $cs.invalidIndex & "'"
+  name.add ", Reveal using sync"
+
+  if cs.reOrgFromCanonical:
+    name.add ", ReOrg from Canonical"
+  return name
+
+method execute(cs: InvalidMissingAncestorReOrgSyncTest, env: TestEnv): bool =
+  var sec = env.addEngine(true, cs.reOrgFromCanonical)
+
+  # Remove the original client so that it does not receive the payloads created on the canonical chain
+  if not cs.reOrgFromCanonical:
+    env.clMock.removeEngine(env.engine)
+
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  let shadow = Shadow(
+    cAHeight: 5,
+    n: 10
+  )
+
+  # Produce blocks before starting the test
+  # Default is to produce 5 PoS blocks before the common ancestor
+  if cs.commonAncestorHeight.isSome:
+    shadow.cAHeight = cs.commonAncestorHeight.get
+
+  # Save the common ancestor
+  doAssert(shadow.cAHeight != 0, "Invalid common ancestor height: " & $shadow.cAHeight)
+  testCond env.clMock.produceBlocks(shadow.cAHeight, BlockProcessCallbacks())
+
+  # Amount of blocks to deviate starting from the common ancestor
+  # Default is to deviate 10 payloads from the common ancestor
+  if cs.deviatingPayloadCount.isSome:
+    shadow.n = cs.deviatingPayloadCount.get
+
+  # Slice to save the side B chain
+  # Append the common ancestor
+  shadow.payloads.add env.clMock.latestExecutableData
+
+  # Produce blocks but at the same time create an side chain which contains an invalid payload at some point (INV_P)
+  # CommonAncestor◄─▲── P1 ◄─ P2 ◄─ P3 ◄─ ... ◄─ Pn
+  #                 │
+  #                 └── P1' ◄─ P2' ◄─ ... ◄─ INV_P ◄─ ... ◄─ Pn'
+  info "Starting canonical chain production"
+  var pbRes = env.clMock.produceBlocks(shadow.n, BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      # Function to send at least one transaction each block produced.
+      # Empty Txs Payload with invalid stateRoot discovered an issue in geth sync, hence this is customizable.
+      if not cs.emptyTransactions:
+        # Send the transaction to the globals.PrevRandaoContractAddr
+        let tc = BaseTx(
+          recipient:  some(prevRandaoContractAddr),
+          amount:     1.u256,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        )
+        let ok = env.sendNextTx(env.clMock.nextBlockProducer, tc)
+        testCond ok:
+          fatal "Error trying to send transaction: "
+      return true
+    ,
+    onGetPayload: proc(): bool =
+      var
+        # Insert extraData to ensure we deviate from the main payload, which contains empty extradata
+        pHash = shadow.payloads[^1].blockHash
+        customizer = CustomPayloadData(
+          parentHash: some(ethHash pHash),
+          extraData:  some(@[0x01.byte]),
+        )
+        sidePayload = customizer.customizePayload(env.clMock.latestExecutableData)
+
+      if shadow.payloads.len == cs.invalidIndex:
+        sidePayload = env.generateInvalidPayload(sidePayload, cs.invalidField)
+
+      shadow.payloads.add sidePayload
+      # TODO: This could be useful to try to produce an invalid block that has some invalid field not included in the ExecutableData
+      #let sideBlock = sidePayload.basePayload
+      #if shadow.payloads.len == cs.invalidIndex:
+      #  var uncle *types.Block
+      #  if cs.invalidField == InvalidOmmers:
+      #    let number = sideBlock.blockNumber.uint64-1
+      #    doAssert(env.clMock.executedPayloadHistory.hasKey(number), "FAIL: Unable to get uncle block")
+      #    let unclePayload = env.clMock.executedPayloadHistory[number]
+      #      # Uncle is a PoS payload
+      #      uncle, err = ExecutableDataToBlock(*unclePayload)
+      #
+      #  # Invalidate fields not available in the ExecutableData
+      #  sideBlock, err = generateInvalidPayloadBlock(sideBlock, uncle, cs.invalidField)
+      return true
+  ))
+  testCond pbRes
+
+  info "Starting side chain production"
+  pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Note: We perform the test in the middle of payload creation by the CL Mock, in order to be able to
+    # re-org back into this chain and use the new payload without issues.
+    onGetPayload: proc(): bool =
+      # Now let's send the side chain to the client using newPayload/sync
+      for i in 1..<shadow.n:
+        # Send the payload
+        var payloadValidStr = "VALID"
+        if i == cs.invalidIndex:
+          payloadValidStr = "INVALID"
+        elif i > cs.invalidIndex:
+          payloadValidStr = "VALID with INVALID ancestor"
+
+        info "Invalid chain payload", i, msg=payloadValidStr
+
+        if i < cs.invalidIndex:
+          let p = shadow.payloads[i]
+          let version = sec.version(p.timestamp)
+          let r = sec.client.newPayload(version, p)
+          #r.ExpectationDescription = "Sent modified payload to secondary client, expected to be accepted"
+          r.expectStatusEither([PayloadExecutionStatus.valid, PayloadExecutionStatus.accepted])
+          let fcu = ForkchoiceStateV1(
+            headblockHash: p.blockHash,
+          )
+          let s = sec.client.forkchoiceUpdated(version, fcu)
+          #s.ExpectationDescription = "Sent modified payload forkchoice updated to secondary client, expected to be accepted"
+          s.expectStatusEither([PayloadExecutionStatus.valid, PayloadExecutionStatus.syncing])
+
+        else:
+          debugEcho "i: ", i, " cs.invalidIndex: ", cs.invalidIndex
+          doAssert(false, "Should not happen")
+          #invalidBlock, err = ExecutableDataToBlock(*shadow.payloads[i])
+          #if err = secondaryClient.SetBlock(invalidBlock, shadow.payloads[i-1].blockNumber, shadow.payloads[i-1].StateRoot); err != nil (
+          #  fatal "TEST ISSUE - Failed to set invalid block: " err)
+          #)
+          #info "Invalid block successfully set %d (%s): " i, payloadValidStr, invalidBlock.Hash())
+
+      # Check that the second node has the correct head
+      var res = sec.client.latestHeader()
+      testCond res.isOk:
+        fatal "TEST ISSUE - Secondary Node unable to reatrieve latest header: ", msg=res.error
+
+      let head = res.get()
+      testCond head.blockHash == ethHash(shadow.payloads[shadow.n-1].blockHash):
+        fatal "TEST ISSUE - Secondary Node has invalid blockHash",
+          got=head.blockHash.short,
+          want=shadow.payloads[shadow.n-1].blockHash.short,
+          gotNum=head.blockNumber,
+          wantNum=shadow.payloads[shadow.n].blockNumber
+
+      info "Secondary Node has correct block"
+
+      if not cs.reOrgFromCanonical:
+        # Add the main client as a peer of the secondary client so it is able to sync
+        sec.connect(env.engine.node)
+
+        let res = env.engine.client.latestHeader()
+        testCond res.isOk:
+          fatal "Unable to query main client for latest block", msg=res.error
+
+        let head = res.get
+        info "Latest block on main client before sync",
+          hash=head.blockHash.short,
+          number=head.blockNumber
+
+      # If we are syncing through p2p, we need to keep polling until the client syncs the missing payloads
+      #[for (
+        r = env.engine.client.newPayload(shadow.payloads[shadow.n])
+        info "Response from main client: " r.Status)
+        s = env.engine.client.forkchoiceUpdated(ForkchoiceStateV1(
+          headblockHash: shadow.payloads[shadow.n].blockHash,
+        ), nil, shadow.payloads[shadow.n].timestamp)
+        info "Response from main client fcu: " s.Response.PayloadStatus)
+
+        if r.Status.Status == PayloadExecutionStatus.invalid (
+          # We also expect that the client properly returns the LatestValidHash of the block on the
+          # side chain that is immediately prior to the invalid payload (or zero if parent is PoW)
+          var lvh common.Hash
+          if shadow.cAHeight != 0 || cs.invalidIndex != 1 (
+            # Parent is NOT Proof of Work
+            lvh = shadow.payloads[cs.invalidIndex-1].blockHash
+          )
+          r.expectLatestValidHash(lvh)
+          # Response on ForkchoiceUpdated should be the same
+          s.expectPayloadStatus(PayloadExecutionStatus.invalid)
+          s.expectLatestValidHash(lvh)
+          break
+        elif test.PayloadStatus(r.Status.Status) == PayloadExecutionStatus.valid (
+          ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+          defer cancel()
+          latestBlock, err = t.Eth.BlockByNumber(ctx, nil)
+          if err != nil (
+            fatal "Unable to get latest block: " err)
+          )
+
+          # Print last shadow.n blocks, for debugging
+          k = latestBlock.blockNumber().Int64() - int64(shadow.n)
+          if k < 0 (
+            k = 0
+          )
+          for ; k <= latestBlock.blockNumber().Int64(); k++ (
+            ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+            defer cancel()
+            latestBlock, err = t.Eth.BlockByNumber(ctx, big.NewInt(k))
+            if err != nil (
+              fatal "Unable to get block %d: " k, err)
+            )
+            js, _ = json.MarshalIndent(latestBlock.Header(), "", "  ")
+            info "Block %d: %s", t.TestName, k, js)
+          )
+
+          fatal "Client returned VALID on an invalid chain: " r.Status)
+        )
+
+        select (
+        case <-time.After(time.Second):
+          continue
+        case <-t.TimeoutContext.Done():
+          fatal "Timeout waiting for main client to detect invalid chain", t.TestName)
+        )
+      )
+
+      if !cs.reOrgFromCanonical (
+        # We need to send the canonical chain to the main client here
+        for i = env.clMock.firstPoSBlockNumber.Uint64(); i <= env.clMock.latestExecutedPayload.blockNumber; i++ (
+          if payload, ok = env.clMock.executedPayloadHistory[i]; ok (
+            r = env.engine.client.newPayload(payload)
+            r.expectStatus(PayloadExecutionStatus.valid)
+          )
+        )
+      )
+
+      # Resend the latest correct fcU
+      r = env.engine.client.forkchoiceUpdated(env.clMock.latestForkchoice, nil, env.clMock.latestPayloadBuilt.timestamp)
+      r.expectNoError()
+      # After this point, the CL Mock will send the next payload of the canonical chain]#
+      return true
+  ))
+
+  testCond pbRes
+  return true

hive_integration/nodocker/engine/engine/invalid_payload.nim

@@ -0,0 +1,426 @@
+import
+  std/strutils,
+  chronicles,
+  ./engine_spec,
+  ../helper,
+  ../cancun/customizer,
+  ../../../../nimbus/common
+
+
+# Generate test cases for each field of NewPayload, where the payload contains a single invalid field and a valid hash.
+type
+  InvalidPayloadTestCase* = ref object of EngineSpec
+    # invalidField is the field that will be modified to create an invalid payload
+    invalidField*: InvalidPayloadBlockField
+    # syncing is true if the client is expected to be in syncing mode after receiving the invalid payload
+    syncing*: bool
+    # emptyTransactions is true if the payload should not contain any transactions
+    emptyTransactions*: bool
+    # If true, the payload can be detected to be invalid even when syncing,
+    # but this check is optional and both `INVALID` and `syncing` are valid responses.
+    invalidDetectedOnSync*: bool
+    # If true, latest valid hash can be nil for this test.
+    nilLatestValidHash*: bool
+
+  InvalidPayloadShadow = ref object
+    alteredPayload       : ExecutableData
+    invalidDetectedOnSync: bool
+    nilLatestValidHash   : bool
+
+method withMainFork(cs: InvalidPayloadTestCase, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: InvalidPayloadTestCase): string =
+  var name = "Invalid " & $cs.invalidField & " NewPayload"
+  if cs.syncing:
+    name.add " - syncing"
+
+  if cs.emptyTransactions:
+    name.add " - Empty Transactions"
+
+  if cs.txType.get(TxLegacy) == TxEip1559:
+    name.add " - "
+    name.add $cs.txType.get
+
+  return name
+
+method execute(cs: InvalidPayloadTestCase, env: TestEnv): bool =
+  # To allow sending the primary engine client into syncing state,
+  # we need a secondary client to guide the payload creation
+  let sec = if cs.syncing: env.addEngine()
+            else: EngineEnv(nil)
+
+  # Wait until TTD is reached by all clients
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  let txFunc = proc(): bool =
+    if not cs.emptyTransactions:
+      # Function to send at least one transaction each block produced
+      # Send the transaction to the globals.PrevRandaoContractAddr
+      let eng = env.clMock.nextBlockProducer
+      let ok = env.sendNextTx(
+        eng,
+        BaseTx(
+          recipient:  some(prevRandaoContractAddr),
+          amount:     1.u256,
+          txType:     cs.txType,
+          gasLimit:   75000.GasInt,
+        ),
+      )
+      testCond ok:
+        fatal "Error trying to send transaction"
+    return true
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks(
+    # Make sure at least one transaction is included in each block
+    onPayloadProducerSelected: txFunc,
+  ))
+
+  if cs.syncing:
+    # Disconnect the main engine client from the CL Mocker and produce a block
+    env.clMock.removeEngine(env.engine)
+    testCond env.clMock.produceSingleBlock(BlockProcessCallbacks(
+      onPayloadProducerSelected: txFunc,
+    ))
+
+    ## This block is now unknown to the main client, sending an fcU will set it to cs.syncing mode
+    let version = env.engine.version(env.clMock.latestPayloadBuilt.timestamp)
+    let r = env.engine.client.forkchoiceUpdated(version, env.clMock.latestForkchoice)
+    r.expectPayloadStatus(PayloadExecutionStatus.syncing)
+
+  let shadow = InvalidPayloadShadow(
+    invalidDetectedOnSync: cs.invalidDetectedOnSync,
+    nilLatestValidHash   : cs.nilLatestValidHash,
+  )
+
+  var pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Make sure at least one transaction is included in the payload
+    onPayloadProducerSelected: txFunc,
+    # Run test after the new payload has been obtained
+    onGetPayload: proc(): bool =
+      # Alter the payload while maintaining a valid hash and send it to the client, should produce an error
+
+      # We need at least one transaction for most test cases to work
+      if not cs.emptyTransactions and env.clMock.latestPayloadBuilt.transactions.len == 0:
+        # But if the payload has no transactions, the test is invalid
+        fatal "No transactions in the base payload"
+        return false
+
+      let execData = env.clMock.latestExecutableData
+      shadow.alteredPayload = env.generateInvalidPayload(execData, cs.invalidField)
+
+      if execData.versionedHashes.isSome and cs.invalidField == RemoveTransaction:
+        let vs = execData.versionedHashes.get
+        if vs.len > 0:
+          # If the payload has versioned hashes, and we removed any transaction, it's highly likely the client will
+          # be able to detect the invalid payload even when syncing because of the blob gas used.
+          shadow.invalidDetectedOnSync = true
+          shadow.nilLatestValidHash = true
+
+      # Depending on the field we modified, we expect a different status
+      var version = env.engine.version(shadow.alteredPayload.timestamp)
+      let r = env.engine.client.newPayload(version, shadow.alteredPayload)
+      if cs.syncing or cs.invalidField == InvalidparentHash:
+        # Execution specification::
+        # (status: ACCEPTED, latestValidHash: null, validationError: null) if the following conditions are met:
+        #  - the blockHash of the payload is valid
+        #  - the payload doesn't extend the canonical chain
+        #  - the payload hasn't been fully validated
+        # (status: syncing, latestValidHash: null, validationError: null)
+        # if the payload extends the canonical chain and requisite data for its validation is missing
+        # (the client can assume the payload extends the canonical because the linking payload could be missing)
+        if shadow.invalidDetectedOnSync:
+          # For some fields, the client can detect the invalid payload even when it doesn't have the parent.
+          # However this behavior is up to the client, so we can't expect it to happen and syncing is also valid.
+          # `VALID` response is still incorrect though.
+          r.expectStatusEither([PayloadExecutionStatus.invalid, PayloadExecutionStatus.accepted, PayloadExecutionStatus.syncing])
+          # TODO: It seems like latestValidHash==nil should always be expected here.
+        else:
+          r.expectStatusEither([PayloadExecutionStatus.accepted, PayloadExecutionStatus.syncing])
+          r.expectLatestValidHash()
+      else:
+        r.expectStatus(PayloadExecutionStatus.invalid)
+        if not (shadow.nilLatestValidHash and r.get.latestValidHash.isNone):
+          r.expectLatestValidHash(shadow.alteredPayload.parentHash)
+
+      # Send the forkchoiceUpdated with a reference to the invalid payload.
+      let fcState = ForkchoiceStateV1(
+        headblockHash:      shadow.alteredPayload.blockHash,
+        safeblockHash:      shadow.alteredPayload.blockHash,
+        finalizedblockHash: shadow.alteredPayload.blockHash,
+      )
+
+      var attr = env.clMock.latestPayloadAttributes
+      attr.timestamp = w3Qty(shadow.alteredPayload.timestamp, 1)
+      attr.prevRandao = w3Hash()
+      attr.suggestedFeeRecipient = w3Address()
+
+      # Execution specification:
+      #  (payloadStatus: (status: INVALID, latestValidHash: null, validationError: errorMessage | null), payloadId: null)
+      #  obtained from the Payload validation process if the payload is deemed INVALID
+      version = env.engine.version(shadow.alteredPayload.timestamp)
+      let s = env.engine.client.forkchoiceUpdated(version, fcState, some(attr))
+      if not cs.syncing:
+        # Execution specification:
+        #  (payloadStatus: (status: INVALID, latestValidHash: null, validationError: errorMessage | null), payloadId: null)
+        #  obtained from the Payload validation process if the payload is deemed INVALID
+        # Note: syncing/ACCEPTED is acceptable here as long as the block produced after this test is produced successfully
+        s.expectStatusEither([PayloadExecutionStatus.syncing, PayloadExecutionStatus.accepted, PayloadExecutionStatus.invalid])
+      else:
+        # At this moment the response should be syncing
+        s.expectPayloadStatus(PayloadExecutionStatus.syncing)
+
+        # When we send the previous payload, the client must now be capable of determining that the invalid payload is actually invalid
+        let p = env.engine.client.newPayload(env.clMock.latestExecutedPayload)
+        p.expectStatus(PayloadExecutionStatus.valid)
+        p.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)
+
+
+        # Another option here could be to send an fcU to the previous payload,
+        # but this does not seem like something the CL would do.
+        #s = env.engine.client.forkchoiceUpdated(ForkchoiceStateV1(
+        #  headblockHash:      previousPayload.blockHash,
+        #  safeblockHash:      previousPayload.blockHash,
+        #  finalizedblockHash: previousPayload.blockHash,
+        #), nil)
+        #s.expectPayloadStatus(Valid)
+
+
+        let q = env.engine.client.newPayload(version, shadow.alteredPayload)
+        if cs.invalidField == InvalidparentHash:
+          # There is no invalid parentHash, if this value is incorrect,
+          # it is assumed that the block is missing and we need to sync.
+          # ACCEPTED also valid since the CLs normally use these interchangeably
+          q.expectStatusEither([PayloadExecutionStatus.syncing, PayloadExecutionStatus.accepted])
+          q.expectLatestValidHash()
+        elif cs.invalidField == InvalidNumber:
+          # A payload with an invalid number can force us to start a sync cycle
+          # as we don't know if that block might be a valid future block.
+          q.expectStatusEither([PayloadExecutionStatus.invalid, PayloadExecutionStatus.syncing])
+          if q.get.status == PayloadExecutionStatus.invalid:
+            q.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)
+          else:
+            q.expectLatestValidHash()
+        else:
+          # Otherwise the response should be INVALID.
+          q.expectStatus(PayloadExecutionStatus.invalid)
+          if not (shadow.nilLatestValidHash and r.get.latestValidHash.isNone):
+            q.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)
+
+        # Try sending the fcU again, this time we should get the proper invalid response.
+        # At this moment the response should be INVALID
+        if cs.invalidField != InvalidparentHash:
+          let version = env.engine.version(shadow.alteredPayload.timestamp)
+          let s = env.engine.client.forkchoiceUpdated(version, fcState)
+          # Note: syncing is acceptable here as long as the block produced after this test is produced successfully
+          s.expectStatusEither([PayloadExecutionStatus.syncing, PayloadExecutionStatus.invalid])
+
+      # Finally, attempt to fetch the invalid payload using the JSON-RPC endpoint
+      let p = env.engine.client.headerByHash(ethHash shadow.alteredPayload.blockHash)
+      p.expectError()
+      return true
+  ))
+  testCond pbRes
+
+  if cs.syncing:
+    # Send the valid payload and its corresponding forkchoiceUpdated
+    let r = env.engine.client.newPayload(env.clMock.latestExecutedPayload)
+    r.expectStatus(PayloadExecutionStatus.valid)
+    r.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)
+
+    let version = env.engine.version(env.clMock.latestExecutedPayload.timestamp)
+    let s = env.engine.client.forkchoiceUpdated(version, env.clMock.latestForkchoice)
+    s.expectPayloadStatus(PayloadExecutionStatus.valid)
+    s.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)
+
+    # Add main client again to the CL Mocker
+    env.clMock.addEngine(env.engine)
+
+
+  # Lastly, attempt to build on top of the invalid payload
+  pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Run test after the new payload has been obtained
+    onGetPayload: proc(): bool =
+      if env.clMock.latestPayloadBuilt.parentHash == shadow.alteredPayload.blockHash:
+        # In some instances the payload is indiscernible from the altered one because the
+        # difference lies in the new payload parameters, in this case skip this check.
+        return true
+
+      let customizer = CustomPayloadData(
+        parentHash: some(ethHash shadow.alteredPayload.blockHash),
+      )
+
+      let followUpAlteredPayload = customizer.customizePayload(env.clMock.latestExecutableData)
+      info "Sending customized Newpayload",
+        parentHash=env.clMock.latestPayloadBuilt.parentHash.short,
+        hash=shadow.alteredPayload.blockHash.short
+
+      # Response status can be ACCEPTED (since parent payload could have been thrown out by the client)
+      # or syncing (parent payload is thrown out and also client assumes that the parent is part of canonical chain)
+      # or INVALID (client still has the payload and can verify that this payload is incorrectly building on top of it),
+      # but a VALID response is incorrect.
+      let version = env.engine.version(followUpAlteredPayload.timestamp)
+      let r = env.engine.client.newPayload(version, followUpAlteredPayload)
+      r.expectStatusEither([PayloadExecutionStatus.accepted, PayloadExecutionStatus.invalid, PayloadExecutionStatus.syncing])
+      if r.get.status in [PayloadExecutionStatus.accepted, PayloadExecutionStatus.syncing]:
+        r.expectLatestValidHash()
+      elif r.get.status == PayloadExecutionStatus.invalid:
+        if not (shadow.nilLatestValidHash or r.get.latestValidHash.isNone):
+          r.expectLatestValidHash(shadow.alteredPayload.parentHash)
+
+      return true
+  ))
+  testCond pbRes
+  return true
+
+# Build on top of the latest valid payload after an invalid payload had been received:
+# P <- INV_P, newPayload(INV_P), fcU(head: P, payloadAttributes: attrs) + getPayload(…)
+type
+  PayloadBuildAfterInvalidPayloadTest* = ref object of EngineSpec
+    invalidField*: InvalidPayloadBlockField
+
+method withMainFork(cs: PayloadBuildAfterInvalidPayloadTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: PayloadBuildAfterInvalidPayloadTest): string =
+  "Payload Build after New Invalid payload: Invalid " & $cs.invalidField
+
+method execute(cs: PayloadBuildAfterInvalidPayloadTest, env: TestEnv): bool =
+  # Add a second client to build the invalid payload
+  let sec = env.addEngine()
+
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  # Produce another block, but at the same time send an invalid payload from the other client
+  let pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onPayloadAttributesGenerated: proc(): bool =
+      # We are going to use the client that was not selected
+      # by the CLMocker to produce the invalid payload
+      var invalidPayloadProducer = env.engine
+      if env.clMock.nextBlockProducer == invalidPayloadProducer:
+        invalidPayloadProducer = sec
+
+      var inv_p: ExecutableData
+      block:
+        # Get a payload from the invalid payload producer and invalidate it
+        let
+          customizer = BasePayloadAttributesCustomizer(
+            prevRandao: some(common.Hash256()),
+            suggestedFeerecipient: some(ZeroAddr),
+          )
+          payloadAttributes = customizer.getPayloadAttributes(env.clMock.latestPayloadAttributes)
+          version = env.engine.version(env.clMock.latestHeader.timestamp)
+          r = invalidPayloadProducer.client.forkchoiceUpdated(version, env.clMock.latestForkchoice, some(payloadAttributes))
+
+        r.expectPayloadStatus(PayloadExecutionStatus.valid)
+        # Wait for the payload to be produced by the EL
+        let period = chronos.seconds(1)
+        waitFor sleepAsync(period)
+
+        let
+          versione = env.engine.version(payloadAttributes.timestamp)
+          s = invalidPayloadProducer.client.getPayload(r.get.payloadID.get, versione)
+        s.expectNoError()
+
+        var src = ExecutableData(basePayload: s.get.executionPayload)
+        inv_p = env.generateInvalidPayload(src, InvalidStateRoot)
+
+      # Broadcast the invalid payload
+      let
+        version = env.engine.version(inv_p.timestamp)
+        r = env.engine.client.newPayload(version, inv_p)
+
+      r.expectStatus(PayloadExecutionStatus.invalid)
+      r.expectLatestValidHash(env.clMock.latestForkchoice.headblockHash)
+
+      let s = sec.client.newPayload(version, inv_p)
+      s.expectStatus(PayloadExecutionStatus.invalid)
+      s.expectLatestValidHash(env.clMock.latestForkchoice.headblockHash)
+
+      # Let the block production continue.
+      # At this point the selected payload producer will
+      # try to continue creating a valid payload.
+      return true
+  ))
+  testCond pbRes
+  return true
+
+type
+  InvalidTxChainIDTest* = ref object of EngineSpec
+  InvalidTxChainIDShadow = ref object
+    invalidTx: Transaction
+
+method withMainFork(cs: InvalidTxChainIDTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: InvalidTxChainIDTest): string =
+  "Build Payload with Invalid ChainID Transaction " & $cs.txType
+
+# Attempt to produce a payload after a transaction with an invalid Chain ID was sent to the client
+# using `eth_sendRawTransaction`.
+method execute(cs: InvalidTxChainIDTest, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  # Send a transaction with an incorrect ChainID.
+  # Transaction must be not be included in payload creation.
+  var shadow = InvalidTxChainIDShadow()
+  let pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    # Run test after a new payload has been broadcast
+    onPayloadAttributesGenerated: proc(): bool =
+      let txCreator = BaseTx(
+        recipient:  some(prevRandaoContractAddr),
+        amount:     1.u256,
+        txType:     cs.txType,
+        gasLimit:   75000,
+      )
+
+      let
+        sender = env.accounts(0)
+        eng = env.clMock.nextBlockProducer
+        res = eng.client.nonceAt(sender.address)
+
+      testCond res.isOk:
+        fatal "Unable to get address nonce", msg=res.error
+
+      let
+        nonce = res.get
+        tx = env.makeTx(txCreator, sender, nonce)
+        chainId = eng.com.chainId
+
+      let txCustomizerData = CustomTransactionData(
+        chainID: some((chainId.uint64 + 1'u64).ChainId)
+      )
+
+      shadow.invalidTx = env.customizeTransaction(sender, tx, txCustomizerData)
+      testCond env.sendTx(shadow.invalidTx):
+        info "Error on sending transaction with incorrect chain ID"
+
+      return true
+  ))
+
+  testCond pbRes
+
+  # Verify that the latest payload built does NOT contain the invalid chain Tx
+  let txHash = shadow.invalidTx.rlpHash
+  if txInPayload(env.clMock.latestPayloadBuilt, txHash):
+    fatal "Invalid chain ID tx was included in payload"
+    return false
+
+  return true

hive_integration/nodocker/engine/engine/misc.nim

@@ -0,0 +1,42 @@
+import
+  ./engine_spec,
+  ../../../../nimbus/common/hardforks
+
+# Runs a sanity test on a post Merge fork where a previous fork's (London) number is not zero
+type
+  NonZeroPreMergeFork* = ref object of EngineSpec
+
+method withMainFork(cs: NonZeroPreMergeFork, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: NonZeroPreMergeFork): string =
+  "Pre-Merge Fork Number > 0"
+
+method getForkConfig*(cs: NonZeroPreMergeFork): ChainConfig =
+  let forkConfig = procCall getForkConfig(BaseSpec(cs))
+  if forkConfig.isNil:
+    return nil
+    
+  # Merge fork & pre-merge happen at block 1
+  forkConfig.londonBlock = some(1.u256)
+  forkConfig.mergeForkBlock = some(1.u256)
+
+  # Post-merge fork happens at block 2
+  let mainFork = BaseSpec(cs).getMainFork()
+  if mainFork == ForkCancun:
+    forkConfig.shanghaiTime = forkConfig.cancunTime
+
+  return forkConfig
+
+method execute(cs: NonZeroPreMergeFork, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Simply produce a couple of blocks without transactions (if London is not active at genesis
+  # we can't send type-2 transactions) and check that the chain progresses without issues
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  return true

hive_integration/nodocker/engine/engine/payload_attributes.nim

@@ -0,0 +1,79 @@
+import
+  std/strutils,
+  chronicles,
+  ./engine_spec,
+  ../cancun/customizer
+
+type
+  InvalidPayloadAttributesTest* = ref object of EngineSpec
+    description*: string
+    customizer* : PayloadAttributesCustomizer
+    syncing*    : bool
+    errorOnSync*: bool
+
+method withMainFork(cs: InvalidPayloadAttributesTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: InvalidPayloadAttributesTest): string =
+  var desc = "Invalid PayloadAttributes: " & cs.description
+  if cs.syncing:
+    desc.add " (Syncing)"
+  desc
+
+method execute(cs: InvalidPayloadAttributesTest, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  # Send a forkchoiceUpdated with invalid PayloadAttributes
+  let pbRes = env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onNewPayloadBroadcast: proc(): bool =
+      # Try to apply the new payload with invalid attributes
+      var fcu = env.clMock.latestForkchoice
+      if cs.syncing:
+        # Setting a random hash will put the client into `SYNCING`
+        fcu.headblockHash = Web3Hash.randomBytes()
+      else:
+        fcu.headblockHash = env.clMock.latestPayloadBuilt.blockHash
+
+      info "Sending EngineForkchoiceUpdated with invalid payload attributes",
+        syncing=cs.syncing, description=cs.description
+
+      # Get the payload attributes
+      var originalAttr = env.clMock.latestPayloadAttributes
+      originalAttr.timestamp = w3Qty(originalAttr.timestamp, 1)
+      let attr = cs.customizer.getPayloadAttributes(originalAttr)
+
+      # 0) Check headBlock is known and there is no missing data, if not respond with SYNCING
+      # 1) Check headBlock is VALID, if not respond with INVALID
+      # 2) Apply forkchoiceState
+      # 3) Check payloadAttributes, if invalid respond with error: code: Invalid payload attributes
+      # 4) Start payload build process and respond with VALID
+      let version = env.engine.version(env.clMock.latestPayloadBuilt.timestamp)
+      if cs.syncing:
+        # If we are SYNCING, the outcome should be SYNCING regardless of the validity of the payload atttributes
+        let r = env.engine.client.forkchoiceUpdated(version, fcu, some(attr))
+        if cs.errorOnSync:
+          r.expectError()
+        else:
+          r.expectPayloadStatus(PayloadExecutionStatus.syncing)
+          r.expectPayloadID(none(PayloadID))
+      else:
+        let r = env.engine.client.forkchoiceUpdated(version, fcu, some(attr))
+        r.expectError()
+
+        # Check that the forkchoice was applied, regardless of the error
+        let s = env.engine.client.latestHeader()
+        #s.ExpectationDescription = "Forkchoice is applied even on invalid payload attributes"
+        s.expectHash(ethHash fcu.headblockHash)
+
+      return true
+    ))
+
+  testCond pbRes
+  return true

hive_integration/nodocker/engine/engine/payload_execution.nim

@@ -0,0 +1,463 @@
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  ReExecutePayloadTest* = ref object of EngineSpec
+    payloadCount*: int
+
+method withMainFork(cs: ReExecutePayloadTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ReExecutePayloadTest): string =
+  "Re-Execute Payload"
+
+# Consecutive Payload Execution: Secondary client should be able to set the forkchoiceUpdated to payloads received consecutively
+method execute(cs: ReExecutePayloadTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # How many Payloads we are going to re-execute
+  let payloadReExecCount = if cs.payloadCount > 0: cs.payloadCount
+                           else: 10
+  # Create those blocks
+  let pbRes = env.clMock.produceBlocks(payloadReExecCount, BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      # Send at least one transaction per payload
+      _, err = env.sendNextTx(
+        env.clMock.nextBlockProducer,
+        BaseTx(
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", t.TestName, err)
+      )
+    ),
+    onGetPayload: proc(): bool =
+      # Check that the transaction was included
+      if len(env.clMock.latestPayloadBuilt.transactions) == 0 (
+        fatal "Client failed to include the expected transaction in payload built", t.TestName)
+      )
+    ),
+  ))
+
+  # Re-execute the payloads
+  r = env.engine.client.blockNumber()
+  r.expectNoError()
+  lastBlock = r.blockNumber
+  info "Started re-executing payloads at block: %v", t.TestName, lastBlock)
+
+  for i = lastBlock - uint64(payloadReExecCount) + 1; i <= lastBlock; i++ (
+    payload, found = env.clMock.executedPayloadHistory[i]
+    if !found (
+      fatal "(test issue) Payload with index %d does not exist", i)
+    )
+
+    r = env.engine.client.newPayload(payload)
+    r.expectStatus(PayloadExecutionStatus.valid)
+    r.expectLatestValidHash(payload.blockHash)
+  )
+)
+
+type
+  InOrderPayloadExecutionTest* = ref object of EngineSpec
+
+method withMainFork(cs: InOrderPayloadExecutionTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: InOrderPayloadExecutionTest): string =
+  "In-Order Consecutive Payload Execution"
+
+# Consecutive Payload Execution: Secondary client should be able to set the forkchoiceUpdated to payloads received consecutively
+method execute(cs: InOrderPayloadExecutionTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Send a single block to allow sending newer transaction types on the payloads
+  env.clMock.produceSingleBlock(BlockProcessCallbacks())
+
+  # First prepare payloads on a first client, which will also contain multiple transactions
+
+  # We will be also verifying that the transactions are correctly interpreted in the canonical chain,
+  # prepare a random account to receive funds.
+  recipient = EthAddress.randomBytes()
+  amountPerTx = big.NewInt(1000)
+  txPerPayload = 20
+  payloadCount = 10
+  txsIncluded = 0
+
+  env.clMock.produceBlocks(payloadCount, BlockProcessCallbacks(
+    # We send the transactions after we got the Payload ID, before the CLMocker gets the prepared Payload
+    onPayloadProducerSelected: proc(): bool =
+      _, err = env.sendNextTxs(
+        env.clMock.nextBlockProducer,
+        BaseTx(
+          recipient:  &recipient,
+          amount:     amountPerTx,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+        uint64(txPerPayload),
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", t.TestName, err)
+      )
+    ),
+    onGetPayload: proc(): bool =
+      if len(env.clMock.latestPayloadBuilt.Transactions) < (txPerPayload / 2) (
+        fatal "Client failed to include all the expected transactions in payload built: %d < %d", t.TestName, len(env.clMock.latestPayloadBuilt.Transactions), (txPerPayload / 2))
+      )
+      txsIncluded += len(env.clMock.latestPayloadBuilt.Transactions)
+    ),
+  ))
+
+  expectedBalance = amountPerTx.Mul(amountPerTx, big.NewInt(int64(txsIncluded)))
+
+  # Check balance on this first client
+  r = env.engine.client.balanceAt(recipient, nil)
+  r.expectBalanceEqual(expectedBalance)
+
+  # Start a second client to send newPayload consecutively without fcU
+  let sec = env.addEngine(false, false)
+
+  # Send the forkchoiceUpdated with the latestExecutedPayload hash, we should get SYNCING back
+  fcU = ForkchoiceStateV1(
+    headblockHash:      env.clMock.latestExecutedPayload.blockHash,
+    safeblockHash:      env.clMock.latestExecutedPayload.blockHash,
+    finalizedblockHash: env.clMock.latestExecutedPayload.blockHash,
+  )
+
+  s = sec.client.forkchoiceUpdated(fcU, nil, env.clMock.latestExecutedPayload.timestamp)
+  s.expectPayloadStatus(PayloadExecutionStatus.syncing)
+  s.expectLatestValidHash(nil)
+  s.ExpectNoValidationError()
+
+  # Send all the payloads in the increasing order
+  for k = env.clMock.firstPoSBlockNumber.Uint64(); k <= env.clMock.latestExecutedPayload.blockNumber; k++ (
+    payload = env.clMock.executedPayloadHistory[k]
+
+    s = sec.client.newPayload(payload)
+    s.expectStatus(PayloadExecutionStatus.valid)
+    s.expectLatestValidHash(payload.blockHash)
+
+  )
+
+  s = sec.client.forkchoiceUpdated(fcU, nil, env.clMock.latestExecutedPayload.timestamp)
+  s.expectPayloadStatus(PayloadExecutionStatus.valid)
+  s.expectLatestValidHash(fcU.headblockHash)
+  s.ExpectNoValidationError()
+
+  # At this point we should have our funded account balance equal to the expected value.
+  q = sec.client.balanceAt(recipient, nil)
+  q.expectBalanceEqual(expectedBalance)
+
+  # Add the client to the CLMocker
+  env.clMock.addEngine(secondaryClient)
+
+  # Produce a single block on top of the canonical chain, all clients must accept this
+  env.clMock.produceSingleBlock(BlockProcessCallbacks())
+
+  # Head must point to the latest produced payload
+  p = sec.client.TestHeaderByNumber(nil)
+  p.expectHash(env.clMock.latestExecutedPayload.blockHash)
+)
+
+type
+  MultiplePayloadsExtendingCanonicalChainTest* = ref object of EngineSpec
+    # How many parallel payloads to execute
+    payloadCount*: int
+    # If set to true, the head will be set to the first payload executed by the client
+    # If set to false, the head will be set to the latest payload executed by the client
+    setHeadToFirstPayloadReceived*: bool
+
+method withMainFork(cs: MultiplePayloadsExtendingCanonicalChainTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: MultiplePayloadsExtendingCanonicalChainTest): string =
+  name = "Multiple New Payloads Extending Canonical Chain"
+  if s.SetHeadToFirstPayloadReceived (
+    name += " (FcU to first payload received)"
+  )
+  return name
+)
+
+# Consecutive Payload Execution: Secondary client should be able to set the forkchoiceUpdated to payloads received consecutively
+method execute(cs: MultiplePayloadsExtendingCanonicalChainTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  callbacks = BlockProcessCallbacks(
+    # We send the transactions after we got the Payload ID, before the CLMocker gets the prepared Payload
+    onPayloadProducerSelected: proc(): bool =
+      let recipient = EthAddress.randomBytes()
+      _, err = env.sendNextTx(
+        env.clMock.nextBlockProducer,
+        BaseTx(
+          recipient:  &recipient,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", t.TestName, err)
+      )
+    ),
+  )
+
+  reExecFunc = proc(): bool =
+    payloadCount = 80
+    if cs.payloadCount > 0 (
+      payloadCount = cs.payloadCount
+    )
+
+    basePayload = env.clMock.latestPayloadBuilt
+
+    # Check that the transaction was included
+    if len(basePayload.Transactions) == 0 (
+      fatal "Client failed to include the expected transaction in payload built", t.TestName)
+    )
+
+    # Fabricate and send multiple new payloads by changing the PrevRandao field
+    for i = 0; i < payloadCount; i++ (
+      newPrevRandao = common.Hash256.randomBytes()
+      customizer = CustomPayloadData(
+        prevRandao: &newPrevRandao,
+      )
+      newPayload, err = customizePayload(basePayload)
+      if err != nil (
+        fatal "Unable to customize payload %v: %v", t.TestName, i, err)
+      )
+
+      r = env.engine.client.newPayload(newPayload)
+      r.expectStatus(PayloadExecutionStatus.valid)
+      r.expectLatestValidHash(newPayload.blockHash)
+    )
+  )
+
+  if cs.SetHeadToFirstPayloadReceived (
+    # We are going to set the head of the chain to the first payload executed by the client
+    # Therefore our re-execution function must be executed after the payload was broadcast
+    callbacks.onNewPayloadBroadcast = reExecFunc
+  else:
+    # Otherwise, we execute the payloads after we get the canonical one so it's
+    # executed last
+    callbacks.onGetPayload = reExecFunc
+  )
+
+  env.clMock.produceSingleBlock(callbacks)
+  # At the end the CLMocker continues to try to execute fcU with the original payload, which should not fail
+)
+
+type
+  NewPayloadOnSyncingClientTest* = ref object of EngineSpec
+
+method withMainFork(cs: NewPayloadOnSyncingClientTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: NewPayloadOnSyncingClientTest): string =
+  "Valid NewPayload->ForkchoiceUpdated on Syncing Client"
+
+# Send a valid payload on a client that is currently SYNCING
+method execute(cs: NewPayloadOnSyncingClientTest, env: TestEnv): bool =
+  var
+      # Set a random transaction recipient
+  let recipient = EthAddress.randomBytes()
+    previousPayload ExecutableData
+  sec = env.addEngine()
+
+  # Wait until TTD is reached by all clients
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+
+
+  # Disconnect the first engine client from the CL Mocker and produce a block
+  env.clMock.removeEngine(env.engine)
+  env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      # Send at least one transaction per payload
+      _, err = env.sendNextTx(
+        env.clMock.nextBlockProducer,
+        BaseTx(
+          recipient:  &recipient,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", t.TestName, err)
+      )
+    ),
+    onGetPayload: proc(): bool =
+      # Check that the transaction was included
+      if len(env.clMock.latestPayloadBuilt.Transactions) == 0 (
+        fatal "Client failed to include the expected transaction in payload built", t.TestName)
+      )
+    ),
+  ))
+
+  previousPayload = env.clMock.latestPayloadBuilt
+
+  # Send the fcU to set it to syncing mode
+  r = env.engine.client.forkchoiceUpdated(env.clMock.latestForkchoice, nil, env.clMock.latestHeader.Time)
+  r.expectPayloadStatus(PayloadExecutionStatus.syncing)
+
+  env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      # Send at least one transaction per payload
+      _, err = env.sendNextTx(
+        env.clMock.nextBlockProducer,
+        BaseTx(
+          recipient:  &recipient,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", t.TestName, err)
+      )
+    ),
+    # Run test after the new payload has been obtained
+    onGetPayload: proc(): bool =
+      # Send the new payload from the second client to the first, it won't be able to validate it
+      r = env.engine.client.newPayload(env.clMock.latestPayloadBuilt)
+      r.expectStatusEither(test.Accepted, PayloadExecutionStatus.syncing)
+      r.expectLatestValidHash(nil)
+
+      # Send the forkchoiceUpdated with a reference to the valid payload on the SYNCING client.
+      var (
+        random                = common.Hash256()
+        suggestedFeeRecipient = common.Address()
+      )
+      payloadAttributesCustomizer = &BasePayloadAttributesCustomizer(
+        Random:                &random,
+        SuggestedFeerecipient: &suggestedFeeRecipient,
+      )
+      newPayloadAttributes, err = payloadAttributesCustomizer.GetPayloadAttributes(env.clMock.latestPayloadAttributes)
+      if err != nil (
+        fatal "Unable to customize payload attributes: %v", t.TestName, err)
+      )
+      s = env.engine.client.forkchoiceUpdated(ForkchoiceStateV1(
+        headblockHash:      env.clMock.latestPayloadBuilt.blockHash,
+        safeblockHash:      env.clMock.latestPayloadBuilt.blockHash,
+        finalizedblockHash: env.clMock.latestPayloadBuilt.blockHash,
+      ), newPayloadAttributes, env.clMock.latestPayloadBuilt.timestamp)
+      s.expectPayloadStatus(PayloadExecutionStatus.syncing)
+
+      # Send the previous payload to be able to continue
+      p = env.engine.client.newPayload(previousPayload)
+      p.expectStatus(PayloadExecutionStatus.valid)
+      p.expectLatestValidHash(previousPayload.blockHash)
+
+      # Send the new payload again
+
+      p = env.engine.client.newPayload(env.clMock.latestPayloadBuilt)
+      p.expectStatus(PayloadExecutionStatus.valid)
+      p.expectLatestValidHash(env.clMock.latestPayloadBuilt.blockHash)
+
+      s = env.engine.client.forkchoiceUpdated(ForkchoiceStateV1(
+        headblockHash:      env.clMock.latestPayloadBuilt.blockHash,
+        safeblockHash:      env.clMock.latestPayloadBuilt.blockHash,
+        finalizedblockHash: env.clMock.latestPayloadBuilt.blockHash,
+      ), nil, env.clMock.latestPayloadBuilt.timestamp)
+      s.expectPayloadStatus(PayloadExecutionStatus.valid)
+
+    return true
+  ))
+
+  testCond pbRes
+  return true
+
+type
+  NewPayloadWithMissingFcUTest* = ref object of EngineSpec
+
+method withMainFork(cs: NewPayloadWithMissingFcUTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: NewPayloadWithMissingFcUTest): string =
+  "NewPayload with Missing ForkchoiceUpdated"
+
+# Send a valid `newPayload` in correct order but skip `forkchoiceUpdated` until the last payload
+method execute(cs: NewPayloadWithMissingFcUTest, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Get last genesis block hash
+  genesisHash = env.engine.client.latestHeader().Header.Hash()
+
+  # Produce blocks on the main client, these payloads will be replayed on the secondary client.
+  env.clMock.produceBlocks(5, BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      var recipient common.Address
+      randomBytes(recipient[:])
+      # Send at least one transaction per payload
+      _, err = env.sendNextTx(
+        env.clMock.nextBlockProducer,
+        BaseTx(
+          recipient:  &recipient,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", t.TestName, err)
+      )
+    ),
+    onGetPayload: proc(): bool =
+      # Check that the transaction was included
+      if len(env.clMock.latestPayloadBuilt.Transactions) == 0 (
+        fatal "Client failed to include the expected transaction in payload built", t.TestName)
+      )
+    ),
+  ))
+
+  var sec = env.addEngine()
+
+  # Send each payload in the correct order but skip the ForkchoiceUpdated for each
+  for i = env.clMock.firstPoSBlockNumber.Uint64(); i <= env.clMock.latestHeadNumber.Uint64(); i++ (
+    payload = env.clMock.executedPayloadHistory[i]
+    p = sec.newPayload(payload)
+    p.expectStatus(PayloadExecutionStatus.valid)
+    p.expectLatestValidHash(payload.blockHash)
+  )
+
+  # Verify that at this point, the client's head still points to the last non-PoS block
+  r = sec.latestHeader()
+  r.expectHash(genesisHash)
+
+  # Verify that the head correctly changes after the last ForkchoiceUpdated
+  fcU = ForkchoiceStateV1(
+    headblockHash:      env.clMock.executedPayloadHistory[env.clMock.latestHeadNumber.Uint64()].blockHash,
+    safeblockHash:      env.clMock.executedPayloadHistory[env.clMock.latestHeadNumber.Uint64()-1].blockHash,
+    finalizedblockHash: env.clMock.executedPayloadHistory[env.clMock.latestHeadNumber.Uint64()-2].blockHash,
+  )
+  p = sec.forkchoiceUpdated(fcU, nil, env.clMock.latestHeader.Time)
+  p.expectPayloadStatus(PayloadExecutionStatus.valid)
+  p.expectLatestValidHash(fcU.headblockHash)
+
+  # Now the head should've changed to the latest PoS block
+  s = sec.latestHeader()
+  s.expectHash(fcU.headblockHash)
+)

hive_integration/nodocker/engine/engine/payload_id.nim

@@ -0,0 +1,88 @@
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  PayloadAttributesFieldChange* = enum
+    PayloadAttributesIncreasetimestamp         = "Increase timestamp"
+    PayloadAttributesRandom                    = "Modify Random"
+    PayloadAttributesSuggestedFeeRecipient     = "Modify SuggestedFeeRecipient"
+    PayloadAttributesAddWithdrawal             = "Add Withdrawal"
+    PayloadAttributesModifyWithdrawalAmount    = "Modify Withdrawal Amount"
+    PayloadAttributesModifyWithdrawalIndex     = "Modify Withdrawal Index"
+    PayloadAttributesModifyWithdrawalValidator = "Modify Withdrawal Validator"
+    PayloadAttributesModifyWithdrawalAddress   = "Modify Withdrawal Address"
+    PayloadAttributesRemoveWithdrawal          = "Remove Withdrawal"
+    PayloadAttributesParentBeaconRoot          = "Modify Parent Beacon Root"
+
+  UniquePayloadIDTest* = ref object of EngineSpec
+    fieldModification*: PayloadAttributesFieldChange
+
+method withMainFork(cs: UniquePayloadIDTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: UniquePayloadIDTest): string =
+  "Unique Payload ID - " & $cs.fieldModification
+
+# Check that the payload id returned on a forkchoiceUpdated call is different
+# when the attributes change
+method execute(cs: UniquePayloadIDTest, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onPayloadAttributesGenerated: proc(): bool =
+      payloadAttributes = env.clMock.latestPayloadAttributes
+      case cs.fieldModification (
+      of PayloadAttributesIncreasetimestamp:
+        payloadAttributes.timestamp += 1
+      of PayloadAttributesRandom:
+        payloadAttributes.Random[0] = payloadAttributes.Random[0] + 1
+      of PayloadAttributesSuggestedFeerecipient:
+        payloadAttributes.SuggestedFeeRecipient[0] = payloadAttributes.SuggestedFeeRecipient[0] + 1
+      of PayloadAttributesAddWithdrawal:
+        newWithdrawal = &types.Withdrawal()
+        payloadAttributes.Withdrawals = append(payloadAttributes.Withdrawals, newWithdrawal)
+      of PayloadAttributesRemoveWithdrawal:
+        payloadAttributes.Withdrawals = payloadAttributes.Withdrawals[1:]
+      of PayloadAttributesModifyWithdrawalAmount,
+        PayloadAttributesModifyWithdrawalIndex,
+        PayloadAttributesModifyWithdrawalValidator,
+        PayloadAttributesModifyWithdrawalAddress:
+        if len(payloadAttributes.Withdrawals) == 0 (
+          fatal "Cannot modify withdrawal when there are no withdrawals")
+        )
+        modifiedWithdrawal = *payloadAttributes.Withdrawals[0]
+        case cs.fieldModification (
+        of PayloadAttributesModifyWithdrawalAmount:
+          modifiedWithdrawal.Amount += 1
+        of PayloadAttributesModifyWithdrawalIndex:
+          modifiedWithdrawal.Index += 1
+        of PayloadAttributesModifyWithdrawalValidator:
+          modifiedWithdrawal.Validator += 1
+        of PayloadAttributesModifyWithdrawalAddress:
+          modifiedWithdrawal.Address[0] = modifiedWithdrawal.Address[0] + 1
+        )
+        payloadAttributes.Withdrawals = append(types.Withdrawals(&modifiedWithdrawal), payloadAttributes.Withdrawals[1:]...)
+      of PayloadAttributesParentBeaconRoot:
+        if payloadAttributes.BeaconRoot == nil (
+          fatal "Cannot modify parent beacon root when there is no parent beacon root")
+        )
+        newBeaconRoot = *payloadAttributes.BeaconRoot
+        newBeaconRoot[0] = newBeaconRoot[0] + 1
+        payloadAttributes.BeaconRoot = &newBeaconRoot
+      default:
+        fatal "Unknown field change: %s", cs.fieldModification)
+      )
+
+      # Request the payload with the modified attributes and add the payload ID to the list of known IDs
+      let version = env.engine.version(env.clMock.latestHeader.timestamp)
+      let r = env.engine.client.forkchoiceUpdated(version, env.clMock.latestForkchoice, some(payloadAttributes))
+      r.expectNoError()
+      env.clMock.addPayloadID(env.engine, r.get.payloadID.get)
+    ),
+  ))
+)

hive_integration/nodocker/engine/engine/prev_randao.nim

@@ -0,0 +1,77 @@
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  PrevRandaoTransactionTest* = ref object of EngineSpec
+    blockCount int
+
+method withMainFork(cs: PrevRandaoTransactionTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: PrevRandaoTransactionTest): string =
+  return "PrevRandao Opcode Transactions Test (%s)", cs.txType)
+)
+
+method execute(cs: PrevRandaoTransactionTest, env: TestEnv): bool =
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Create a single block to not having to build on top of genesis
+  env.clMock.produceSingleBlock(BlockProcessCallbacks())
+
+  startBlockNumber = env.clMock.latestHeader.blockNumber.Uint64() + 1
+
+  # Send transactions in PoS, the value of the storage in these blocks must match the prevRandao value
+  var (
+    blockCount     = 10
+    currentTxIndex = 0
+    txs            = make([]typ.Transaction, 0)
+  )
+  if cs.blockCount > 0 (
+    blockCount = cs.blockCount
+  )
+  env.clMock.produceBlocks(blockCount, BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      tx, err = env.sendNextTx(
+        t.TestContext,
+        t.Engine,
+        &BaseTx(
+          recipient:  prevRandaoContractAddr,
+          amount:     big0,
+          payload:    nil,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", t.TestName, err)
+      )
+      txs = append(txs, tx)
+      currentTxIndex++
+    ),
+    onForkchoiceBroadcast: proc(): bool =
+      # Check the transaction tracing, which is client specific
+      expectedPrevRandao = env.clMock.prevRandaoHistory[env.clMock.latestHeader.blockNumber.Uint64()+1]
+      ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+      defer cancel()
+      if err = DebugPrevRandaoTransaction(ctx, t.Client.RPC(), t.Client.Type, txs[currentTxIndex-1],
+        &expectedPrevRandao); err != nil (
+        fatal "Error during transaction tracing: %v", t.TestName, err)
+      )
+    ),
+  ))
+
+  for i = uint64(startBlockNumber); i <= env.clMock.latestExecutedPayload.blockNumber; i++ (
+    checkPrevRandaoValue(t, env.clMock.prevRandaoHistory[i], i)
+  )
+)
+
+func checkPrevRandaoValue(t *test.Env, expectedPrevRandao common.Hash, blockNumber uint64) (
+  storageKey = common.Hash256()
+  storageKey[31] = byte(blockNumber)
+  r = env.engine.client.TestStorageAt(globals.PrevRandaoContractAddr, storageKey, nil)
+  r.ExpectStorageEqual(expectedPrevRandao)
+)

hive_integration/nodocker/engine/engine/reorg.nim

@@ -0,0 +1,818 @@
+import
+  std/strutils,
+  chronicles,
+  ./engine_spec,
+  ../cancun/customizer
+
+type
+  SidechainReOrgTest* = ref object of EngineSpec
+
+method withMainFork(cs: SidechainReOrgTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: SidechainReOrgTest): string =
+  "Sidechain Reorg"
+
+# Reorg to a Sidechain using ForkchoiceUpdated
+method execute(cs: SidechainReOrgTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  # Produce two payloads, send fcU with first payload, check transaction outcome, then reorg, check transaction outcome again
+
+  # This single transaction will change its outcome based on the payload
+  tx, err = t.sendNextTx(
+    t.TestContext,
+    t.Engine,
+    BaseTx(
+      recipient:  &globals.PrevRandaoContractAddr,
+      amount:     big0,
+      payload:    nil,
+      txType:     cs.txType,
+      gasLimit:   75000,
+    ),
+  )
+  if err != nil (
+    fatal "Error trying to send transaction: %v", t.TestName, err)
+  )
+  info "sent tx %v", t.TestName, tx.Hash())
+
+  env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onNewPayloadBroadcast: proc(): bool =
+      # At this point the CLMocker has a payload that will result in a specific outcome,
+      # we can produce an alternative payload, send it, fcU to it, and verify the changes
+      alternativePrevRandao = common.Hash()
+      rand.Read(alternativePrevRandao[:])
+      timestamp = env.clMock.latestPayloadBuilt.timestamp + 1
+      payloadAttributes, err = (BasePayloadAttributesCustomizer(
+        Timestamp: &timestamp,
+        Random:    &alternativePrevRandao,
+      )).getPayloadAttributes(env.clMock.LatestPayloadAttributes)
+      if err != nil (
+        fatal "Unable to customize payload attributes: %v", t.TestName, err)
+      )
+
+      r = env.engine.client.forkchoiceUpdated(
+        &env.clMock.latestForkchoice,
+        payloadAttributes,
+        env.clMock.latestPayloadBuilt.timestamp,
+      )
+      r.expectNoError()
+
+      await sleepAsync(env.clMock.PayloadProductionClientDelay)
+
+      g = env.engine.client.getPayload(r.Response.PayloadID, payloadAttributes)
+      g.expectNoError()
+      alternativePayload = g.Payload
+      if len(alternativePayload.Transactions) == 0 (
+        fatal "alternative payload does not contain the prevRandao opcode tx", t.TestName)
+      )
+
+      s = env.engine.client.newPayload(alternativePayload)
+      s.expectStatus(PayloadExecutionStatus.valid)
+      s.expectLatestValidHash(alternativePayload.blockHash)
+
+      # We sent the alternative payload, fcU to it
+      p = env.engine.client.forkchoiceUpdated(api.ForkchoiceStateV1(
+        headBlockHash:      alternativePayload.blockHash,
+        safeBlockHash:      env.clMock.latestForkchoice.safeBlockHash,
+        finalizedBlockHash: env.clMock.latestForkchoice.finalizedBlockHash,
+      ), nil, alternativePayload.timestamp)
+      p.expectPayloadStatus(PayloadExecutionStatus.valid)
+
+      # PrevRandao should be the alternative prevRandao we sent
+      checkPrevRandaoValue(t, alternativePrevRandao, alternativePayload.blockNumber)
+    ),
+  ))
+  # The reorg actually happens after the CLMocker continues,
+  # verify here that the reorg was successful
+  latestBlockNum = env.clMock.LatestHeadNumber.Uint64()
+  checkPrevRandaoValue(t, env.clMock.PrevRandaoHistory[latestBlockNum], latestBlockNum)
+
+)
+
+# Test performing a re-org that involves removing or modifying a transaction
+type
+  TransactionReOrgScenario = enum
+    TransactionReOrgScenarioReOrgOut            = "Re-Org Out"
+    TransactionReOrgScenarioReOrgBackIn         = "Re-Org Back In"
+    TransactionReOrgScenarioReOrgDifferentBlock = "Re-Org to Different Block"
+    TransactionReOrgScenarioNewPayloadOnRevert  = "New Payload on Revert Back"
+
+type
+  TransactionReOrgTest* = ref object of EngineSpec
+    TransactionCount int
+    Scenario         TransactionReOrgScenario
+
+method withMainFork(cs: TransactionReOrgTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: TransactionReOrgTest): string =
+  name = "Transaction Re-Org"
+  if s.Scenario != "" (
+    name = fmt.Sprintf("%s, %s", name, s.Scenario)
+  )
+  return name
+
+# Test transaction status after a forkchoiceUpdated re-orgs to an alternative hash where a transaction is not present
+method execute(cs: TransactionReOrgTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test (So we don't try to reorg back to the genesis block)
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  # Create transactions that modify the state in order to check after the reorg.
+  var (
+    err                error
+    txCount            = spec.TransactionCount
+    sstoreContractAddr = common.HexToAddress("0000000000000000000000000000000000000317")
+  )
+
+  if txCount == 0 (
+    # Default is to send 5 transactions
+    txCount = 5
+  )
+
+  # Send a transaction on each payload of the canonical chain
+  sendTransaction = func(i int) (Transaction, error) (
+    data = common.LeftPadBytes([]byte(byte(i)), 32)
+    t.Logf("transactionReorg, i=%v, data=%v\n", i, data)
+    return t.sendNextTx(
+      t.TestContext,
+      t.Engine,
+      BaseTx(
+        recipient:  &sstoreContractAddr,
+        amount:     big0,
+        payload:    data,
+        txType:     cs.txType,
+        gasLimit:   75000,
+        ForkConfig: t.ForkConfig,
+      ),
+    )
+
+  )
+
+  var (
+    altPayload ExecutableData
+    nextTx     Transaction
+    tx         Transaction
+  )
+
+  for i = 0; i < txCount; i++ (
+
+    # Generate two payloads, one with the transaction and the other one without it
+    env.clMock.produceSingleBlock(BlockProcessCallbacks(
+      OnPayloadAttributesGenerated: proc(): bool =
+        # At this point we have not broadcast the transaction.
+        if spec.Scenario == TransactionReOrgScenarioReOrgOut (
+          # Any payload we get should not contain any
+          payloadAttributes = env.clMock.LatestPayloadAttributes
+          rand.Read(payloadAttributes.Random[:])
+          r = env.engine.client.forkchoiceUpdated(env.clMock.latestForkchoice, payloadAttributes, env.clMock.latestHeader.Time)
+          r.expectNoError()
+          if r.Response.PayloadID == nil (
+            fatal "No payload ID returned by forkchoiceUpdated", t.TestName)
+          )
+          g = env.engine.client.getPayload(r.Response.PayloadID, payloadAttributes)
+          g.expectNoError()
+          altPayload = &g.Payload
+
+          if len(altPayload.Transactions) != 0 (
+            fatal "Empty payload contains transactions: %v", t.TestName, altPayload)
+          )
+        )
+
+        if spec.Scenario != TransactionReOrgScenarioReOrgBackIn (
+          # At this point we can broadcast the transaction and it will be included in the next payload
+          # Data is the key where a `1` will be stored
+          tx, err = sendTransaction(i)
+          if err != nil (
+            fatal "Error trying to send transaction: %v", t.TestName, err)
+          )
+
+          # Get the receipt
+          ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+          defer cancel()
+          receipt, _ = t.Eth.TransactionReceipt(ctx, tx.Hash())
+          if receipt != nil (
+            fatal "Receipt obtained before tx included in block: %v", t.TestName, receipt)
+          )
+        )
+
+      ),
+      onGetpayload: proc(): bool =
+        # Check that indeed the payload contains the transaction
+        if tx != nil (
+          if !TransactionInPayload(env.clMock.latestPayloadBuilt, tx) (
+            fatal "Payload built does not contain the transaction: %v", t.TestName, env.clMock.latestPayloadBuilt)
+          )
+        )
+
+        if spec.Scenario == TransactionReOrgScenarioReOrgDifferentBlock || spec.Scenario == TransactionReOrgScenarioNewPayloadOnRevert (
+          # Create side payload with different hash
+          var err error
+          customizer = &CustomPayloadData(
+            extraData: &([]byte(0x01)),
+          )
+          altPayload, err = customizer.customizePayload(env.clMock.latestPayloadBuilt)
+          if err != nil (
+            fatal "Error creating reorg payload %v", err)
+          )
+
+          if altPayload.parentHash != env.clMock.latestPayloadBuilt.parentHash (
+            fatal "Incorrect parent hash for payloads: %v != %v", t.TestName, altPayload.parentHash, env.clMock.latestPayloadBuilt.parentHash)
+          )
+          if altPayload.blockHash == env.clMock.latestPayloadBuilt.blockHash (
+            fatal "Incorrect hash for payloads: %v == %v", t.TestName, altPayload.blockHash, env.clMock.latestPayloadBuilt.blockHash)
+          )
+        elif spec.Scenario == TransactionReOrgScenarioReOrgBackIn (
+          # At this point we broadcast the transaction and request a new payload from the client that must
+          # contain the transaction.
+          # Since we are re-orging out and back in on the next block, the verification of this transaction
+          # being included happens on the next block
+          nextTx, err = sendTransaction(i)
+          if err != nil (
+            fatal "Error trying to send transaction: %v", t.TestName, err)
+          )
+
+          if i == 0 (
+            # We actually can only do this once because the transaction carries over and we cannot
+            # impede it from being included in the next payload
+            forkchoiceUpdated = env.clMock.latestForkchoice
+            payloadAttributes = env.clMock.LatestPayloadAttributes
+            rand.Read(payloadAttributes.SuggestedFeeRecipient[:])
+            f = env.engine.client.forkchoiceUpdated(
+              &forkchoiceUpdated,
+              &payloadAttributes,
+              env.clMock.latestHeader.Time,
+            )
+            f.expectPayloadStatus(PayloadExecutionStatus.valid)
+
+            # Wait a second for the client to prepare the payload with the included transaction
+
+            await sleepAsync(env.clMock.PayloadProductionClientDelay)
+
+            g = env.engine.client.getPayload(f.Response.PayloadID, env.clMock.LatestPayloadAttributes)
+            g.expectNoError()
+
+            if !TransactionInPayload(g.Payload, nextTx) (
+              fatal "Payload built does not contain the transaction: %v", t.TestName, g.Payload)
+            )
+
+            # Send the new payload and forkchoiceUpdated to it
+            n = env.engine.client.newPayload(g.Payload)
+            n.expectStatus(PayloadExecutionStatus.valid)
+
+            forkchoiceUpdated.headBlockHash = g.Payload.blockHash
+
+            s = env.engine.client.forkchoiceUpdated(forkchoiceUpdated, nil, g.Payload.timestamp)
+            s.expectPayloadStatus(PayloadExecutionStatus.valid)
+          )
+        )
+      ),
+      onNewPayloadBroadcast: proc(): bool =
+        if tx != nil (
+          # Get the receipt
+          ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+          defer cancel()
+          receipt, _ = t.Eth.TransactionReceipt(ctx, tx.Hash())
+          if receipt != nil (
+            fatal "Receipt obtained before tx included in block (NewPayload): %v", t.TestName, receipt)
+          )
+        )
+      ),
+      onForkchoiceBroadcast: proc(): bool =
+        if spec.Scenario != TransactionReOrgScenarioReOrgBackIn (
+          # Transaction is now in the head of the canonical chain, re-org and verify it's removed
+          # Get the receipt
+          txt = env.engine.client.txReceipt(tx.Hash())
+          txt.expectBlockHash(env.clMock.latestForkchoice.headBlockHash)
+
+          if altPayload.parentHash != env.clMock.latestPayloadBuilt.parentHash (
+            fatal "Incorrect parent hash for payloads: %v != %v", t.TestName, altPayload.parentHash, env.clMock.latestPayloadBuilt.parentHash)
+          )
+          if altPayload.blockHash == env.clMock.latestPayloadBuilt.blockHash (
+            fatal "Incorrect hash for payloads: %v == %v", t.TestName, altPayload.blockHash, env.clMock.latestPayloadBuilt.blockHash)
+          )
+
+          if altPayload == nil (
+            fatal "No payload to re-org to", t.TestName)
+          )
+          r = env.engine.client.newPayload(altPayload)
+          r.expectStatus(PayloadExecutionStatus.valid)
+          r.expectLatestValidHash(altPayload.blockHash)
+
+          s = env.engine.client.forkchoiceUpdated(api.ForkchoiceStateV1(
+            headBlockHash:      altPayload.blockHash,
+            safeBlockHash:      env.clMock.latestForkchoice.safeBlockHash,
+            finalizedBlockHash: env.clMock.latestForkchoice.finalizedBlockHash,
+          ), nil, altPayload.timestamp)
+          s.expectPayloadStatus(PayloadExecutionStatus.valid)
+
+          p = env.engine.client.headerByNumber(Head)
+          p.expectHash(altPayload.blockHash)
+
+          txt = env.engine.client.txReceipt(tx.Hash())
+          if spec.Scenario == TransactionReOrgScenarioReOrgOut (
+            if txt.Receipt != nil (
+              receiptJson, _ = json.MarshalIndent(txt.Receipt, "", "  ")
+              fatal "Receipt was obtained when the tx had been re-org'd out: %s", t.TestName, receiptJson)
+            )
+          elif spec.Scenario == TransactionReOrgScenarioReOrgDifferentBlock || spec.Scenario == TransactionReOrgScenarioNewPayloadOnRevert (
+            txt.expectBlockHash(altPayload.blockHash)
+          )
+
+          # Re-org back
+          if spec.Scenario == TransactionReOrgScenarioNewPayloadOnRevert (
+            r = env.engine.client.newPayload(env.clMock.latestPayloadBuilt)
+            r.expectStatus(PayloadExecutionStatus.valid)
+            r.expectLatestValidHash(env.clMock.latestPayloadBuilt.blockHash)
+          )
+          env.clMock.BroadcastForkchoiceUpdated(env.clMock.latestForkchoice, nil, 1)
+        )
+
+        if tx != nil (
+          # Now it should be back with main payload
+          txt = env.engine.client.txReceipt(tx.Hash())
+          txt.expectBlockHash(env.clMock.latestForkchoice.headBlockHash)
+
+          if spec.Scenario != TransactionReOrgScenarioReOrgBackIn (
+            tx = nil
+          )
+        )
+
+        if spec.Scenario == TransactionReOrgScenarioReOrgBackIn && i > 0 (
+          # Reasoning: Most of the clients do not re-add blob transactions to the pool
+          # after a re-org, so we need to wait until the next tx is sent to actually
+          # verify.
+          tx = nextTx
+        )
+
+      ),
+    ))
+
+  )
+
+  if tx != nil (
+    # Produce one last block and verify that the block contains the transaction
+    env.clMock.produceSingleBlock(BlockProcessCallbacks(
+      onForkchoiceBroadcast: proc(): bool =
+        if !TransactionInPayload(env.clMock.latestPayloadBuilt, tx) (
+          fatal "Payload built does not contain the transaction: %v", t.TestName, env.clMock.latestPayloadBuilt)
+        )
+        # Get the receipt
+        ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+        defer cancel()
+        receipt, _ = t.Eth.TransactionReceipt(ctx, tx.Hash())
+        if receipt == nil (
+          fatal "Receipt not obtained after tx included in block: %v", t.TestName, receipt)
+        )
+      ),
+    ))
+
+  )
+
+)
+
+# Test that performing a re-org back into a previous block of the canonical chain does not produce errors and the chain
+# is still capable of progressing.
+type
+  ReOrgBackToCanonicalTest* = ref object of EngineSpec
+    # Depth of the re-org to back in the canonical chain
+    ReOrgDepth uint64
+    # Number of transactions to send on each payload
+    TransactionPerPayload uint64
+    # Whether to execute a sidechain payload on the re-org
+    ExecuteSidePayloadOnReOrg bool
+
+method withMainFork(cs: ReOrgBackToCanonicalTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ReOrgBackToCanonicalTest): string =
+  name = fmt.Sprintf("Re-Org Back into Canonical Chain, Depth=%d", s.ReOrgDepth)
+
+  if s.ExecuteSidePayloadOnReOrg (
+    name += ", Execute Side Payload on Re-Org"
+  )
+  return name
+
+proc getDepth(cs: ReOrgBackToCanonicalTest): int =
+  if s.ReOrgDepth == 0 (
+    return 3
+  )
+  return s.ReOrgDepth
+)
+
+# Test that performing a re-org back into a previous block of the canonical chain does not produce errors and the chain
+# is still capable of progressing.
+method execute(cs: ReOrgBackToCanonicalTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Check the CLMock configured safe and finalized
+  if env.clMock.slotsToSafe.Cmp(new(big.Int).SetUint64(spec.ReOrgDepth)) <= 0 (
+    fatal "[TEST ISSUE] CLMock configured slots to safe less than re-org depth: %v <= %v", t.TestName, env.clMock.slotsToSafe, spec.ReOrgDepth)
+  )
+  if env.clMock.slotsToFinalized.Cmp(new(big.Int).SetUint64(spec.ReOrgDepth)) <= 0 (
+    fatal "[TEST ISSUE] CLMock configured slots to finalized less than re-org depth: %v <= %v", t.TestName, env.clMock.slotsToFinalized, spec.ReOrgDepth)
+  )
+
+  # Produce blocks before starting the test (So we don't try to reorg back to the genesis block)
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  # We are going to reorg back to a previous hash several times
+  previousHash = env.clMock.latestForkchoice.headBlockHash
+  previousTimestamp = env.clMock.latestPayloadBuilt.timestamp
+
+  if spec.ExecuteSidePayloadOnReOrg (
+    var (
+      sidePayload                 ExecutableData
+      sidePayloadParentForkchoice api.ForkchoiceStateV1
+      sidePayloadParentTimestamp  uint64
+    )
+    env.clMock.produceSingleBlock(BlockProcessCallbacks(
+      OnPayloadAttributesGenerated: proc(): bool =
+        payloadAttributes = env.clMock.LatestPayloadAttributes
+        rand.Read(payloadAttributes.Random[:])
+        r = env.engine.client.forkchoiceUpdated(env.clMock.latestForkchoice, payloadAttributes, env.clMock.latestHeader.Time)
+        r.expectNoError()
+        if r.Response.PayloadID == nil (
+          fatal "No payload ID returned by forkchoiceUpdated", t.TestName)
+        )
+        g = env.engine.client.getPayload(r.Response.PayloadID, payloadAttributes)
+        g.expectNoError()
+        sidePayload = &g.Payload
+        sidePayloadParentForkchoice = env.clMock.latestForkchoice
+        sidePayloadParentTimestamp = env.clMock.latestHeader.Time
+      ),
+    ))
+    # Continue producing blocks until we reach the depth of the re-org
+    testCond env.clMock.produceBlocks(int(spec.GetDepth()-1), BlockProcessCallbacks(
+      onPayloadProducerSelected: proc(): bool =
+        # Send a transaction on each payload of the canonical chain
+        var err error
+        _, err = t.sendNextTxs(
+          t.TestContext,
+          t.Engine,
+          BaseTx(
+            recipient:  &ZeroAddr,
+            amount:     big1,
+            payload:    nil,
+            txType:     cs.txType,
+            gasLimit:   75000,
+            ForkConfig: t.ForkConfig,
+          ),
+          spec.TransactionPerPayload,
+        )
+        if err != nil (
+          fatal "Error trying to send transactions: %v", t.TestName, err)
+        )
+      ),
+    ))
+    # On the last block, before executing the next payload of the canonical chain,
+    # re-org back to the parent of the side payload and execute the side payload first
+    env.clMock.produceSingleBlock(BlockProcessCallbacks(
+      onGetpayload: proc(): bool =
+        # We are about to execute the new payload of the canonical chain, re-org back to
+        # the side payload
+        f = env.engine.client.forkchoiceUpdated(sidePayloadParentForkchoice, nil, sidePayloadParentTimestamp)
+        f.expectPayloadStatus(PayloadExecutionStatus.valid)
+        f.expectLatestValidHash(sidePayloadParentForkchoice.headBlockHash)
+        # Execute the side payload
+        n = env.engine.client.newPayload(sidePayload)
+        n.expectStatus(PayloadExecutionStatus.valid)
+        n.expectLatestValidHash(sidePayload.blockHash)
+        # At this point the next canonical payload will be executed by the CL mock, so we can
+        # continue producing blocks
+      ),
+    ))
+  else:
+    testCond env.clMock.produceBlocks(int(spec.GetDepth()), BlockProcessCallbacks(
+      onForkchoiceBroadcast: proc(): bool =
+        # Send a fcU with the headBlockHash pointing back to the previous block
+        forkchoiceUpdatedBack = api.ForkchoiceStateV1(
+          headBlockHash:      previousHash,
+          safeBlockHash:      env.clMock.latestForkchoice.safeBlockHash,
+          finalizedBlockHash: env.clMock.latestForkchoice.finalizedBlockHash,
+        )
+
+        # It is only expected that the client does not produce an error and the CL Mocker is able to progress after the re-org
+        r = env.engine.client.forkchoiceUpdated(forkchoiceUpdatedBack, nil, previousTimestamp)
+        r.expectNoError()
+
+        # Re-send the ForkchoiceUpdated that the CLMock had sent
+        r = env.engine.client.forkchoiceUpdated(env.clMock.latestForkchoice, nil, env.clMock.LatestExecutedPayload.timestamp)
+        r.expectNoError()
+      ),
+    ))
+  )
+
+  # Verify that the client is pointing to the latest payload sent
+  r = env.engine.client.headerByNumber(Head)
+  r.expectHash(env.clMock.latestPayloadBuilt.blockHash)
+
+)
+
+type
+  ReOrgBackFromSyncingTest* = ref object of EngineSpec
+
+method withMainFork(cs: ReOrgBackFromSyncingTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ReOrgBackFromSyncingTest): string =
+  name = "Re-Org Back to Canonical Chain From Syncing Chain"
+  return name
+)
+
+# Test that performs a re-org back to the canonical chain after re-org to syncing/unavailable chain.
+method execute(cs: ReOrgBackFromSyncingTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce an alternative chain
+  sidechainPayloads = make([]ExecutableData, 0)
+  testCond env.clMock.produceBlocks(10, BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      # Send a transaction on each payload of the canonical chain
+      var err error
+      _, err = t.sendNextTx(
+        t.TestContext,
+        t.Engine,
+        BaseTx(
+          recipient:  &ZeroAddr,
+          amount:     big1,
+          payload:    nil,
+          txType:     cs.txType,
+          gasLimit:   75000,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transactions: %v", t.TestName, err)
+      )
+    ),
+    onGetpayload: proc(): bool =
+      # Check that at least one transaction made it into the payload
+      if len(env.clMock.latestPayloadBuilt.Transactions) == 0 (
+        fatal "No transactions in payload: %v", t.TestName, env.clMock.latestPayloadBuilt)
+      )
+      # Generate an alternative payload by simply adding extraData to the block
+      altParentHash = env.clMock.latestPayloadBuilt.parentHash
+      if len(sidechainPayloads) > 0 (
+        altParentHash = sidechainPayloads[len(sidechainPayloads)-1].blockHash
+      )
+      customizer = &CustomPayloadData(
+        parentHash: &altParentHash,
+        extraData:  &([]byte(0x01)),
+      )
+      altPayload, err = customizer.customizePayload(env.clMock.latestPayloadBuilt)
+      if err != nil (
+        fatal "Unable to customize payload: %v", t.TestName, err)
+      )
+      sidechainPayloads = append(sidechainPayloads, altPayload)
+    ),
+  ))
+
+  env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onGetpayload: proc(): bool =
+      # Re-org to the unavailable sidechain in the middle of block production
+      # to be able to re-org back to the canonical chain
+      r = env.engine.client.newPayload(sidechainPayloads[len(sidechainPayloads)-1])
+      r.expectStatusEither(PayloadExecutionStatus.syncing, test.Accepted)
+      r.expectLatestValidHash(nil)
+      # We are going to send one of the alternative payloads and fcU to it
+      forkchoiceUpdatedBack = api.ForkchoiceStateV1(
+        headBlockHash:      sidechainPayloads[len(sidechainPayloads)-1].blockHash,
+        safeBlockHash:      env.clMock.latestForkchoice.safeBlockHash,
+        finalizedBlockHash: env.clMock.latestForkchoice.finalizedBlockHash,
+      )
+
+      # It is only expected that the client does not produce an error and the CL Mocker is able to progress after the re-org
+      s = env.engine.client.forkchoiceUpdated(forkchoiceUpdatedBack, nil, sidechainPayloads[len(sidechainPayloads)-1].timestamp)
+      s.expectLatestValidHash(nil)
+      s.expectPayloadStatus(PayloadExecutionStatus.syncing)
+
+      # After this, the CLMocker will continue and try to re-org to canonical chain once again
+      # CLMocker will fail the test if this is not possible, so nothing left to do.
+    ),
+  ))
+)
+
+type
+  ReOrgPrevValidatedPayloadOnSideChainTest* = ref object of EngineSpec
+
+method withMainFork(cs: ReOrgPrevValidatedPayloadOnSideChainTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ReOrgPrevValidatedPayloadOnSideChainTest): string =
+  name = "Re-org to Previously Validated Sidechain Payload"
+  return name
+)
+
+# Test that performs a re-org to a previously validated payload on a side chain.
+method execute(cs: ReOrgPrevValidatedPayloadOnSideChainTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce blocks before starting the test
+  testCond env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  var (
+    sidechainPayloads     = make([]ExecutableData, 0)
+    sidechainPayloadCount = 5
+  )
+
+  # Produce a canonical chain while at the same time generate a side chain to which we will re-org.
+  testCond env.clMock.produceBlocks(sidechainPayloadCount, BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      # Send a transaction on each payload of the canonical chain
+      var err error
+      _, err = t.sendNextTx(
+        t.TestContext,
+        t.Engine,
+        BaseTx(
+          recipient:  &ZeroAddr,
+          amount:     big1,
+          payload:    nil,
+          txType:     cs.txType,
+          gasLimit:   75000,
+          ForkConfig: t.ForkConfig,
+        ),
+      )
+      if err != nil (
+        fatal "Error trying to send transactions: %v", t.TestName, err)
+      )
+    ),
+    onGetpayload: proc(): bool =
+      # Check that at least one transaction made it into the payload
+      if len(env.clMock.latestPayloadBuilt.Transactions) == 0 (
+        fatal "No transactions in payload: %v", t.TestName, env.clMock.latestPayloadBuilt)
+      )
+      # The side chain will consist simply of the same payloads with extra data appended
+      extraData = []byte("side")
+      customData = CustomPayloadData(
+        extraData: &extraData,
+      )
+      if len(sidechainPayloads) > 0 (
+        customData.parentHash = &sidechainPayloads[len(sidechainPayloads)-1].blockHash
+      )
+      altPayload, err = customData.customizePayload(env.clMock.latestPayloadBuilt)
+      if err != nil (
+        fatal "Unable to customize payload: %v", t.TestName, err)
+      )
+      sidechainPayloads = append(sidechainPayloads, altPayload)
+
+      r = env.engine.client.newPayload(altPayload)
+      r.expectStatus(PayloadExecutionStatus.valid)
+      r.expectLatestValidHash(altPayload.blockHash)
+    ),
+  ))
+
+  # Attempt to re-org to one of the sidechain payloads, but not the leaf,
+  # and also build a new payload from this sidechain.
+  env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onGetpayload: proc(): bool =
+      var (
+        prevRandao            = common.Hash()
+        suggestedFeeRecipient = common.Address(0x12, 0x34)
+      )
+      rand.Read(prevRandao[:])
+      payloadAttributesCustomizer = &BasePayloadAttributesCustomizer(
+        Random:                &prevRandao,
+        SuggestedFeerecipient: &suggestedFeeRecipient,
+      )
+
+      reOrgPayload = sidechainPayloads[len(sidechainPayloads)-2]
+      reOrgPayloadAttributes = sidechainPayloads[len(sidechainPayloads)-1].PayloadAttributes
+
+      newPayloadAttributes, err = payloadAttributesCustomizer.getPayloadAttributes(reOrgPayloadAttributes)
+      if err != nil (
+        fatal "Unable to customize payload attributes: %v", t.TestName, err)
+      )
+
+      r = env.engine.client.forkchoiceUpdated(api.ForkchoiceStateV1(
+        headBlockHash:      reOrgPayload.blockHash,
+        safeBlockHash:      env.clMock.latestForkchoice.safeBlockHash,
+        finalizedBlockHash: env.clMock.latestForkchoice.finalizedBlockHash,
+      ), newPayloadAttributes, reOrgPayload.timestamp)
+      r.expectPayloadStatus(PayloadExecutionStatus.valid)
+      r.expectLatestValidHash(reOrgPayload.blockHash)
+
+      p = env.engine.client.getPayload(r.Response.PayloadID, newPayloadAttributes)
+      p.expectPayloadParentHash(reOrgPayload.blockHash)
+
+      s = env.engine.client.newPayload(p.Payload)
+      s.expectStatus(PayloadExecutionStatus.valid)
+      s.expectLatestValidHash(p.Payload.blockHash)
+
+      # After this, the CLMocker will continue and try to re-org to canonical chain once again
+      # CLMocker will fail the test if this is not possible, so nothing left to do.
+    ),
+  ))
+)
+
+type
+  SafeReOrgToSideChainTest* = ref object of EngineSpec
+
+method withMainFork(cs: SafeReOrgToSideChainTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: SafeReOrgToSideChainTest): string =
+  name = "Safe Re-Org to Side Chain"
+  return name
+)
+
+# Test that performs a re-org of the safe block to a side chain.
+method execute(cs: SafeReOrgToSideChainTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Produce an alternative chain
+  sidechainPayloads = make([]ExecutableData, 0)
+
+  if s.slotsToSafe.Uint64() != 1 (
+    fatal "[TEST ISSUE] CLMock configured slots to safe not equal to 1: %v", t.TestName, s.slotsToSafe)
+  )
+  if s.slotsToFinalized.Uint64() != 2 (
+    fatal "[TEST ISSUE] CLMock configured slots to finalized not equal to 2: %v", t.TestName, s.slotsToFinalized)
+  )
+
+  # Produce three payloads `P1`, `P2`, `P3`, along with the side chain payloads `P2'`, `P3'`
+  # First payload is finalized so no alternative payload
+  env.clMock.produceSingleBlock(BlockProcessCallbacks())
+  testCond env.clMock.produceBlocks(2, BlockProcessCallbacks(
+    onGetpayload: proc(): bool =
+      # Generate an alternative payload by simply adding extraData to the block
+      altParentHash = env.clMock.latestPayloadBuilt.parentHash
+      if len(sidechainPayloads) > 0 (
+        altParentHash = sidechainPayloads[len(sidechainPayloads)-1].blockHash
+      )
+      customizer = &CustomPayloadData(
+        parentHash: &altParentHash,
+        extraData:  &([]byte(0x01)),
+      )
+      altPayload, err = customizer.customizePayload(env.clMock.latestPayloadBuilt)
+      if err != nil (
+        fatal "Unable to customize payload: %v", t.TestName, err)
+      )
+      sidechainPayloads = append(sidechainPayloads, altPayload)
+    ),
+  ))
+
+  # Verify current state of labels
+  head = env.engine.client.headerByNumber(Head)
+  head.expectHash(env.clMock.latestPayloadBuilt.blockHash)
+
+  safe = env.engine.client.headerByNumber(Safe)
+  safe.expectHash(env.clMock.executedPayloadHistory[2].blockHash)
+
+  finalized = env.engine.client.headerByNumber(Finalized)
+  finalized.expectHash(env.clMock.executedPayloadHistory[1].blockHash)
+
+  # Re-org the safe/head blocks to point to the alternative side chain
+  env.clMock.produceSingleBlock(BlockProcessCallbacks(
+    onGetpayload: proc(): bool =
+      for _, p = range sidechainPayloads (
+        r = env.engine.client.newPayload(p)
+        r.expectStatusEither(PayloadExecutionStatus.valid, test.Accepted)
+      )
+      r = env.engine.client.forkchoiceUpdated(api.ForkchoiceStateV1(
+        headBlockHash:      sidechainPayloads[1].blockHash,
+        safeBlockHash:      sidechainPayloads[0].blockHash,
+        finalizedBlockHash: env.clMock.executedPayloadHistory[1].blockHash,
+      ), nil, sidechainPayloads[1].timestamp)
+      r.expectPayloadStatus(PayloadExecutionStatus.valid)
+
+      head = env.engine.client.headerByNumber(Head)
+      head.expectHash(sidechainPayloads[1].blockHash)
+
+      safe = env.engine.client.headerByNumber(Safe)
+      safe.expectHash(sidechainPayloads[0].blockHash)
+
+      finalized = env.engine.client.headerByNumber(Finalized)
+      finalized.expectHash(env.clMock.executedPayloadHistory[1].blockHash)
+
+    ),
+  ))
+)

hive_integration/nodocker/engine/engine/rpc.nim

@@ -0,0 +1,104 @@
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  BlockStatusRPCcheckType = enum
+    LatestOnNewPayload            = "Latest Block on NewPayload"
+    LatestOnHeadblockHash         = "Latest Block on HeadblockHash Update"
+    SafeOnSafeblockHash           = "Safe Block on SafeblockHash Update"
+    FinalizedOnFinalizedblockHash = "Finalized Block on FinalizedblockHash Update"
+
+type
+  BlockStatus* = ref object of EngineSpec
+    checkType: BlockStatusRPCcheckType
+    # TODO: Syncing   bool
+
+method withMainFork(cs: BlockStatus, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: BlockStatus): string =
+  "RPC" & $b.checkType
+
+# Test to verify Block information available at the Eth RPC after NewPayload/ForkchoiceUpdated
+method execute(cs: BlockStatus, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS Block
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  case b.checkType
+  of SafeOnSafeblockHash, FinalizedOnFinalizedblockHash:
+    var number *big.Int
+    if b.checkType == SafeOnSafeblockHash:
+      number = Safe
+    else:
+      number = Finalized
+
+    p = env.engine.client.TestHeaderByNumber(number)
+    p.expectError()
+  )
+
+  # Produce blocks before starting the test
+  env.clMock.produceBlocks(5, BlockProcessCallbacks())
+
+  var tx typ.Transaction
+  callbacks = BlockProcessCallbacks(
+    onPayloadProducerSelected: proc(): bool =
+      let tc = BaseTx(
+          recipient:  &ZeroAddr,
+          amount:     1.u256,
+          payload:    nil,
+          txType:     cs.txType,
+          gasLimit:   75000,
+          ForkConfig: t.ForkConfig,
+        ),
+
+      tx, err = env.sendNextTx(
+      )
+      if err != nil (
+        fatal "Error trying to send transaction: %v", err)
+      )
+    ),
+  )
+
+  switch b.checkType (
+  case LatestOnNewPayload:
+    callbacks.onGetPayload = proc(): bool
+      r = env.engine.client.latestHeader()
+      r.expectHash(env.clMock.latestForkchoice.headblockHash)
+
+      s = env.engine.client.TestBlockNumber()
+      s.ExpectNumber(env.clMock.latestHeadNumber.Uint64())
+
+      p = env.engine.client.latestHeader()
+      p.expectHash(env.clMock.latestForkchoice.headblockHash)
+
+      # Check that the receipt for the transaction we just sent is still not available
+      q = env.engine.client.txReceipt(tx.Hash())
+      q.expectError()
+    )
+  case LatestOnHeadblockHash:
+    callbacks.onForkchoiceBroadcast = proc(): bool
+      r = env.engine.client.latestHeader()
+      r.expectHash(env.clMock.latestForkchoice.headblockHash)
+
+      s = env.engine.client.txReceipt(tx.Hash())
+      s.ExpectTransactionHash(tx.Hash())
+    )
+  case SafeOnSafeblockHash:
+    callbacks.onSafeBlockChange = proc(): bool
+      r = env.engine.client.TestHeaderByNumber(Safe)
+      r.expectHash(env.clMock.latestForkchoice.safeblockHash)
+    )
+  case FinalizedOnFinalizedblockHash:
+    callbacks.onFinalizedBlockChange = proc(): bool
+      r = env.engine.client.TestHeaderByNumber(Finalized)
+      r.expectHash(env.clMock.latestForkchoice.finalizedblockHash)
+    )
+  )
+
+  # Perform the test
+  env.clMock.produceSingleBlock(callbacks)
+)

hive_integration/nodocker/engine/engine/suggested_fee_recipient.nim

@@ -0,0 +1,81 @@
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  SuggestedFeeRecipientTest* = ref object of EngineSpec
+    transactionCount: int
+
+method withMainFork(cs: SuggestedFeeRecipientTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: SuggestedFeeRecipientTest): string =
+  "Suggested Fee Recipient Test " & $cs.txType
+
+method execute(cs: SuggestedFeeRecipientTest, env: TestEnv): bool =
+  # Wait until this client catches up with latest PoS
+  let ok = waitFor env.clMock.waitForTTD()
+  testCond ok
+
+  # Create a single block to not having to build on top of genesis
+  testCond env.clMock.produceSingleBlock(BlockProcessCallbacks())
+
+  # Verify that, in a block with transactions, fees are accrued by the suggestedFeeRecipient
+  let
+    feeRecipient = EthAddress.randomBytes()
+    txRecipient = EthAddress.randomBytes()
+
+  # Send multiple transactions
+  for i = 0; i < cs.transactionCount; i++ (
+    _, err = env.sendNextTx(
+      t.TestContext,
+      t.Engine,
+      &BaseTx(
+        recipient:  &txRecipient,
+        amount:     big0,
+        payload:    nil,
+        txType:     cs.txType,
+        gasLimit:   75000,
+      ),
+    )
+    if err != nil (
+      fatal "Error trying to send transaction: %v", t.TestName, err)
+    )
+  )
+  # Produce the next block with the fee recipient set
+  env.clMock.nextFeeRecipient = feeRecipient
+  env.clMock.produceSingleBlock(BlockProcessCallbacks())
+
+  # Calculate the fees and check that they match the balance of the fee recipient
+  r = env.engine.client.TestBlockByNumber(Head)
+  r.ExpecttransactionCountEqual(cs.transactionCount)
+  r.ExpectCoinbase(feeRecipient)
+  blockIncluded = r.Block
+
+  feeRecipientFees = big.NewInt(0)
+  for _, tx = range blockIncluded.Transactions() (
+    effGasTip, err = tx.EffectiveGasTip(blockIncluded.BaseFee())
+    if err != nil (
+      fatal "unable to obtain EffectiveGasTip: %v", t.TestName, err)
+    )
+    ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+    defer cancel()
+    receipt, err = t.Eth.TransactionReceipt(ctx, tx.Hash())
+    if err != nil (
+      fatal "unable to obtain receipt: %v", t.TestName, err)
+    )
+    feeRecipientFees = feeRecipientFees.Add(feeRecipientFees, effGasTip.Mul(effGasTip, big.NewInt(int64(receipt.GasUsed))))
+  )
+
+  s = env.engine.client.TestBalanceAt(feeRecipient, nil)
+  s.expectBalanceEqual(feeRecipientFees)
+
+  # Produce another block without txns and get the balance again
+  env.clMock.nextFeeRecipient = feeRecipient
+  env.clMock.produceSingleBlock(BlockProcessCallbacks())
+
+  s = env.engine.client.TestBalanceAt(feeRecipient, nil)
+  s.expectBalanceEqual(feeRecipientFees)
+)

hive_integration/nodocker/engine/engine/versioning.nim

@@ -0,0 +1,65 @@
+# Test versioning of the Engine API methods
+import
+  std/strutils,
+  ./engine_spec
+
+type
+  EngineNewPayloadVersionTest* = ref object of EngineSpec
+
+method withMainFork(cs: EngineNewPayloadVersionTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+# Test modifying the ForkchoiceUpdated version on Payload Request to the previous/upcoming version
+# when the timestamp payload attribute does not match the upgraded/downgraded version.
+type
+  ForkchoiceUpdatedOnPayloadRequestTest* = ref object of EngineSpec
+    ForkchoiceUpdatedCustomizer
+
+method withMainFork(cs: ForkchoiceUpdatedOnPayloadRequestTest, fork: EngineFork): BaseSpec =
+  var res = cs.clone()
+  res.mainFork = fork
+  return res
+
+method getName(cs: ForkchoiceUpdatedOnPayloadRequestTest): string =
+  return "ForkchoiceUpdated Version on Payload Request: " + cs.BaseSpec.GetName()
+
+method execute(cs: ForkchoiceUpdatedOnPayloadRequestTest, env: TestEnv): bool =
+  # Wait until TTD is reached by this client
+  let ok = waitFor env.clMockWaitForTTD()
+  testCond ok
+
+  env.clMock.produceSingleBlock(clmock.BlockProcessCallbacks(
+    onPayloadAttributesGenerated: proc(): bool =
+      var (
+        payloadAttributes                    = &env.clMockLatestPayloadAttributes
+        expectedStatus    test.PayloadStatus = PayloadExecutionStatus.valid
+        expectedError     *int
+        err               error
+      )
+      cs.SetEngineAPIVersionResolver(t.ForkConfig)
+      testEngine = t.TestEngine.WithEngineAPIVersionResolver(cs.ForkchoiceUpdatedCustomizer)
+      payloadAttributes, err = cs.GetPayloadAttributes(payloadAttributes)
+      if err != nil (
+        t.Fatalf("FAIL: Error getting custom payload attributes: %v", err)
+      )
+      expectedError, err = cs.GetExpectedError()
+      if err != nil (
+        t.Fatalf("FAIL: Error getting custom expected error: %v", err)
+      )
+      if cs.GetExpectInvalidStatus() (
+        expectedStatus = PayloadExecutionStatus.invalid
+      )
+
+      r = env.engine.client.forkchoiceUpdated(env.clMockLatestForkchoice, payloadAttributes, env.clMockLatestHeader.Time)
+      r.ExpectationDescription = cs.Expectation
+      if expectedError != nil (
+        r.expectErrorCode(*expectedError)
+      else:
+        r.expectNoError()
+        r.expectPayloadStatus(expectedStatus)
+      )
+    ),
+  ))
+)

hive_integration/nodocker/engine/engine_client.nim

@@ -234,11 +234,6 @@ proc maybeChainId(n: Option[HexQuantityStr]): Option[ChainId] =
     return none(ChainId)
   some(hexToInt(string n.get, int).ChainId)
 
-proc maybeInt64(n: Option[HexQuantityStr]): Option[int64] =
-  if n.isNone:
-    return none(int64)
-  some(hexToInt(string n.get, int64))
-
 proc maybeInt(n: Option[HexQuantityStr]): Option[int] =
   if n.isNone:
     return none(int)
@@ -398,14 +393,13 @@ proc blockNumber*(client: RpcClient): Result[uint64, string] =
     let res = waitFor client.eth_blockNumber()
     return ok(hexToInt(string res, uint64))
 
-proc headerByNumber*(client: RpcClient, number: uint64, output: var common.BlockHeader): Result[void, string] =
+proc headerByNumber*(client: RpcClient, number: uint64): Result[common.BlockHeader, string] =
   wrapTry:
     let qty = encodeQuantity(number)
     let res = waitFor client.eth_getBlockByNumber(string qty, false)
     if res.isNone:
       return err("failed to get blockHeader: " & $number)
-    output = toBlockHeader(res.get())
-    return ok()
+    return ok(res.get.toBlockHeader)
 
 proc blockByNumber*(client: RpcClient, number: uint64, output: var common.EthBlock): Result[void, string] =
   wrapTry:
@@ -419,22 +413,19 @@ proc blockByNumber*(client: RpcClient, number: uint64, output: var common.EthBlo
     output.withdrawals = toWithdrawals(blk.withdrawals)
     return ok()
 
-proc headerByHash*(client: RpcClient, hash: Hash256, output: var common.BlockHeader): Result[void, string] =
+proc headerByHash*(client: RpcClient, hash: Hash256): Result[common.BlockHeader, string] =
   wrapTry:
     let res = waitFor client.eth_getBlockByHash(hash, false)
     if res.isNone:
       return err("failed to get block: " & hash.data.toHex)
-    let blk = res.get()
-    output = toBlockHeader(blk)
-    return ok()
+    return ok(res.get.toBlockHeader)
 
-proc latestHeader*(client: RpcClient, output: var common.BlockHeader): Result[void, string] =
+proc latestHeader*(client: RpcClient): Result[common.BlockHeader, string] =
   wrapTry:
     let res = waitFor client.eth_getBlockByNumber("latest", false)
     if res.isNone:
       return err("failed to get latest blockHeader")
-    output = toBlockHeader(res.get())
-    return ok()
+    return ok(res.get.toBlockHeader)
 
 proc latestBlock*(client: RpcClient, output: var common.EthBlock): Result[void, string] =
   wrapTry:
@@ -447,13 +438,12 @@ proc latestBlock*(client: RpcClient, output: var common.EthBlock): Result[void,
     output.withdrawals = toWithdrawals(blk.withdrawals)
     return ok()
 
-proc namedHeader*(client: RpcClient, name: string, output: var common.BlockHeader): Result[void, string] =
+proc namedHeader*(client: RpcClient, name: string): Result[common.BlockHeader, string] =
   wrapTry:
     let res = waitFor client.eth_getBlockByNumber(name, false)
     if res.isNone:
       return err("failed to get named blockHeader")
-    output = toBlockHeader(res.get())
-    return ok()
+    return ok(res.get.toBlockHeader)
 
 proc sendTransaction*(client: RpcClient, tx: common.Transaction): Result[void, string] =
   wrapTry:

hive_integration/nodocker/engine/engine_env.nim

@@ -20,7 +20,9 @@ import
     beacon/beacon_engine,
     common
   ],
-  ../../../tests/test_helpers
+  ../../../tests/test_helpers,
+  ../../../nimbus/beacon/web3_eth_conv,
+  ../../../nimbus/beacon/execution_types
 
 export
   results
@@ -149,9 +151,9 @@ proc close*(env: EngineEnv) =
   waitFor env.sealer.stop()
   waitFor env.server.closeWait()
 
-proc setRealTTD*(env: EngineEnv, ttdValue: int64) =
+proc setRealTTD*(env: EngineEnv) =
   let genesis = env.com.genesisHeader
-  let realTTD = genesis.difficulty + ttdValue.u256
+  let realTTD = genesis.difficulty
   env.com.setTTD some(realTTD)
   env.ttd = realTTD
 
@@ -181,7 +183,7 @@ proc peer*(env: EngineEnv): Peer =
   for peer in env.node.peers:
     return peer
 
-proc getTxsInPool*(env: EngineEnv, txHashes: openArray[Hash256]): seq[Transaction] =
+proc getTxsInPool*(env: EngineEnv, txHashes: openArray[common.Hash256]): seq[Transaction] =
   result = newSeqOfCap[Transaction](txHashes.len)
   for txHash in txHashes:
     let res = env.txPool.getItem(txHash)
@@ -193,3 +195,16 @@ proc getTxsInPool*(env: EngineEnv, txHashes: openArray[Hash256]): seq[Transactio
 proc numTxsInPool*(env: EngineEnv): int =
   env.txPool.numTxs
 
+func version*(env: EngineEnv, time: EthTime): Version =
+  if env.com.isCancunOrLater(time):
+    Version.V3
+  elif env.com.isShanghaiOrlater(time):
+    Version.V2
+  else:
+    Version.V1
+
+func version*(env: EngineEnv, time: Web3Quantity): Version =
+  env.version(time.EthTime)
+
+func version*(env: EngineEnv, time: uint64): Version =
+  env.version(time.EthTime)

hive_integration/nodocker/engine/engine_tests.nim

@@ -1,368 +1,454 @@
 import
+  eth/common/eth_types,
   ./engine/engine_spec,
   ./types,
   ./test_env,
-  ./base_spec
+  ./base_spec,
+  ./cancun/customizer,
+  ../../nimbus/common/chain_config
+
+import
+  ./engine/misc,
+  ./engine/payload_attributes,
+  ./engine/invalid_ancestor,
+  ./engine/invalid_payload,
+  ./engine/bad_hash
+
+proc getGenesis(cs: EngineSpec, param: NetworkParams) =
+  # Set the terminal total difficulty
+  let realTTD = param.genesis.difficulty + cs.ttd.u256
+  param.config.terminalTotalDifficulty = some(realTTD)
+  if param.genesis.difficulty <= realTTD:
+    param.config.terminalTotalDifficultyPassed = some(true)
+
+  # Set the genesis timestamp if provided
+  if cs.genesisTimestamp != 0:
+    param.genesis.timestamp = cs.genesisTimestamp.EthTime
 
 proc specExecute(ws: BaseSpec): bool =
-  var
-    ws = EngineSpec(ws)
-    env = TestEnv.new(ws.chainFile, false)
+  let
+    cs = EngineSpec(ws)
+    forkConfig = ws.getForkConfig()
 
-  env.engine.setRealTTD(ws.ttd)
+  if forkConfig.isNil:
+    echo "because fork configuration is not possible, skip test: ", cs.getName()
+    return true
+
+  let conf = envConfig(forkConfig)
+  cs.getGenesis(conf.networkParams)
+  let env  = TestEnv.new(conf)
+  env.engine.setRealTTD()
   env.setupCLMock()
-  ws.configureCLMock(env.clMock)
-  result = ws.exec(env)
+  #cs.configureCLMock(env.clMock)
+  result = cs.execute(env)
   env.close()
 
-let engineTestList* = [
-  # Engine API Negative Test Cases
-  TestDesc(
-    name: "Invalid Terminal Block in ForkchoiceUpdated",
-    run: specExecute,
-    spec: EngineSpec(
-      exec: invalidTerminalBlockForkchoiceUpdated,
-      ttd: 1000000
-  ))#[,
-  TestDesc(
-    name: "Invalid GetPayload Under PoW",
-    run: specExecute,
-    spec: EngineSpec(
-      exec: invalidGetPayloadUnderPoW,
-      ttd: 1000000
-  )),
-  TestDesc(
-    name: "Invalid Terminal Block in NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidTerminalBlockNewPayload,
-      ttd:  1000000,
-  )),
-  TestDesc(
-    name: "Inconsistent Head in ForkchoiceState",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  inconsistentForkchoiceState1,
-  )),
-  TestDesc(
-    name: "Inconsistent Safe in ForkchoiceState",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  inconsistentForkchoiceState2,
-  )),
-  TestDesc(
-    name: "Inconsistent Finalized in ForkchoiceState",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  inconsistentForkchoiceState3,
-  )),
-  TestDesc(
-    name: "Unknown HeadBlockHash",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  unknownHeadBlockHash,
-  )),
-  TestDesc(
-    name: "Unknown SafeBlockHash",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  unknownSafeBlockHash,
-  )),
-  TestDesc(
-    name: "Unknown FinalizedBlockHash",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  unknownFinalizedBlockHash,
-  )),
-  TestDesc(
-    name: "ForkchoiceUpdated Invalid Payload Attributes",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayloadAttributes1,
-  )),
-  TestDesc(
-    name: "ForkchoiceUpdated Invalid Payload Attributes (Syncing)",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayloadAttributes2,
-  )),
-  TestDesc(
-    name: "Pre-TTD ForkchoiceUpdated After PoS Switch",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  preTTDFinalizedBlockHash,
-      ttd:  2,
-  )),
+# Execution specification reference:
+# https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md
+
+#[var (
+  big0      = new(big.Int)
+  big1      = u256(1)
+  Head      *big.Int # Nil
+  Pending   = u256(-2)
+  Finalized = u256(-3)
+  Safe      = u256(-4)
+)
+]#
+
+# Register all test combinations for Paris
+proc makeEngineTest*(): seq[EngineSpec] =
+  # Misc Tests
+  # Pre-merge & merge fork occur at block 1, post-merge forks occur at block 2
+  result.add NonZeroPreMergeFork(forkHeight: 2)
+
+  # Payload Attributes Tests
+  block:
+    let list = [
+      InvalidPayloadAttributesTest(
+        description: "Zero timestamp",
+        customizer: BasePayloadAttributesCustomizer(
+          timestamp: some(0'u64),
+        ),
+      ),
+      InvalidPayloadAttributesTest(
+        description: "Parent timestamp",
+        customizer: TimestampDeltaPayloadAttributesCustomizer(
+          timestampDelta: -1,
+        ),
+      ),
+    ]
+
+    for x in list:
+      result.add x
+      let y = x.clone()
+      y.syncing = true
+      result.add y
+
+  # Invalid Transaction ChainID Tests
+  result.add InvalidTxChainIDTest(
+    txType: some(TxLegacy),
+  )
+
+  result.add InvalidTxChainIDTest(
+    txType: some(TxEip1559),
+  )
+
+  # Invalid Ancestor Re-Org Tests (Reveal Via NewPayload)
+  for invalidIndex in [1, 9, 10]:
+    for emptyTxs in [false, true]:
+      result.add InvalidMissingAncestorReOrgTest(
+        slotsToSafe:       32,
+        slotsToFinalized:  64,
+        sidechainLength:   10,
+        invalidIndex:      invalidIndex,
+        invalidField:      InvalidStateRoot,
+        emptyTransactions: emptyTxs,
+      )
+
   # Invalid Payload Tests
-  TestDesc(
-    name: "Bad Hash on NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  badHashOnNewPayload1,
-  )),
-  TestDesc(
-    name: "Bad Hash on NewPayload Syncing",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  badHashOnNewPayload2,
-  )),
-  TestDesc(
-    name: "Bad Hash on NewPayload Side Chain",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  badHashOnNewPayload3,
-  )),
-  TestDesc(
-    name: "Bad Hash on NewPayload Side Chain Syncing",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  badHashOnNewPayload4,
-  )),
-  TestDesc(
-    name: "ParentHash==BlockHash on NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  parentHashOnExecPayload,
-  )),
-  TestDesc(
-    name: "Invalid Transition Payload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec: invalidTransitionPayload,
-      ttd: 393504,
-      chainFile: "blocks_2_td_393504.rlp",
-  )),
-  TestDesc(
-    name: "Invalid ParentHash NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload1,
-  )),
-  TestDesc(
-    name: "Invalid StateRoot NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload2,
-  )),
-  TestDesc(
-    name: "Invalid StateRoot NewPayload, Empty Transactions",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload3,
-  )),
-  TestDesc(
-    name: "Invalid ReceiptsRoot NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload4,
-  )),
-  TestDesc(
-    name: "Invalid Number NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload5,
-  )),
-  TestDesc(
-    name: "Invalid GasLimit NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload6,
-  )),
-  TestDesc(
-    name: "Invalid GasUsed NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload7,
-  )),
-  TestDesc(
-    name: "Invalid Timestamp NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload8,
-  )),
-  TestDesc(
-    name: "Invalid PrevRandao NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload9,
-  )),
-  TestDesc(
-    name: "Invalid Incomplete Transactions NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload10,
-  )),
-  TestDesc(
-    name: "Invalid Transaction Signature NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload11,
-  )),
-  TestDesc(
-    name: "Invalid Transaction Nonce NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload12,
-  )),
-  TestDesc(
-    name: "Invalid Transaction GasPrice NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload13,
-  )),
-  TestDesc(
-    name: "Invalid Transaction Gas NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload14,
-  )),
-  TestDesc(
-    name: "Invalid Transaction Value NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidPayload15,
-  )),
+  const
+    invalidPayloadBlockFields = [
+      InvalidParentHash,
+      InvalidStateRoot,
+      InvalidReceiptsRoot,
+      InvalidNumber,
+      InvalidGasLimit,
+      InvalidGasUsed,
+      InvalidTimestamp,
+      InvalidPrevRandao,
+      RemoveTransaction,
+    ]
 
-  # Invalid Ancestor Re-Org Tests (Reveal via newPayload)
-  TestDesc(
-    name: "Invalid Ancestor Chain Re-Org, Invalid StateRoot, Invalid P1', Reveal using newPayload",
-    slotsToFinalized: 20,
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidMissingAncestor1,
-  )),
-  TestDesc(
-    name: "Invalid Ancestor Chain Re-Org, Invalid StateRoot, Invalid P9', Reveal using newPayload",
-    slotsToFinalized: 20,
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidMissingAncestor2,
-  )),
-  TestDesc(
-    name: "Invalid Ancestor Chain Re-Org, Invalid StateRoot, Invalid P10', Reveal using newPayload",
-    slotsToFinalized: 20,
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  invalidMissingAncestor3,
-  )),
+  for invalidField in invalidPayloadBlockFields:
+    for syncing in [false, true]:
+     if invalidField == InvalidStateRoot:
+       result.add InvalidPayloadTestCase(
+          invalidField:      invalidField,
+          syncing:           syncing,
+          emptyTransactions: true,
+       )
 
-  # Eth RPC Status on ForkchoiceUpdated Events
-  TestDesc(
-    name: "Latest Block after NewPayload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  blockStatusExecPayload1,
-  )),
-  TestDesc(
-    name: "Latest Block after NewPayload (Transition Block)",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  blockStatusExecPayload2,
-      ttd:  5,
-  )),
-  TestDesc(
-    name: "Latest Block after New HeadBlock",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  blockStatusHeadBlock1,
-  )),
-  TestDesc(
-    name: "Latest Block after New HeadBlock (Transition Block)",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  blockStatusHeadBlock2,
-      ttd:  5,
-  )),
-  TestDesc(
-    name: "safe Block after New SafeBlockHash",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  blockStatusSafeBlock,
-      ttd:  5,
-  )),
-  TestDesc(
-    name: "finalized Block after New FinalizedBlockHash",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  blockStatusFinalizedBlock,
-      ttd:  5,
-  )),
-  TestDesc(
-    name: "Latest Block after Reorg",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  blockStatusReorg,
-  )),
+     result.add InvalidPayloadTestCase(
+        invalidField: invalidField,
+        syncing:      syncing,
+     )
 
-  # Payload Tests
-  TestDesc(
-    name: "Re-Execute Payload",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  reExecPayloads,
-  )),
-  TestDesc(
-    name: "Multiple New Payloads Extending Canonical Chain",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  multipleNewCanonicalPayloads,
-  )),
-  TestDesc(
-    name: "Out of Order Payload Execution",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  outOfOrderPayloads,
-  )),
+  # Register bad hash tests
+  for syncing in [false, true]:
+    for sidechain in [false, true]:
+      result.add BadHashOnNewPayload(
+        syncing:   syncing,
+        sidechain: sidechain,
+      )
 
-  # Transaction Reorg using Engine API
-  TestDesc(
-    name: "Transaction Reorg",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  transactionReorg,
-  )),
-  TestDesc(
-    name: "Sidechain Reorg",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  sidechainReorg,
-  )),
-  TestDesc(
-    name: "Re-Org Back into Canonical Chain",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  reorgBack,
-  )),
-  TestDesc(
-    name: "Re-Org Back to Canonical Chain From Syncing Chain",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  reorgBackFromSyncing,
-  )),
+  # Parent hash == block hash tests
+  result.add ParentHashOnNewPayload(syncing: false)
+  result.add ParentHashOnNewPayload(syncing: true)
 
-  # Suggested Fee Recipient in Payload creation
-  TestDesc(
-    name: "Suggested Fee Recipient Test",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  suggestedFeeRecipient,
-  )),
+  result.add PayloadBuildAfterInvalidPayloadTest(
+    invalidField: InvalidStateRoot,
+  )
+
+#[
+  const
+    invalidReorgList = [
+      InvalidStateRoot,
+      InvalidReceiptsRoot,
+      # TODO: InvalidNumber, Test is causing a panic on the secondary node, disabling for now.
+      InvalidGasLimit,
+      InvalidGasUsed,
+      InvalidTimestamp,
+      # TODO: InvalidPrevRandao, Test consistently fails with Failed to set invalid block: missing trie node.
+      RemoveTransaction,
+      InvalidTransactionSignature,
+      InvalidTransactionNonce,
+      InvalidTransactionGas,
+      InvalidTransactionGasPrice,
+      InvalidTransactionValue,
+      # InvalidOmmers, Unsupported now
+    ]
+
+    eightList = [
+      InvalidReceiptsRoot,
+      InvalidGasLimit,
+      InvalidGasUsed,
+      InvalidTimestamp,
+      InvalidPrevRandao
+    ]
+
+  # Invalid Ancestor Re-Org Tests (Reveal Via Sync)
+  for invalidField in invalidReorgList:
+    for reOrgFromCanonical in [false, true]:
+      var invalidIndex = 9
+      if invalidField in eightList:
+        invalidIndex = 8
+
+      if invalidField == InvalidStateRoot:
+        result.add InvalidMissingAncestorReOrgSyncTest(
+          timeoutSeconds:   60,
+          slotsToSafe:      32,
+          slotsToFinalized: 64,
+          invalidField:       invalidField,
+          reOrgFromCanonical: reOrgFromCanonical,
+          emptyTransactions:  true,
+          invalidIndex:       invalidIndex,
+        )
+
+      result.add InvalidMissingAncestorReOrgSyncTest(
+        timeoutSeconds:   60,
+        slotsToSafe:      32,
+        slotsToFinalized: 64,
+        invalidField:       invalidField,
+        reOrgFromCanonical: reOrgFromCanonical,
+        invalidIndex:       invalidIndex,
+      )
+]#
+#[
+  # Register RPC tests
+  for _, field := range []BlockStatusRPCCheckType(
+    LatestOnNewPayload,
+    LatestOnHeadBlockHash,
+    SafeOnSafeBlockHash,
+    FinalizedOnFinalizedBlockHash,
+  ) (
+    result.add BlockStatus(CheckType: field))
+  )
+
+  # Register ForkchoiceUpdate tests
+  for _, field := range []ForkchoiceStateField(
+    HeadBlockHash,
+    SafeBlockHash,
+    FinalizedBlockHash,
+  ) (
+    result.add
+      InconsistentForkchoiceTest(
+        Field: field,
+      ),
+      ForkchoiceUpdatedUnknownBlockHashTest(
+        Field: field,
+      ),
+    )
+  )
+
+  # Payload ID Tests
+  for _, payloadAttributeFieldChange := range []PayloadAttributesFieldChange(
+    PayloadAttributesIncreaseTimestamp,
+    PayloadAttributesRandom,
+    PayloadAttributesSuggestedFeeRecipient,
+  ) (
+    result.add UniquePayloadIDTest(
+      FieldModification: payloadAttributeFieldChange,
+    ))
+  )
+
+  # Endpoint Versions Tests
+  # Early upgrade of ForkchoiceUpdated when requesting a payload
+  result.add
+    ForkchoiceUpdatedOnPayloadRequestTest(
+      BaseSpec: test.BaseSpec(
+        Name: "Early upgrade",
+        About: `
+        Early upgrade of ForkchoiceUpdated when requesting a payload.
+        The test sets the fork height to 1, and the block timestamp increments to 2
+        seconds each block.
+        CL Mock prepares the payload attributes for the first block, which should contain
+        the attributes of the next fork.
+        The test then reduces the timestamp by 1, but still uses the next forkchoice updated
+        version, which should result in UNSUPPORTED_FORK_ERROR error.
+        `,
+        forkHeight:              1,
+        BlockTimestampIncrement: 2,
+      ),
+      ForkchoiceUpdatedcustomizer: UpgradeForkchoiceUpdatedVersion(
+        ForkchoiceUpdatedcustomizer: BaseForkchoiceUpdatedCustomizer(
+          PayloadAttributescustomizer: TimestampDeltaPayloadAttributesCustomizer(
+            PayloadAttributescustomizer: BasePayloadAttributesCustomizer(),
+            TimestampDelta:              -1,
+          ),
+          ExpectedError: globals.UNSUPPORTED_FORK_ERROR,
+        ),
+      ),
+    ),
+  )
+
+  # Payload Execution Tests
+  result.add
+    ReExecutePayloadTest(),
+    InOrderPayloadExecutionTest(),
+    MultiplePayloadsExtendingCanonicalChainTest(
+      SetHeadToFirstPayloadReceived: true,
+    ),
+    MultiplePayloadsExtendingCanonicalChainTest(
+      SetHeadToFirstPayloadReceived: false,
+    ),
+    NewPayloadOnSyncingClientTest(),
+    NewPayloadWithMissingFcUTest(),
+  )
+
+
+
+  # Invalid Transaction Payload Tests
+  for _, invalidField := range []InvalidPayloadBlockField(
+    InvalidTransactionSignature,
+    InvalidTransactionNonce,
+    InvalidTransactionGasPrice,
+    InvalidTransactionGasTipPrice,
+    InvalidTransactionGas,
+    InvalidTransactionValue,
+    InvalidTransactionChainID,
+  ) (
+    invalidDetectedOnSync := invalidField == InvalidTransactionChainID
+    for _, syncing in   [false, true) (
+      if invalidField != InvalidTransactionGasTipPrice (
+        for _, testTxType := range []TestTransactionType(TxLegacy, TxEip1559) (
+          result.add InvalidPayloadTestCase(
+            BaseSpec: test.BaseSpec(
+              txType: some( testTxType,
+            ),
+            InvalidField:          invalidField,
+            Syncing:               syncing,
+            InvalidDetectedOnSync: invalidDetectedOnSync,
+          ))
+        )
+      ) else (
+        result.add InvalidPayloadTestCase(
+          BaseSpec: test.BaseSpec(
+            txType: some( TxEip1559,
+          ),
+          InvalidField:          invalidField,
+          Syncing:               syncing,
+          InvalidDetectedOnSync: invalidDetectedOnSync,
+        ))
+      )
+    )
+
+  )
+
+  # Re-org using the Engine API tests
+
+  # Sidechain re-org tests
+  result.add
+    SidechainReOrgTest(),
+    ReOrgBackFromSyncingTest(
+      BaseSpec: test.BaseSpec(
+        slotsToSafe:      u256(32),
+        slotsToFinalized: u256(64),
+      ),
+    ),
+    ReOrgPrevValidatedPayloadOnSideChainTest(
+      BaseSpec: test.BaseSpec(
+        slotsToSafe:      u256(32),
+        slotsToFinalized: u256(64),
+      ),
+    ),
+    SafeReOrgToSideChainTest(
+      BaseSpec: test.BaseSpec(
+        slotsToSafe:      u256(1),
+        slotsToFinalized: u256(2),
+      ),
+    ),
+  )
+
+	// Re-org a transaction out of a block, or into a new block
+	result.add
+		TransactionReOrgTest{
+			Scenario: TransactionReOrgScenarioReOrgOut,
+		},
+		TransactionReOrgTest{
+			Scenario: TransactionReOrgScenarioReOrgDifferentBlock,
+		},
+		TransactionReOrgTest{
+			Scenario: TransactionReOrgScenarioNewPayloadOnRevert,
+		},
+		TransactionReOrgTest{
+			Scenario: TransactionReOrgScenarioReOrgBackIn,
+		},
+	)
+
+  # Re-Org back into the canonical chain tests
+  result.add
+    ReOrgBackToCanonicalTest(
+      BaseSpec: test.BaseSpec(
+        slotsToSafe:      u256(10),
+        slotsToFinalized: u256(20),
+        TimeoutSeconds:   60,
+      ),
+      TransactionPerPayload: 1,
+      ReOrgDepth:            5,
+    ),
+    ReOrgBackToCanonicalTest(
+      BaseSpec: test.BaseSpec(
+        slotsToSafe:      u256(32),
+        slotsToFinalized: u256(64),
+        TimeoutSeconds:   120,
+      ),
+      TransactionPerPayload:     50,
+      ReOrgDepth:                10,
+      ExecuteSidePayloadOnReOrg: true,
+    ),
+  )
+
+  # Suggested Fee Recipient Tests
+  result.add
+    SuggestedFeeRecipientTest(
+      BaseSpec: test.BaseSpec(
+        txType: some( TxLegacy,
+      ),
+      TransactionCount: 20,
+    ),
+    SuggestedFeeRecipientTest(
+      BaseSpec: test.BaseSpec(
+        txType: some( TxEip1559,
+      ),
+      TransactionCount: 20,
+    ),
+  )
 
   # PrevRandao opcode tests
-  TestDesc(
-    name: "PrevRandao Opcode Transactions",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  prevRandaoOpcodeTx,
-      ttd:  10,
-  )),
+  result.add
+    PrevRandaoTransactionTest(
+      BaseSpec: test.BaseSpec(
+        txType: some( TxLegacy,
+      ),
+    ),
+    PrevRandaoTransactionTest(
+      BaseSpec: test.BaseSpec(
+        txType: some( TxEip1559,
+      ),
+    ),
+  )
 
-  # Multi-Client Sync tests
-  TestDesc(
-    name: "Sync Client Post Merge",
-    run: specExecute,
-    spec: EngineSpec(
-      exec:  postMergeSync,
-      ttd:  10,
-  )),]#
-]
+  # Fork ID Tests
+  for genesisTimestamp := uint64(0); genesisTimestamp <= 1; genesisTimestamp++ (
+    for forkTime := uint64(0); forkTime <= 2; forkTime++ (
+      for prevForkTime := uint64(0); prevForkTime <= forkTime; prevForkTime++ (
+        for currentBlock := 0; currentBlock <= 1; currentBlock++ (
+          result.add
+            ForkIDSpec(
+              BaseSpec: test.BaseSpec(
+                MainFork:         config.Paris,
+                Genesistimestamp: pUint64(genesisTimestamp),
+                ForkTime:         forkTime,
+                PreviousForkTime: prevForkTime,
+              ),
+              ProduceBlocksBeforePeering: currentBlock,
+            ),
+          )
+        )
+      )
+    )
+  )
+]#
+
+
+proc fillEngineTests*(): seq[TestDesc] =
+  let list = makeEngineTest()
+  for x in list:
+    result.add TestDesc(
+      name: x.getName(),
+      run: specExecute,
+      spec: x,
+    )
+
+let engineTestList* = fillEngineTests()

hive_integration/nodocker/engine/helper.nim

@@ -1,338 +1,8 @@
 import
-  std/[typetraits],
-  nimcrypto/sysrand,
-  eth/[common, rlp, keys],
-  json_rpc/[rpcclient],
-  ../../../nimbus/transaction,
-  ../../../nimbus/utils/utils,
+  eth/[common, rlp],
   ../../../nimbus/beacon/execution_types,
   ../../../nimbus/beacon/web3_eth_conv
 
-type
-  ExecutableData* = object
-    parentHash*   : common.Hash256
-    feeRecipient* : EthAddress
-    stateRoot*    : common.Hash256
-    receiptsRoot* : common.Hash256
-    logsBloom*    : BloomFilter
-    prevRandao*   : common.Hash256
-    number*       : uint64
-    gasLimit*     : GasInt
-    gasUsed*      : GasInt
-    timestamp*    : EthTime
-    extraData*    : common.Blob
-    baseFeePerGas*: UInt256
-    blockHash*    : common.Hash256
-    transactions* : seq[Transaction]
-    withdrawals*  : Option[seq[Withdrawal]]
-    blobGasUsed*  : Option[uint64]
-    excessBlobGas*: Option[uint64]
-
-  CustomPayload* = object
-    parentHash*   : Option[common.Hash256]
-    feeRecipient* : Option[EthAddress]
-    stateRoot*    : Option[common.Hash256]
-    receiptsRoot* : Option[common.Hash256]
-    logsBloom*    : Option[BloomFilter]
-    prevRandao*   : Option[common.Hash256]
-    number*       : Option[uint64]
-    gasLimit*     : Option[GasInt]
-    gasUsed*      : Option[GasInt]
-    timestamp*    : Option[EthTime]
-    extraData*    : Option[common.Blob]
-    baseFeePerGas*: Option[UInt256]
-    blockHash*    : Option[common.Hash256]
-    transactions* : Option[seq[Transaction]]
-    withdrawals*  : Option[seq[Withdrawal]]
-    blobGasUsed*  : Option[uint64]
-    excessBlobGas*: Option[uint64]
-    beaconRoot*   : Option[common.Hash256]
-    removeWithdrawals*: bool
-
-  InvalidPayloadField* = enum
-    InvalidParentHash
-    InvalidStateRoot
-    InvalidReceiptsRoot
-    InvalidNumber
-    InvalidGasLimit
-    InvalidGasUsed
-    InvalidTimestamp
-    InvalidPrevRandao
-    RemoveTransaction
-    InvalidTransactionSignature
-    InvalidTransactionNonce
-    InvalidTransactionGas
-    InvalidTransactionGasPrice
-    InvalidTransactionValue
-
-  SignatureVal = object
-    V: int64
-    R: UInt256
-    S: UInt256
-
-  CustomTx = object
-    nonce   : Option[AccountNonce]
-    gasPrice: Option[GasInt]
-    gasLimit: Option[GasInt]
-    to      : Option[EthAddress]
-    value   : Option[UInt256]
-    data    : Option[seq[byte]]
-    sig     : Option[SignatureVal]
-
-proc customizePayload*(basePayload: ExecutableData, customData: CustomPayload): ExecutionPayload =
-  let txs = if customData.transactions.isSome:
-              customData.transactions.get
-            else:
-              basePayload.transactions
-  let txRoot = calcTxRoot(txs)
-
-  let wdRoot = if customData.withdrawals.isSome:
-                 some(calcWithdrawalsRoot(customData.withdrawals.get))
-               elif basePayload.withdrawals.isSome:
-                 some(calcWithdrawalsRoot(basePayload.withdrawals.get))
-               else:
-                 none(common.Hash256)
-
-  var customHeader = common.BlockHeader(
-    parentHash:    basePayload.parentHash,
-    ommersHash:    EMPTY_UNCLE_HASH,
-    coinbase:      basePayload.feeRecipient,
-    stateRoot:     basePayload.stateRoot,
-    txRoot:        txRoot,
-    receiptRoot:   basePayload.receiptsRoot,
-    bloom:         basePayload.logsBloom,
-    difficulty:    0.u256,
-    blockNumber:   basePayload.number.toBlockNumber,
-    gasLimit:      basePayload.gasLimit,
-    gasUsed:       basePayload.gasUsed,
-    timestamp:     basePayload.timestamp,
-    extraData:     basePayload.extraData,
-    mixDigest:     basePayload.prevRandao,
-    nonce:         default(BlockNonce),
-    fee:           some(basePayload.baseFeePerGas),
-    withdrawalsRoot: wdRoot,
-    blobGasUsed:   basePayload.blobGasUsed,
-    excessBlobGas: basePayload.excessBlobGas,
-  )
-
-  # Overwrite custom information
-  if customData.parentHash.isSome:
-    customHeader.parentHash = customData.parentHash.get
-
-  if customData.feeRecipient.isSome:
-    customHeader.coinbase = customData.feeRecipient.get
-
-  if customData.stateRoot.isSome:
-    customHeader.stateRoot = customData.stateRoot.get
-
-  if customData.receiptsRoot.isSome:
-    customHeader.receiptRoot = customData.receiptsRoot.get
-
-  if customData.logsBloom.isSome:
-    customHeader.bloom = customData.logsBloom.get
-
-  if customData.prevRandao.isSome:
-    customHeader.mixDigest = customData.prevRandao.get
-
-  if customData.number.isSome:
-    customHeader.blockNumber = toBlockNumber(customData.number.get)
-
-  if customData.gasLimit.isSome:
-    customHeader.gasLimit = customData.gasLimit.get
-
-  if customData.gasUsed.isSome:
-    customHeader.gasUsed = customData.gasUsed.get
-
-  if customData.timestamp.isSome:
-    customHeader.timestamp = customData.timestamp.get
-
-  if customData.extraData.isSome:
-    customHeader.extraData = customData.extraData.get
-
-  if customData.baseFeePerGas.isSome:
-    customHeader.baseFee = customData.baseFeePerGas.get
-
-  if customData.blobGasUsed.isSome:
-    customHeader.blobGasUsed = customData.blobGasUsed
-
-  if customData.excessBlobGas.isSome:
-    customHeader.excessBlobGas = customData.excessBlobGas
-
-  if customData.beaconRoot.isSome:
-    customHeader.parentBeaconBlockRoot = customData.beaconRoot
-
-  # Return the new payload
-  result = ExecutionPayload(
-    parentHash:    w3Hash customHeader.parentHash,
-    feeRecipient:  w3Addr customHeader.coinbase,
-    stateRoot:     w3Hash customHeader.stateRoot,
-    receiptsRoot:  w3Hash customHeader.receiptRoot,
-    logsBloom:     w3Bloom customHeader.bloom,
-    prevRandao:    w3PrevRandao customHeader.mixDigest,
-    blockNumber:   w3Qty customHeader.blockNumber,
-    gasLimit:      w3Qty customHeader.gasLimit,
-    gasUsed:       w3Qty customHeader.gasUsed,
-    timestamp:     w3Qty customHeader.timestamp,
-    extraData:     w3ExtraData customHeader.extraData,
-    baseFeePerGas: customHeader.baseFee,
-    blockHash:     w3Hash customHeader.blockHash,
-    blobGasUsed:   w3Qty customHeader.blobGasUsed,
-    excessBlobGas: w3Qty customHeader.excessBlobGas,
-  )
-
-  for tx in txs:
-    let txData = rlp.encode(tx)
-    result.transactions.add TypedTransaction(txData)
-
-  let wds = if customData.withdrawals.isSome:
-              customData.withdrawals
-            elif basePayload.withdrawals.isSome:
-              basePayload.withdrawals
-            else:
-              none(seq[Withdrawal])
-
-  if wds.isSome and customData.removeWithdrawals.not:
-    result.withdrawals = some(w3Withdrawals(wds.get))
-
-proc toExecutableData*(payload: ExecutionPayload): ExecutableData =
-  result = ExecutableData(
-    parentHash    : ethHash payload.parentHash,
-    feeRecipient  : distinctBase payload.feeRecipient,
-    stateRoot     : ethHash payload.stateRoot,
-    receiptsRoot  : ethHash payload.receiptsRoot,
-    logsBloom     : distinctBase payload.logsBloom,
-    prevRandao    : ethHash payload.prevRandao,
-    number        : uint64 payload.blockNumber,
-    gasLimit      : GasInt payload.gasLimit,
-    gasUsed       : GasInt payload.gasUsed,
-    timestamp     : ethTime payload.timestamp,
-    extraData     : distinctBase payload.extraData,
-    baseFeePerGas : payload.baseFeePerGas,
-    blockHash     : ethHash payload.blockHash,
-    blobGasUsed   : u64 payload.blobGasUsed,
-    excessBlobGas : u64 payload.excessBlobGas,
-    transactions  : ethTxs payload.transactions,
-    withdrawals   : ethWithdrawals payload.withdrawals,
-  )
-
-proc customizePayload*(basePayload: ExecutionPayload, customData: CustomPayload): ExecutionPayload =
-  customizePayload(basePayload.toExecutableData, customData)
-
-proc customizeTx(baseTx: Transaction, vaultKey: PrivateKey, customTx: CustomTx): Transaction =
-  # Create a modified transaction base, from the base transaction and customData mix
-  var modTx = Transaction(
-    txType  : TxLegacy,
-    nonce   : baseTx.nonce,
-    gasPrice: baseTx.gasPrice,
-    gasLimit: baseTx.gasLimit,
-    to      : baseTx.to,
-    value   : baseTx.value,
-    payload : baseTx.payload
-   )
-
-  if customTx.nonce.isSome:
-    modTx.nonce = customTx.nonce.get
-
-  if customTx.gasPrice.isSome:
-    modTx.gasPrice = customTx.gasPrice.get
-
-  if customTx.gasLimit.isSome:
-    modTx.gasLimit = customTx.gasLimit.get
-
-  if customTx.to.isSome:
-    modTx.to = customTx.to
-
-  if customTx.value.isSome:
-    modTx.value = customTx.value.get
-
-  if customTx.data.isSome:
-    modTx.payload = customTx.data.get
-
-  if customTx.sig.isSome:
-    let sig = customTx.sig.get
-    modTx.V = sig.V
-    modTx.R = sig.R
-    modTx.S = sig.S
-    modTx
-  else:
-    # If a custom signature was not specified, simply sign the transaction again
-    let chainId = baseTx.chainId
-    signTransaction(modTx, vaultKey, chainId, eip155 = true)
-
-proc modifyHash(x: common.Hash256): common.Hash256 =
-  result = x
-  result.data[^1] = byte(255 - x.data[^1].int)
-
-proc generateInvalidPayload*(basePayload: ExecutableData,
-                             payloadField: InvalidPayloadField,
-                             vaultKey: PrivateKey): ExecutionPayload =
-
-  var customPayload: CustomPayload
-
-  case payloadField
-  of InvalidParentHash:
-    customPayload.parentHash = some(modifyHash(basePayload.parentHash))
-  of InvalidStateRoot:
-    customPayload.stateRoot = some(modifyHash(basePayload.stateRoot))
-  of InvalidReceiptsRoot:
-    customPayload.receiptsRoot = some(modifyHash(basePayload.receiptsRoot))
-  of InvalidNumber:
-    customPayload.number = some(basePayload.number - 1'u64)
-  of InvalidGasLimit:
-    customPayload.gasLimit = some(basePayload.gasLimit * 2)
-  of InvalidGasUsed:
-    customPayload.gasUsed = some(basePayload.gasUsed - 1)
-  of InvalidTimestamp:
-    customPayload.timestamp = some(basePayload.timestamp - 1)
-  of InvalidPrevRandao:
-    # This option potentially requires a transaction that uses the PREVRANDAO opcode.
-    # Otherwise the payload will still be valid.
-    var randomHash: common.Hash256
-    doAssert randomBytes(randomHash.data) == 32
-    customPayload.prevRandao = some(randomHash)
-  of RemoveTransaction:
-    let emptyTxs: seq[Transaction] = @[]
-    customPayload.transactions = some(emptyTxs)
-  of InvalidTransactionSignature,
-    InvalidTransactionNonce,
-    InvalidTransactionGas,
-    InvalidTransactionGasPrice,
-    InvalidTransactionValue:
-
-    doAssert(basePayload.transactions.len != 0, "No transactions available for modification")
-
-    var baseTx = basePayload.transactions[0]
-    var customTx: CustomTx
-    case payloadField
-    of InvalidTransactionSignature:
-      let sig = SignatureVal(
-        V: baseTx.V,
-        R: baseTx.R - 1.u256,
-        S: baseTx.S
-      )
-      customTx.sig = some(sig)
-    of InvalidTransactionNonce:
-      customTx.nonce = some(baseTx.nonce - 1)
-    of InvalidTransactionGas:
-      customTx.gasLimit = some(0.GasInt)
-    of InvalidTransactionGasPrice:
-      customTx.gasPrice = some(0.GasInt)
-    of InvalidTransactionValue:
-      # Vault account initially has 0x123450000000000000000, so this value should overflow
-      customTx.value = some(UInt256.fromHex("0x123450000000000000001"))
-    else:
-      discard
-
-    let modTx = customizeTx(baseTx, vaultKey, customTx)
-    customPayload.transactions = some(@[modTx])
-
-  customizePayload(basePayload, customPayload)
-
-proc generateInvalidPayload*(basePayload: ExecutionPayload,
-                             payloadField: InvalidPayloadField,
-                             vaultKey = default(PrivateKey)): ExecutionPayload =
-  generateInvalidPayload(basePayload.toExecutableData, payloadField, vaultKey)
-
 proc txInPayload*(payload: ExecutionPayload, txHash: common.Hash256): bool =
   for txBytes in payload.transactions:
     let currTx = rlp.decode(common.Blob txBytes, Transaction)

hive_integration/nodocker/engine/test_env.nim

@@ -9,7 +9,9 @@ import
   ./engine_client,
   ./client_pool,
   ./engine_env,
-  ./tx_sender
+  ./tx_sender,
+  ./types,
+  ./cancun/customizer
 
 export
   clmock,
@@ -79,6 +81,9 @@ func client*(env: TestEnv): RpcHttpClient =
 func engine*(env: TestEnv): EngineEnv =
   env.clients.first
 
+func sender*(env: TesTenv): TxSender =
+  env.sender
+
 proc setupCLMock*(env: TestEnv) =
   env.clmock = newCLMocker(env.engine, env.engine.com)
 
@@ -149,6 +154,20 @@ proc sendTx*(env: TestEnv, sender: TestAccount, eng: EngineEnv, tc: BlobTx): Res
 proc replaceTx*(env: TestEnv, sender: TestAccount, eng: EngineEnv, tc: BlobTx): Result[Transaction, void] =
   env.sender.replaceTx(sender, eng.client, tc)
 
+proc makeTx*(env: TestEnv, tc: BaseTx, sender: TestAccount, nonce: AccountNonce): Transaction =
+  env.sender.makeTx(tc, sender, nonce)
+
+proc customizeTransaction*(env: TestEnv,
+                           acc: TestAccount,
+                           baseTx: Transaction,
+                           custTx: CustomTransactionData): Transaction =
+  env.sender.customizeTransaction(acc, baseTx, custTx)
+
+proc generateInvalidPayload*(env: TestEnv,
+                             data: ExecutableData,
+                             payloadField: InvalidPayloadBlockField): ExecutableData =
+  env.sender.generateInvalidPayload(data, payloadField)
+
 proc verifyPoWProgress*(env: TestEnv, lastBlockHash: common.Hash256): bool =
   let res = waitFor env.client.verifyPoWProgress(lastBlockHash)
   if res.isErr:

hive_integration/nodocker/engine/tx_sender.nim

@@ -32,9 +32,9 @@ type
     blobCount* : int
 
   TestAccount* = object
-    key    : PrivateKey
-    address: EthAddress
-    index  : int
+    key*    : PrivateKey
+    address*: EthAddress
+    index*  : int
 
   TxSender* = ref object
     accounts: seq[TestAccount]
@@ -47,6 +47,11 @@ type
     key*    : PrivateKey
     nonce*  : AccountNonce
 
+  CustSig* = object
+    V*: int64
+    R*: UInt256
+    S*: UInt256
+
   CustomTransactionData* = object
     nonce*              : Option[uint64]
     gasPriceOrGasFeeCap*: Option[GasInt]
@@ -56,7 +61,7 @@ type
     value*              : Option[UInt256]
     data*               : Option[seq[byte]]
     chainId*            : Option[ChainId]
-    signature*          : Option[UInt256]
+    signature*          : Option[CustSig]
 
 const
   TestAccountCount = 1000
@@ -81,7 +86,7 @@ proc createAccounts(sender: TxSender) =
   for i in 0..<TestAccountCount:
     sender.accounts.add createAccount(i.int)
 
-proc getNextAccount(sender: TxSender): TestAccount =
+proc getNextAccount*(sender: TxSender): TestAccount =
   sender.accounts[sender.txSent mod sender.accounts.len]
 
 proc getNextNonce(sender: TxSender, address: EthAddress): uint64 =
@@ -99,7 +104,7 @@ proc fillBalance(sender: TxSender, params: NetworkParams) =
     )
 
 proc new*(_: type TxSender, params: NetworkParams): TxSender =
-  result = TxSender(chainId: params.config.chainId)
+  result = TxSender(chainId: params.config.chainID)
   result.createAccounts()
   result.fillBalance(params)
 
@@ -140,10 +145,10 @@ proc makeTx(params: MakeTxParams, tc: BaseTx): Transaction =
                to      : tc.recipient,
                value   : tc.amount,
                payload : tc.payload,
-               chainId : params.chainId
+               chainId : params.chainID
              )
 
-  signTransaction(tx, params.key, params.chainId, eip155 = true)
+  signTransaction(tx, params.key, params.chainID, eip155 = true)
 
 proc makeTx(params: MakeTxParams, tc: BigInitcodeTx): Transaction =
   var tx = tc
@@ -162,7 +167,7 @@ proc makeTx(params: MakeTxParams, tc: BigInitcodeTx): Transaction =
 proc makeTx*(sender: TxSender, tc: BaseTx, nonce: AccountNonce): Transaction =
   let acc = sender.getNextAccount()
   let params = MakeTxParams(
-    chainId: sender.chainId,
+    chainId: sender.chainID,
     key: acc.key,
     nonce: nonce
   )
@@ -171,7 +176,7 @@ proc makeTx*(sender: TxSender, tc: BaseTx, nonce: AccountNonce): Transaction =
 proc makeTx*(sender: TxSender, tc: BigInitcodeTx, nonce: AccountNonce): Transaction =
   let acc = sender.getNextAccount()
   let params = MakeTxParams(
-    chainId: sender.chainId,
+    chainId: sender.chainID,
     key: acc.key,
     nonce: nonce
   )
@@ -182,7 +187,7 @@ proc makeNextTx*(sender: TxSender, tc: BaseTx): Transaction =
     acc = sender.getNextAccount()
     nonce = sender.getNextNonce(acc.address)
     params = MakeTxParams(
-      chainId: sender.chainId,
+      chainId: sender.chainID,
       key: acc.key,
       nonce: nonce
     )
@@ -192,15 +197,17 @@ proc sendNextTx*(sender: TxSender, client: RpcClient, tc: BaseTx): bool =
   let tx = sender.makeNextTx(tc)
   let rr = client.sendTransaction(tx)
   if rr.isErr:
-    error "Unable to send transaction", msg=rr.error
+    error "sendNextTx: Unable to send transaction", msg=rr.error
     return false
+
+  inc sender.txSent
   return true
 
 proc sendTx*(sender: TxSender, client: RpcClient, tc: BaseTx, nonce: AccountNonce): bool =
   let
     acc = sender.getNextAccount()
     params = MakeTxParams(
-      chainId: sender.chainId,
+      chainId: sender.chainID,
       key: acc.key,
       nonce: nonce
     )
@@ -208,15 +215,17 @@ proc sendTx*(sender: TxSender, client: RpcClient, tc: BaseTx, nonce: AccountNonc
 
   let rr = client.sendTransaction(tx)
   if rr.isErr:
-    error "Unable to send transaction", msg=rr.error
+    error "sendTx: Unable to send transaction", msg=rr.error
     return false
+
+  inc sender.txSent
   return true
 
 proc sendTx*(sender: TxSender, client: RpcClient, tc: BigInitcodeTx, nonce: AccountNonce): bool =
   let
     acc = sender.getNextAccount()
     params = MakeTxParams(
-      chainId: sender.chainId,
+      chainId: sender.chainID,
       key: acc.key,
       nonce: nonce
     )
@@ -226,6 +235,8 @@ proc sendTx*(sender: TxSender, client: RpcClient, tc: BigInitcodeTx, nonce: Acco
   if rr.isErr:
     error "Unable to send transaction", msg=rr.error
     return false
+
+  inc sender.txSent
   return true
 
 proc sendTx*(client: RpcClient, tx: Transaction): bool =
@@ -249,7 +260,7 @@ proc makeTx*(params: MakeTxParams, tc: BlobTx): Transaction =
 
   let unsignedTx = Transaction(
     txType    : TxEip4844,
-    chainId   : params.chainId,
+    chainId   : params.chainID,
     nonce     : params.nonce,
     maxPriorityFee: gasTipCap,
     maxFee    : gasFeeCap,
@@ -261,7 +272,7 @@ proc makeTx*(params: MakeTxParams, tc: BlobTx): Transaction =
     versionedHashes: data.hashes,
   )
 
-  var tx = signTransaction(unsignedTx, params.key, params.chainId, eip155 = true)
+  var tx = signTransaction(unsignedTx, params.key, params.chainID, eip155 = true)
   tx.networkPayload = NetworkPayload(
     blobs      : data.blobs,
     commitments: data.commitments,
@@ -276,7 +287,7 @@ proc getAccount*(sender: TxSender, idx: int): TestAccount =
 proc sendTx*(sender: TxSender, acc: TestAccount, client: RpcClient, tc: BlobTx): Result[Transaction, void] =
   let
     params = MakeTxParams(
-      chainId: sender.chainId,
+      chainId: sender.chainID,
       key: acc.key,
       nonce: sender.getNextNonce(acc.address),
     )
@@ -286,12 +297,14 @@ proc sendTx*(sender: TxSender, acc: TestAccount, client: RpcClient, tc: BlobTx):
   if rr.isErr:
     error "Unable to send transaction", msg=rr.error
     return err()
+
+  inc sender.txSent
   return ok(tx)
 
 proc replaceTx*(sender: TxSender, acc: TestAccount, client: RpcClient, tc: BlobTx): Result[Transaction, void] =
   let
     params = MakeTxParams(
-      chainId: sender.chainId,
+      chainId: sender.chainID,
       key: acc.key,
       nonce: sender.getLastNonce(acc.address),
     )
@@ -301,7 +314,65 @@ proc replaceTx*(sender: TxSender, acc: TestAccount, client: RpcClient, tc: BlobT
   if rr.isErr:
     error "Unable to send transaction", msg=rr.error
     return err()
+
+  inc sender.txSent
   return ok(tx)
 
-proc customizeTransaction*(sender: TxSender, baseTx: Transaction, custTx: CustomTransactionData): Transaction =
-  discard
+proc makeTx*(sender: TxSender, tc: BaseTx, acc: TestAccount, nonce: AccountNonce): Transaction =
+  let
+    params = MakeTxParams(
+      chainId: sender.chainID,
+      key: acc.key,
+      nonce: nonce,
+    )
+  params.makeTx(tc)
+
+proc customizeTransaction*(sender: TxSender,
+                           acc: TestAccount,
+                           baseTx: Transaction,
+                           custTx: CustomTransactionData): Transaction =
+  # Create a modified transaction base, from the base transaction and custTx mix
+  var modTx = baseTx
+  if custTx.nonce.isSome:
+    modTx.nonce = custTx.nonce.get.AccountNonce
+
+  if custTx.gasPriceOrGasFeeCap.isSome:
+    modTx.gasPrice = custTx.gasPriceOrGasFeeCap.get.GasInt
+
+  if custTx.gas.isSome:
+    modTx.gasLimit = custTx.gas.get.GasInt
+
+  if custTx.to.isSome:
+    modTx.to = custTx.to
+
+  if custTx.value.isSome:
+    modTx.value = custTx.value.get
+
+  if custTx.data.isSome:
+    modTx.payload = custTx.data.get
+
+  if custTx.signature.isSome:
+    let signature = custTx.signature.get
+    modTx.V = signature.V
+    modTx.R = signature.R
+    modTx.S = signature.S
+
+  if baseTx.txType in {TxEip1559, TxEip4844}:
+    if custTx.chainID.isSome:
+      modTx.chainID = custTx.chainID.get
+
+    if custTx.gasPriceOrGasFeeCap.isSome:
+      modTx.maxFee = custTx.gasPriceOrGasFeeCap.get.GasInt
+
+    if custTx.gasTipCap.isSome:
+      modTx.maxPriorityFee = custTx.gasTipCap.get.GasInt
+
+  if baseTx.txType == TxEip4844:
+    if modTx.to.isNone:
+      var address: EthAddress
+      modTx.to = some(address)
+
+  if custTx.signature.isNone:
+    return signTransaction(modTx, acc.key, modTx.chainID, eip155 = true)
+
+  return modTx

hive_integration/nodocker/engine/types.nim

@@ -1,13 +1,25 @@
 import
   std/[options, typetraits, strutils],
   eth/common,
+  nimcrypto/sysrand,
   stew/[byteutils, endians2],
   web3/ethtypes,
   web3/engine_api_types,
   ../../../nimbus/beacon/execution_types,
-  ../../../nimbus/beacon/web3_eth_conv
+  ../../../nimbus/beacon/web3_eth_conv,
+  ../../../nimbus/utils/utils
+
+export
+  execution_types,
+  web3_eth_conv
 
 type
+  EngineFork* = enum
+    ForkNone     = "none"
+    ForkParis    = "Merge"
+    ForkShanghai = "Shanghai"
+    ForkCancun   = "Cancun"
+
   BaseSpec* = ref object of RootObj
     txType*: Option[TxType]
 
@@ -17,7 +29,7 @@ type
     safeSlotsToImportOptimistically*: int
     blockTimestampIncrement*: int
     timeoutSeconds*: int
-    mainFork*: string
+    mainFork*: EngineFork
     genesisTimestamp*: int
     forkHeight*: int
     forkTime*: uint64
@@ -40,9 +52,6 @@ const
   DefaultSleep* = 1
   prevRandaoContractAddr* = hexToByteArray[20]("0000000000000000000000000000000000000316")
   GenesisTimestamp* = 0x1234
-  ForkParis*    = "Paris"
-  ForkShanghai* = "Shanghai"
-  ForkCancun*   = "Cancun"
 
 func toAddress*(x: UInt256): EthAddress =
   var
@@ -52,6 +61,9 @@ func toAddress*(x: UInt256): EthAddress =
     result[x] = mm[i]
     inc x
 
+const
+  ZeroAddr* =  toAddress(0.u256)
+
 func toHash*(x: UInt256): common.Hash256 =
   common.Hash256(data: x.toByteArrayBE)
 
@@ -63,6 +75,21 @@ func timestampToBeaconRoot*(timestamp: Quantity): FixedBytes[32] =
 func beaconRoot*(x: UInt256): FixedBytes[32] =
   FixedBytes[32](x.toByteArrayBE)
 
+proc randomBytes*(_: type common.Hash256): common.Hash256 =
+  doAssert randomBytes(result.data) == 32
+
+proc randomBytes*(_: type common.EthAddress): common.EthAddress =
+  doAssert randomBytes(result) == 20
+
+proc randomBytes*(_: type Web3Hash): Web3Hash =
+  var res: array[32, byte]
+  doAssert randomBytes(res) == 32
+  result = Web3Hash(res)
+
+proc clone*[T](x: T): T =
+  result = T()
+  result[] = x[]
+
 template testCond*(expr: untyped) =
   if not (expr):
     return false
@@ -81,24 +108,6 @@ proc `==`*(a: Option[BlockHash], b: Option[common.Hash256]): bool =
 proc `==`*(a, b: TypedTransaction): bool =
   distinctBase(a) == distinctBase(b)
 
-template testFCU*(res, cond: untyped, validHash: Option[common.Hash256], id = none(PayloadID)) =
-  testCond res.isOk:
-    error "Unexpected FCU Error", msg=res.error
-  let s = res.get()
-  testCond s.payloadStatus.status == PayloadExecutionStatus.cond:
-    error "Unexpected FCU status", expect=PayloadExecutionStatus.cond, get=s.payloadStatus.status
-  testCond s.payloadStatus.latestValidHash == validHash:
-    error "Unexpected FCU latestValidHash", expect=validHash, get=s.payloadStatus.latestValidHash
-  testCond s.payloadId == id:
-    error "Unexpected FCU payloadID", expect=id, get=s.payloadId
-
-template testFCU*(res, cond: untyped) =
-  testCond res.isOk:
-    error "Unexpected FCU Error", msg=res.error
-  let s = res.get()
-  testCond s.payloadStatus.status == PayloadExecutionStatus.cond:
-    error "Unexpected FCU status", expect=PayloadExecutionStatus.cond, get=s.payloadStatus.status
-
 template expectErrorCode*(res: untyped, errCode: int) =
   testCond res.isErr:
     error "unexpected result, want error, get ok"
@@ -121,26 +130,10 @@ template expectPayload*(res: untyped, payload: ExecutionPayload) =
     testCond x.executionPayload == payload.V3:
       error "getPayloadV3 return mismatch payload"
 
-template expectStatus*(res, cond: untyped) =
-  testCond res.isOk:
-    error "Unexpected newPayload error", msg=res.error
-  let s = res.get()
-  testCond s.status == PayloadExecutionStatus.cond:
-    error "Unexpected newPayload status", expect=PayloadExecutionStatus.cond, get=s.status
-
-template expectStatusEither*(res, cond1, cond2: untyped) =
-  testCond res.isOk:
-    error "Unexpected newPayload error", msg=res.error
-  let s = res.get()
-  testCond s.status == PayloadExecutionStatus.cond1 or s.status == PayloadExecutionStatus.cond2:
-    error "Unexpected newPayload status",
-      expect1=PayloadExecutionStatus.cond1,
-      expect2=PayloadExecutionStatus.cond2,
-      get=s.status
-
-template expectWithdrawalsRoot*(res: untyped, h: common.BlockHeader, wdRoot: Option[common.Hash256]) =
+template expectWithdrawalsRoot*(res: untyped, wdRoot: Option[common.Hash256]) =
   testCond res.isOk:
     error "Unexpected error", msg=res.error
+  let h = res.get
   testCond h.withdrawalsRoot == wdRoot:
     error "wdroot mismatch"
 
@@ -154,10 +147,23 @@ template expectLatestValidHash*(res: untyped, expectedHash: Web3Hash) =
   testCond res.isOk:
     error "Unexpected error", msg=res.error
   let s = res.get
-  testCond s.latestValidHash.isSome:
-    error "Expect latest valid hash isSome"
-  testCond s.latestValidHash.get == expectedHash:
-    error "latest valid hash mismatch", expect=expectedHash, get=s.latestValidHash.get
+  when s is PayloadStatusV1:
+    testCond s.latestValidHash.isSome:
+      error "Expect latest valid hash isSome"
+    testCond s.latestValidHash.get == expectedHash:
+      error "latest valid hash mismatch", expect=expectedHash, get=s.latestValidHash.get
+  else:
+    testCond s.payloadStatus.latestValidHash.isSome:
+      error "Expect latest valid hash isSome"
+    testCond s.payloadStatus.latestValidHash.get == expectedHash:
+      error "latest valid hash mismatch", expect=expectedHash, get=s.payloadStatus.latestValidHash.get
+
+template expectLatestValidHash*(res: untyped) =
+  testCond res.isOk:
+    error "Unexpected error", msg=res.error
+  let s = res.get
+  testCond s.latestValidHash.isNone:
+    error "Expect latest valid hash isNone"
 
 template expectErrorCode*(res: untyped, errCode: int, expectedDesc: string) =
   testCond res.isErr:
@@ -169,6 +175,17 @@ template expectNoError*(res: untyped, expectedDesc: string) =
   testCond res.isOk:
     fatal "DEBUG", msg=expectedDesc, err=res.error
 
+template expectStatusEither*(res: untyped, cond: openArray[PayloadExecutionStatus]) =
+  testCond res.isOk:
+    error "Unexpected expectStatusEither error", msg=res.error
+  let s = res.get()
+  when s is PayloadStatusV1:
+    testCond s.status in cond:
+      error "Unexpected expectStatusEither status", expect=cond, get=s.status
+  else:
+    testCond s.payloadStatus.status in cond:
+      error "Unexpected expectStatusEither status", expect=cond, get=s.payloadStatus.status
+
 template expectPayloadStatus*(res: untyped, cond: PayloadExecutionStatus) =
   testCond res.isOk:
     error "Unexpected FCU Error", msg=res.error
@@ -176,9 +193,45 @@ template expectPayloadStatus*(res: untyped, cond: PayloadExecutionStatus) =
   testCond s.payloadStatus.status == cond:
     error "Unexpected FCU status", expect=cond, get=s.payloadStatus.status
 
-template expectNPStatus*(res: untyped, cond: PayloadExecutionStatus) =
+template expectStatus*(res: untyped, cond: PayloadExecutionStatus) =
   testCond res.isOk:
     error "Unexpected newPayload error", msg=res.error
   let s = res.get()
   testCond s.status == cond:
     error "Unexpected newPayload status", expect=cond, get=s.status
+
+template expectPayloadID*(res: untyped, id: Option[PayloadID]) =
+  testCond res.isOk:
+    error "Unexpected expectPayloadID Error", msg=res.error
+  let s = res.get()
+  testCond s.payloadId == id:
+    error "Unexpected expectPayloadID payloadID", expect=id, get=s.payloadId
+
+template expectError*(res: untyped) =
+  testCond res.isErr:
+    error "Unexpected expectError, got noerror"
+
+template expectHash*(res: untyped, hash: common.Hash256) =
+  testCond res.isOk:
+    error "Unexpected expectHash Error", msg=res.error
+  let s = res.get()
+  testCond s.blockHash == hash:
+    error "Unexpected expectHash", expect=hash.short, get=s.blockHash.short
+
+func timestamp*(x: ExecutableData): auto =
+  x.basePayload.timestamp
+
+func parentHash*(x: ExecutableData): auto =
+  x.basePayload.parentHash
+
+func blockHash*(x: ExecutableData): auto =
+  x.basePayload.blockHash
+
+func blockNumber*(x: ExecutableData): auto =
+  x.basePayload.blockNumber
+
+proc `parentHash=`*(x: var ExecutableData, val: auto) =
+  x.basePayload.parentHash = val
+
+proc `blockHash=`*(x: var ExecutableData, val: auto) =
+  x.basePayload.blockHash = val

hive_integration/nodocker/engine/withdrawal_tests.nim

@@ -15,10 +15,10 @@ proc specExecute[T](ws: BaseSpec): bool =
     ws   = T(ws)
     conf = envConfig(ws.getForkConfig())
 
-  discard ws.getGenesis(conf.networkParams)
+  ws.getGenesis(conf.networkParams)
 
   let env  = TestEnv.new(conf)
-  env.engine.setRealTTD(0)
+  env.engine.setRealTTD()
   env.setupCLMock()
   ws.configureCLMock(env.clMock)
   result = ws.execute(env)

hive_integration/nodocker/engine/withdrawals/wd_base_spec.nim

@@ -4,14 +4,13 @@ import
   chronicles,
   chronos,
   stew/byteutils,
-  nimcrypto/sysrand,
   web3/ethtypes,
   ./wd_history,
-  ../helper,
   ../test_env,
   ../engine_client,
   ../types,
   ../base_spec,
+  ../cancun/customizer,
   ../../../nimbus/common/common,
   ../../../nimbus/utils/utils,
   ../../../nimbus/common/chain_config,
@@ -78,7 +77,7 @@ func getWithdrawableAccountCount*(ws: WDBaseSpec):int =
 
 # Append the accounts we are going to withdraw to, which should also include
 # bytecode for testing purposes.
-func getGenesis*(ws: WDBaseSpec, param: NetworkParams): NetworkParams =
+func getGenesis*(ws: WDBaseSpec, param: NetworkParams) =
   # Remove PoW altogether
   param.genesis.difficulty = 0.u256
   param.config.terminalTotalDifficulty = some(0.u256)
@@ -138,8 +137,6 @@ func getGenesis*(ws: WDBaseSpec, param: NetworkParams): NetworkParams =
     balance: 0.u256,
   )
 
-  param
-
 func getTransactionCountPerPayload*(ws: WDBaseSpec): int =
   ws.txPerBlock.get(16)
 
@@ -211,22 +208,12 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
     # contain `withdrawalsRoot`, including genesis.
 
     # Genesis should not contain `withdrawalsRoot` either
-    var h: common.BlockHeader
-    let r = env.client.latestHeader(h)
-    testCond r.isOk:
-      error "failed to ge latest header", msg=r.error
-    testCond h.withdrawalsRoot.isNone:
-      error "genesis should not contains wdsRoot"
+    let r = env.client.latestHeader()
+    r.expectWithdrawalsRoot(none(common.Hash256))
   else:
     # Genesis is post shanghai, it should contain EmptyWithdrawalsRoot
-    var h: common.BlockHeader
-    let r = env.client.latestHeader(h)
-    testCond r.isOk:
-      error "failed to ge latest header", msg=r.error
-    testCond h.withdrawalsRoot.isSome:
-      error "genesis should contains wdsRoot"
-    testCond h.withdrawalsRoot.get == EMPTY_ROOT_HASH:
-      error "genesis should contains wdsRoot==EMPTY_ROOT_HASH"
+    let r = env.client.latestHeader()
+    r.expectWithdrawalsRoot(some(EMPTY_ROOT_HASH))
 
   # Produce any blocks necessary to reach withdrawals fork
   var pbRes = env.clMock.produceBlocks(ws.getPreWithdrawalsBlockCount, BlockProcessCallbacks(
@@ -292,11 +279,11 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
         # Send produced payload but try to include non-nil
         # `withdrawals`, it should fail.
         let emptyWithdrawalsList = newSeq[Withdrawal]()
-        let customizer = CustomPayload(
+        let customizer = CustomPayloadData(
           withdrawals: some(emptyWithdrawalsList),
-          beaconRoot: ethHash env.clMock.latestPayloadAttributes.parentBeaconBlockRoot
+          parentBeaconRoot: ethHash env.clMock.latestPayloadAttributes.parentBeaconBlockRoot
         )
-        let payloadPlusWithdrawals = customizePayload(env.clMock.latestPayloadBuilt, customizer)
+        let payloadPlusWithdrawals = customizer.customizePayload(env.clMock.latestExecutableData).basePayload
         var r = env.client.newPayloadV2(payloadPlusWithdrawals.V1V2)
         #r.ExpectationDescription = "Sent pre-shanghai payload using NewPayloadV2+Withdrawals, error is expected"
         r.expectErrorCode(engineApiInvalidParams)
@@ -304,18 +291,17 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
         # Send valid ExecutionPayloadV1 using engine_newPayloadV2
         r = env.client.newPayloadV2(env.clMock.latestPayloadBuilt.V1V2)
         #r.ExpectationDescription = "Sent pre-shanghai payload using NewPayloadV2, no error is expected"
-        r.expectStatus(valid)
+        r.expectStatus(PayloadExecutionStatus.valid)
       return true
     ,
     onNewPayloadBroadcast: proc(): bool =
       if not ws.skipBaseVerifications:
         # We sent a pre-shanghai FCU.
         # Keep expecting `nil` until Shanghai.
-        var h: common.BlockHeader
-        let r = env.client.latestHeader(h)
+        let r = env.client.latestHeader()
         #r.ExpectationDescription = "Requested "latest" block expecting block to contain
         #" withdrawalRoot=nil, because (block %d).timestamp < shanghaiTime
-        r.expectWithdrawalsRoot(h, none(common.Hash256))
+        r.expectWithdrawalsRoot(none(common.Hash256))
       return true
     ,
     onForkchoiceBroadcast: proc(): bool =
@@ -383,11 +369,11 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
         # with null, and client must respond with `InvalidParamsError`.
         # Note that StateRoot is also incorrect but null withdrawals should
         # be checked first instead of responding `INVALID`
-        let customizer = CustomPayload(
+        let customizer = CustomPayloadData(
           removeWithdrawals: true,
-          beaconRoot: ethHash env.clMock.latestPayloadAttributes.parentBeaconBlockRoot
+          parentBeaconRoot: ethHash env.clMock.latestPayloadAttributes.parentBeaconBlockRoot
         )
-        let nilWithdrawalsPayload = customizePayload(env.clMock.latestPayloadBuilt, customizer)
+        let nilWithdrawalsPayload = customizer.customizePayload(env.clMock.latestExecutableData).basePayload
         let r = env.client.newPayloadV2(nilWithdrawalsPayload.V1V2)
         #r.ExpectationDescription = "Sent shanghai payload using ExecutionPayloadV1, error is expected"
         r.expectErrorCode(engineApiInvalidParams)
@@ -439,14 +425,13 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
           var payload = env.clMock.latestExecutedPayload
 
           # Corrupt the hash
-          var randomHash: common.Hash256
-          testCond randomBytes(randomHash.data) == 32
+          let randomHash = common.Hash256.randomBytes()
           payload.blockHash = w3Hash randomHash
 
           # On engine_newPayloadV2 `INVALID_BLOCK_HASH` is deprecated
           # in favor of reusing `INVALID`
           let n = env.client.newPayloadV2(payload.V1V2)
-          n.expectStatus(invalid)
+          n.expectStatus(PayloadExecutionStatus.invalid)
       return true
     ,
     onForkchoiceBroadcast: proc(): bool =
@@ -477,13 +462,12 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
         let expectedWithdrawalsRoot = some(calcWithdrawalsRoot(wds.list))
 
         # Check the correct withdrawal root on `latest` block
-        var h: common.BlockHeader
-        let r = env.client.latestHeader(h)
+        let r = env.client.latestHeader()
         #r.ExpectationDescription = fmt.Sprintf(`
         #    Requested "latest" block after engine_forkchoiceUpdatedV2,
         #    to verify withdrawalsRoot with the following withdrawals:
         #    %s`, jsWithdrawals)
-        r.expectWithdrawalsRoot(h, expectedWithdrawalsRoot)
+        r.expectWithdrawalsRoot(expectedWithdrawalsRoot)
 
         let res = ws.verifyContractsStorage(env)
         testCond res.isOk:
@@ -504,9 +488,7 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
         error "verify wd error", msg=res.error
 
       # Check the correct withdrawal root on past blocks
-      var h: common.BlockHeader
-      let r = env.client.headerByNumber(bn, h)
-
+      let r = env.client.headerByNumber(bn)
       var expectedWithdrawalsRoot: Option[common.Hash256]
       if bn >= ws.forkHeight.uint64:
         let wds = ws.wdHistory.getWithdrawals(bn)
@@ -516,7 +498,7 @@ proc execute*(ws: WDBaseSpec, env: TestEnv): bool =
       #      Requested block %d to verify withdrawalsRoot with the
       #      following withdrawals:
       #      %s`, block, jsWithdrawals)
-      r.expectWithdrawalsRoot(h, expectedWithdrawalsRoot)
+      r.expectWithdrawalsRoot(expectedWithdrawalsRoot)
 
     # Verify on `latest`
     let bnu = env.clMock.latestExecutedPayload.blockNumber.uint64

hive_integration/nodocker/engine/withdrawals/wd_max_init_code_spec.nim

@@ -7,7 +7,7 @@ import
   ../test_env,
   ../engine_client,
   ../types,
-  ../helper,
+  ../cancun/customizer,
   ../../../nimbus/constants,
   ../../../nimbus/beacon/execution_types,
   ../../../nimbus/beacon/web3_eth_conv
@@ -82,7 +82,7 @@ proc execute*(ws: MaxInitcodeSizeSpec, env: TestEnv): bool =
       error "Invalid tx was not unknown to the client"
 
   # Try to include an invalid tx in new payload
-  let    
+  let
     validTx   = env.makeTx(validTxCreator, txIncluded)
     invalidTx = env.makeTx(invalidTxCreator, txIncluded)
 
@@ -100,14 +100,14 @@ proc execute*(ws: MaxInitcodeSizeSpec, env: TestEnv): bool =
         error "valid Tx bytes mismatch"
 
       # Customize the payload to include a tx with an invalid initcode
-      let customData = CustomPayload(
-        beaconRoot: ethHash env.clMock.latestPayloadAttributes.parentBeaconBlockRoot,
+      let customizer = CustomPayloadData(
+        parentBeaconRoot: ethHash env.clMock.latestPayloadAttributes.parentBeaconBlockRoot,
         transactions: some( @[invalidTx] ),
       )
 
-      let customPayload = customizePayload(env.clMock.latestPayloadBuilt, customData)
+      let customPayload = customizer.customizePayload(env.clMock.latestExecutableData).basePayload
       let res = env.client.newPayloadV2(customPayload.V1V2)
-      res.expectStatus(invalid)
+      res.expectStatus(PayloadExecutionStatus.invalid)
       res.expectLatestValidHash(env.clMock.latestPayloadBuilt.parentHash)
 
       return true

hive_integration/nodocker/engine/withdrawals/wd_payload_body_spec.nim

@@ -88,7 +88,7 @@ func (req GetPayloadBodyRequestByHashIndex) Verify(reqIndex int, testEngine *tes
       } else {
         # signal to request an unknown hash (random)
         randHash := common.Hash{}
-        rand.Read(randHash[:])
+        randomBytes(randHash[:])
         payloads = append(payloads, nil)
         hashes = append(hashes, randHash)
       }
@@ -102,7 +102,7 @@ func (req GetPayloadBodyRequestByHashIndex) Verify(reqIndex int, testEngine *tes
       } else {
         # signal to request an unknown hash (random)
         randHash := common.Hash{}
-        rand.Read(randHash[:])
+        randomBytes(randHash[:])
         payloads = append(payloads, nil)
         hashes = append(hashes, randHash)
       }
@@ -152,10 +152,10 @@ proc execute*(ws: GetPayloadBodiesSpec, t: TestEnv): bool =
         Withdrawals: nextWithdrawals,
       },
     )
-    f.expectPayloadStatus(test.Valid)
+    f.expectPayloadStatus(PayloadExecutionStatus.valid)
 
     # Wait for payload to be built
-    time.Sleep(time.Second)
+    await sleepAsync(time.Second)
 
     # Get the next canonical payload
     p := t.rpcClient.getPayloadV2(f.Response.PayloadID)
@@ -164,16 +164,16 @@ proc execute*(ws: GetPayloadBodiesSpec, t: TestEnv): bool =
 
     # Now we have an extra payload that follows the canonical chain,
     # but we need a side chain for the test.
-    customizer := &helper.CustomPayloadData{
-      Withdrawals: helper.RandomizeWithdrawalsOrder(t.clMock.latestExecutedPayload.Withdrawals),
+    customizer := CustomPayloadData(
+      Withdrawals: RandomizeWithdrawalsOrder(t.clMock.latestExecutedPayload.Withdrawals),
     }
     sidechainCurrent, _, err := customizer.CustomizePayload(&t.clMock.latestExecutedPayload, t.clMock.latestPayloadAttributes.BeaconRoot)
     if err != nil {
       error "Error obtaining custom sidechain payload: %v", t.TestName, err)
     }
-    customizer = &helper.CustomPayloadData{
+    customizer = CustomPayloadData(
       ParentHash:  &sidechainCurrent.BlockHash,
-      Withdrawals: helper.RandomizeWithdrawalsOrder(nextCanonicalPayload.Withdrawals),
+      Withdrawals: RandomizeWithdrawalsOrder(nextCanonicalPayload.Withdrawals),
     }
     sidechainHead, _, err := customizer.CustomizePayload(nextCanonicalPayload, t.clMock.latestPayloadAttributes.BeaconRoot)
     if err != nil {
@@ -182,9 +182,9 @@ proc execute*(ws: GetPayloadBodiesSpec, t: TestEnv): bool =
 
     # Send both sidechain payloads as engine_newPayloadV2
     n1 := t.rpcClient.newPayloadV2(sidechainCurrent)
-    n1.expectStatus(test.Valid)
+    n1.expectStatus(PayloadExecutionStatus.valid)
     n2 := t.rpcClient.newPayloadV2(sidechainHead)
-    n2.expectStatus(test.Valid)
+    n2.expectStatus(PayloadExecutionStatus.valid)
   } else if ws.AfterSync {
     # Spawn a secondary client which will need to sync to the primary client
     secondaryEngine, err := hive_rpc.HiveRPCEngineStarter{}.StartClient(t.T, t.TestContext, t.Genesis, t.ClientParams, t.ClientFiles, t.Engine)
@@ -207,10 +207,10 @@ proc execute*(ws: GetPayloadBodiesSpec, t: TestEnv): bool =
           &t.clMock.latestForkchoice,
           nil,
         )
-        if r.Response.PayloadStatus.Status == test.Valid {
+        if r.Response.PayloadStatus.Status == PayloadExecutionStatus.valid {
           break loop
         }
-        if r.Response.PayloadStatus.Status == test.Invalid {
+        if r.Response.PayloadStatus.Status == PayloadExecutionStatus.invalid {
           error "Syncing client rejected valid chain: %s", t.TestName, r.Response)
         }
       }

hive_integration/nodocker/engine/withdrawals/wd_reorg_spec.nim

@@ -168,7 +168,7 @@ proc execute*(ws: ReorgSpec, env: TestEnv): bool =
         sidechain.attr = some(attr)
         let r = sec.client.forkchoiceUpdated(fcState, attr)
         r.expectNoError()
-        r.testFCU(valid)
+        r.expectPayloadStatus(PayloadExecutionStatus.valid)
         testCond r.get().payloadID.isSome:
           error "Unable to get a payload ID on the sidechain"
         sidechain.payloadId = r.get().payloadID.get()
@@ -192,13 +192,13 @@ proc execute*(ws: ReorgSpec, env: TestEnv): bool =
         payload = env.clMock.latestPayloadBuilt
 
       let r = sec.client.newPayload(payload)
-      r.expectStatus(valid)
+      r.expectStatus(PayloadExecutionStatus.valid)
 
       let fcState = ForkchoiceStateV1(
         headBlockHash: payload.blockHash,
       )
       let p = sec.client.forkchoiceUpdated(payload.version, fcState)
-      p.testFCU(valid)
+      p.expectPayloadStatus(PayloadExecutionStatus.valid)
       return true
   ))
   testCond pbRes
@@ -234,19 +234,19 @@ proc execute*(ws: ReorgSpec, env: TestEnv): bool =
         )
 
       let r = sec.client.forkchoiceUpdatedV2(fcState, some(attr))
-      r.testFCU(valid)
+      r.expectPayloadStatus(PayloadExecutionStatus.valid)
 
       let p = sec.client.getPayloadV2(r.get().payloadID.get)
       p.expectNoError()
 
       let z = p.get()
       let s = sec.client.newPayloadV2(z.executionPayload)
-      s.expectStatus(valid)
+      s.expectStatus(PayloadExecutionStatus.valid)
 
       let fs = ForkchoiceStateV1(headBlockHash: z.executionPayload.blockHash)
 
       let q = sec.client.forkchoiceUpdatedV2(fs)
-      q.testFCU(valid)
+      q.expectPayloadStatus(PayloadExecutionStatus.valid)
 
       inc sidechain.height
       sidechain.sidechain[sidechain.height] = executionPayload(z.executionPayload)
@@ -279,9 +279,9 @@ proc execute*(ws: ReorgSpec, env: TestEnv): bool =
         error "Primary client invalidated side chain"
         return false
 
-      var header: common.BlockHeader
-      let b = env.client.latestHeader(header)
+      let b = env.client.latestHeader()
       testCond b.isOk
+      let header = b.get
       if header.blockHash == ethHash(sidehash):
         # sync successful
         break
@@ -303,11 +303,11 @@ proc execute*(ws: ReorgSpec, env: TestEnv): bool =
         parentHash=payload.parentHash.short
 
       let r = env.client.newPayload(payload)
-      r.expectStatusEither(valid, accepted)
+      r.expectStatusEither([PayloadExecutionStatus.valid, PayloadExecutionStatus.accepted])
 
       let fcState = ForkchoiceStateV1(headBlockHash: payload.blockHash)
       let p = env.client.forkchoiceUpdated(version, fcState)
-      p.testFCU(valid)
+      p.expectPayloadStatus(PayloadExecutionStatus.valid)
       inc payloadNumber
 
 
@@ -326,4 +326,4 @@ proc execute*(ws: ReorgSpec, env: TestEnv): bool =
   # Re-Org back to the canonical chain
   let fcState = ForkchoiceStateV1(headBlockHash: env.clMock.latestPayloadBuilt.blockHash)
   let r = env.client.forkchoiceUpdatedV2(fcState)
-  r.testFCU(valid)
+  r.expectPayloadStatus(PayloadExecutionStatus.valid)