more engine api tests

hive_integration/nodocker/engine/clmock.nim

@@ -21,8 +21,8 @@ type
     executedPayloadHistory*: Table[uint64, ExecutionPayloadV1]
 
     # Latest broadcasted data using the PoS Engine API
-    latestFinalizedNumber*: uint64
-    latestFinalizedHeader*: common.BlockHeader
+    latestHeadNumber*: uint64
+    latestHeader*: common.BlockHeader
     latestPayloadBuilt*   : ExecutionPayloadV1
     latestExecutedPayload*: ExecutionPayloadV1
     latestForkchoice*     : ForkchoiceStateV1
@@ -39,9 +39,9 @@ type
     onGetPayloadID*            : proc(): bool {.gcsafe.}
     onGetPayload*              : proc(): bool {.gcsafe.}
     onNewPayloadBroadcast*     : proc(): bool {.gcsafe.}
-    onHeadBlockForkchoiceBroadcast*      : proc(): bool {.gcsafe.}
-    onSafeBlockForkchoiceBroadcast*      : proc(): bool {.gcsafe.}
-    onFinalizedBlockForkchoiceBroadcast* : proc(): bool {.gcsafe.}
+    onForkchoiceBroadcast*     : proc(): bool {.gcsafe.}
+    onSafeBlockChange *        : proc(): bool {.gcsafe.}
+    onFinalizedBlockChange*    : proc(): bool {.gcsafe.}
 
 
 proc init*(cl: CLMocker, client: RpcClient, ttd: DifficultyInt) =
@@ -58,14 +58,14 @@ proc waitForTTD*(cl: CLMocker): Future[bool] {.async.} =
     error "timeout while waiting for TTD"
     return false
 
-  cl.latestFinalizedHeader = header
+  cl.latestHeader = header
   cl.ttdReached = true
 
-  let headerHash = BlockHash(common.blockHash(cl.latestFinalizedHeader).data)
+  let headerHash = BlockHash(common.blockHash(cl.latestHeader).data)
   cl.latestForkchoice.headBlockHash = headerHash
   cl.latestForkchoice.safeBlockHash = headerHash
   cl.latestForkchoice.finalizedBlockHash = headerHash
-  cl.latestFinalizedNumber = cl.latestFinalizedHeader.blockNumber.truncate(uint64)
+  cl.latestHeadNumber = cl.latestHeader.blockNumber.truncate(uint64)
 
   let res = cl.client.forkchoiceUpdatedV1(cl.latestForkchoice)
   if res.isErr:
@@ -87,7 +87,7 @@ proc pickNextPayloadProducer(cl: CLMocker): bool =
     return false
 
   let lastBlockNumber = nRes.get
-  if cl.latestFinalizedNumber != lastBlockNumber:
+  if cl.latestHeadNumber != lastBlockNumber:
     return false
 
   var header: common.BlockHeader
@@ -97,7 +97,7 @@ proc pickNextPayloadProducer(cl: CLMocker): bool =
     return false
 
   let lastBlockHash = header.blockHash
-  if cl.latestFinalizedHeader.blockHash != lastBlockHash:
+  if cl.latestHeader.blockHash != lastBlockHash:
     error "CLMocker: Failed to obtain a client on the latest block number"
     return false
 
@@ -108,7 +108,7 @@ proc getNextPayloadID(cl: CLMocker): bool =
   var nextPrevRandao: Hash256
   doAssert nimcrypto.randomBytes(nextPrevRandao.data) == 32
 
-  let timestamp = Quantity toUnix(cl.latestFinalizedHeader.timestamp + 1.seconds)
+  let timestamp = Quantity toUnix(cl.latestHeader.timestamp + 1.seconds)
   let payloadAttributes = PayloadAttributesV1(
     timestamp:             timestamp,
     prevRandao:            FixedBytes[32] nextPrevRandao.data,
@@ -116,7 +116,7 @@ proc getNextPayloadID(cl: CLMocker): bool =
   )
 
   # Save random value
-  let number = cl.latestFinalizedHeader.blockNumber.truncate(uint64) + 1
+  let number = cl.latestHeader.blockNumber.truncate(uint64) + 1
   cl.prevRandaoHistory[number] = nextPrevRandao
 
   let res = cl.client.forkchoiceUpdatedV1(cl.latestForkchoice, some(payloadAttributes))
@@ -261,8 +261,8 @@ proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe
   if not cl.broadcastLatestForkchoice():
     return false
 
-  if cb.onHeadBlockForkchoiceBroadcast != nil:
-    if not cb.onHeadBlockForkchoiceBroadcast():
+  if cb.onForkchoiceBroadcast != nil:
+    if not cb.onForkchoiceBroadcast():
       return false
 
   # Broadcast forkchoice updated with new SafeBlock to all clients
@@ -270,8 +270,8 @@ proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe
   if not cl.broadcastLatestForkchoice():
     return false
 
-  if cb.onSafeBlockForkchoiceBroadcast != nil:
-    if not cb.onSafeBlockForkchoiceBroadcast():
+  if cb.onSafeBlockChange != nil:
+    if not cb.onSafeBlockChange():
       return false
 
   # Broadcast forkchoice updated with new FinalizedBlock to all clients
@@ -281,15 +281,15 @@ proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe
 
   # Save the number of the first PoS block
   if cl.firstPoSBlockNumber.isNone:
-    let number = cl.latestFinalizedHeader.blockNumber.truncate(uint64) + 1
+    let number = cl.latestHeader.blockNumber.truncate(uint64) + 1
     cl.firstPoSBlockNumber = some(number)
 
   # Save the header of the latest block in the PoS chain
-  cl.latestFinalizedNumber = cl.latestFinalizedNumber + 1
+  cl.latestHeadNumber = cl.latestHeadNumber + 1
 
   # Check if any of the clients accepted the new payload
   var newHeader: common.BlockHeader
-  let res = cl.client.headerByNumber(cl.latestFinalizedNumber, newHeader)
+  let res = cl.client.headerByNumber(cl.latestHeadNumber, newHeader)
   if res.isErr:
     error "CLMock ProduceSingleBlock", msg=res.error
     return false
@@ -300,10 +300,10 @@ proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe
       hash=newHash.toHex
     return false
 
-  cl.latestFinalizedHeader = newHeader
+  cl.latestHeader = newHeader
 
-  if cb.onFinalizedBlockForkchoiceBroadcast != nil:
-    if not cb.onFinalizedBlockForkchoiceBroadcast():
+  if cb.onFinalizedBlockChange != nil:
+    if not cb.onFinalizedBlockChange():
       return false
 
   return true

hive_integration/nodocker/engine/engine_tests.nim

@@ -68,6 +68,9 @@ proc invalidGetPayloadUnderPoW(t: TestEnv): TestStatus =
   let res = t.rpcClient.getPayloadV1(id)
   testCond res.isErr
 
+  # Check that PoW chain progresses
+  testCond t.verifyPoWProgress(t.gHeader.blockHash)
+
 # Invalid Terminal Block in NewPayload:
 # Client must reject NewPayload directives if the referenced ParentHash does not meet the TTD requirement.
 proc invalidTerminalBlockNewPayload(t: TestEnv): TestStatus =
@@ -96,6 +99,9 @@ proc invalidTerminalBlockNewPayload(t: TestEnv): TestStatus =
   testCond s.status == PayloadExecutionStatus.invalid
   testCond s.latestValidHash.isNone
 
+  # Check that PoW chain progresses
+  testCond t.verifyPoWProgress(t.gHeader.blockHash)
+
 proc unknownHeadBlockHash(t: TestEnv): TestStatus =
   result = TestStatus.OK
 
@@ -208,6 +214,156 @@ proc unknownFinalizedBlockHash(t: TestEnv): TestStatus =
 
   testCond produceSingleBlockRes
 
+# Send an inconsistent ForkchoiceState with a known payload that belongs to a side chain as head, safe or finalized.
+type
+  Inconsistency {.pure.} = enum
+    Head
+    Safe
+    Finalized
+
+  PayloadList = ref object
+    canonicalPayloads  : seq[ExecutableData]
+    alternativePayloads: seq[ExecutableData]
+
+template inconsistentForkchoiceStateGen(procName: untyped, inconsistency: Inconsistency) =
+  proc procName(t: TestEnv): TestStatus =
+    result = TestStatus.OK
+
+    # Wait until TTD is reached by this client
+    let ok = waitFor t.clMock.waitForTTD()
+    testCond ok
+
+    var pList = PayloadList()
+    let clMock = t.clMock
+    let client = t.rpcClient
+
+    # Produce blocks before starting the test
+    let produceBlockRes = clMock.produceBlocks(3, BlockProcessCallbacks(
+      onGetPayload: proc(): bool =
+        # Generate and send an alternative side chain
+        var customData = CustomPayload(
+          extraData: some(@[0x01.byte])
+        )
+
+        if pList.alternativePayloads.len > 0:
+          customData.parentHash = some(pList.alternativePayloads[^1].blockHash)
+
+        let executableData = toExecutableData(clMock.latestPayloadBuilt)
+        let alternativePayload = customizePayload(executableData, customData)
+        pList.alternativePayloads.add(alternativePayload.toExecutableData)
+
+        let latestCanonicalPayload = toExecutableData(clMock.latestPayloadBuilt)
+        pList.canonicalPayloads.add(latestCanonicalPayload)
+
+        # Send the alternative payload
+        let res = client.newPayloadV1(alternativePayload)
+        if res.isErr:
+          return false
+
+        let s = res.get()
+        s.status == PayloadExecutionStatus.valid or s.status == PayloadExecutionStatus.accepted
+    ))
+
+    testCond produceBlockRes
+
+    # Send the invalid ForkchoiceStates
+    let len = pList.alternativePayloads.len
+    var inconsistentFcU = ForkchoiceStateV1(
+      headBlockHash:      Web3BlockHash pList.canonicalPayloads[len-1].blockHash.data,
+      safeBlockHash:      Web3BlockHash pList.canonicalPayloads[len-2].blockHash.data,
+      finalizedBlockHash: Web3BlockHash pList.canonicalPayloads[len-3].blockHash.data,
+    )
+
+    when inconsistency == Inconsistency.Head:
+      inconsistentFcU.headBlockHash = Web3BlockHash pList.alternativePayloads[len-1].blockHash.data
+    elif inconsistency == Inconsistency.Safe:
+      inconsistentFcU.safeBlockHash = Web3BlockHash pList.alternativePayloads[len-2].blockHash.data
+    else:
+      inconsistentFcU.finalizedBlockHash = Web3BlockHash pList.alternativePayloads[len-3].blockHash.data
+
+    var r = client.forkchoiceUpdatedV1(inconsistentFcU)
+    testCond r.isErr
+
+    # Return to the canonical chain
+    r = client.forkchoiceUpdatedV1(clMock.latestForkchoice)
+    testCond r.isOk
+    let s = r.get()
+    testCond s.payloadStatus.status == PayloadExecutionStatus.valid
+
+inconsistentForkchoiceStateGen(inconsistentForkchoiceState1, Inconsistency.Head)
+inconsistentForkchoiceStateGen(inconsistentForkchoiceState2, Inconsistency.Safe)
+inconsistentForkchoiceStateGen(inconsistentForkchoiceState3, Inconsistency.Finalized)
+
+# Verify behavior on a forkchoiceUpdated with invalid payload attributes
+template invalidPayloadAttributesGen(procName: untyped, syncingCond: bool) =
+  proc procName(t: TestEnv): TestStatus =
+    result = TestStatus.OK
+
+    # Wait until TTD is reached by this client
+    let ok = waitFor t.clMock.waitForTTD()
+    testCond ok
+
+    let clMock = t.clMock
+    let client = t.rpcClient
+
+    # Produce blocks before starting the test
+    var produceBlockRes = clMock.produceBlocks(5, BlockProcessCallbacks())
+    testCond produceBlockRes
+
+    # Send a forkchoiceUpdated with invalid PayloadAttributes
+    produceBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
+      onNewPayloadBroadcast: proc(): bool =
+        # Try to apply the new payload with invalid attributes
+        var blockHash: Hash256
+        when syncingCond:
+          # Setting a random hash will put the client into `SYNCING`
+          doAssert nimcrypto.randomBytes(blockHash.data) == 32
+        else:
+          # Set the block hash to the next payload that was broadcasted
+          blockHash = hash256(clMock.latestPayloadBuilt.blockHash)
+
+        let fcu = ForkchoiceStateV1(
+          headBlockHash:      Web3BlockHash blockHash.data,
+          safeBlockHash:      Web3BlockHash blockHash.data,
+          finalizedBlockHash: Web3BlockHash blockHash.data,
+        )
+
+        let attr = PayloadAttributesV1()
+
+        # 0) Check headBlock is known and there is no missing data, if not respond with SYNCING
+        # 1) Check headBlock is VALID, if not respond with INVALID
+        # 2) Apply forkchoiceState
+        # 3) Check payloadAttributes, if invalid respond with error: code: Invalid payload attributes
+        # 4) Start payload build process and respond with VALID
+        when syncingCond:
+          # If we are SYNCING, the outcome should be SYNCING regardless of the validity of the payload atttributes
+          let r = client.forkchoiceUpdatedV1(fcu, some(attr))
+          let s = r.get()
+          if s.payloadStatus.status != PayloadExecutionStatus.syncing:
+            return false
+          if s.payloadId.isSome:
+            return false
+        else:
+          let r = client.forkchoiceUpdatedV1(fcu, some(attr))
+          if r.isOk:
+            debugEcho "EEEE"
+            return false
+
+          # Check that the forkchoice was applied, regardless of the error
+          var header: EthBlockHeader
+          let s = client.latestHeader(header)
+          if s.isErr:
+            return false
+          if header.blockHash != blockHash:
+            return false
+        return true
+    ))
+
+    testCond produceBlockRes
+
+invalidPayloadAttributesGen(invalidPayloadAttributes1, false)
+invalidPayloadAttributesGen(invalidPayloadAttributes2, true)
+
 proc preTTDFinalizedBlockHash(t: TestEnv): TestStatus =
   result = TestStatus.OK
 
@@ -237,7 +393,38 @@ proc preTTDFinalizedBlockHash(t: TestEnv): TestStatus =
   let s = res.get()
   testCond s.payloadStatus.status == PayloadExecutionStatus.valid
 
-proc badHashOnExecPayload(t: TestEnv): TestStatus =
+# Corrupt the hash of a valid payload, client should reject the payload.
+# All possible scenarios:
+#    (fcU)
+# ┌────────┐        ┌────────────────────────┐
+# │  HEAD  │◄───────┤ Bad Hash (!Sync,!Side) │
+# └────┬───┘        └────────────────────────┘
+#    │
+#    │
+# ┌────▼───┐        ┌────────────────────────┐
+# │ HEAD-1 │◄───────┤ Bad Hash (!Sync, Side) │
+# └────┬───┘        └────────────────────────┘
+#    │
+#
+#
+#   (fcU)
+# ********************  ┌───────────────────────┐
+# *  (Unknown) HEAD  *◄─┤ Bad Hash (Sync,!Side) │
+# ********************  └───────────────────────┘
+#    │
+#    │
+# ┌────▼───┐            ┌───────────────────────┐
+# │ HEAD-1 │◄───────────┤ Bad Hash (Sync, Side) │
+# └────┬───┘            └───────────────────────┘
+#    │
+#
+
+type
+  Shadow = ref object
+    hash: Hash256
+
+template badHashOnNewPayloadGen(procName: untyped, syncingCond: bool, sideChain: bool) =
+  proc procName(t: TestEnv): TestStatus =
     result = TestStatus.OK
 
     let ok = waitFor t.clMock.waitForTTD()
@@ -247,10 +434,6 @@ proc badHashOnExecPayload(t: TestEnv): TestStatus =
     let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
     testCond produce5BlockRes
 
-  type
-    Shadow = ref object
-      hash: Hash256
-
     let clMock = t.clMock
     let client = t.rpcClient
     let shadow = Shadow()
@@ -265,13 +448,50 @@ proc badHashOnExecPayload(t: TestEnv): TestStatus =
         invalidPayloadHash.data[^1] = byte(not lastByte)
         shadow.hash = invalidPayloadHash
         alteredPayload.blockHash = BlockHash invalidPayloadHash.data
+
+        when not syncingCond and sideChain:
+          # We alter the payload by setting the parent to a known past block in the
+          # canonical chain, which makes this payload a side chain payload, and also an invalid block hash
+          # (because we did not update the block hash appropriately)
+          alteredPayload.parentHash = Web3BlockHash clMock.latestHeader.parentHash.data
+        elif syncingCond:
+          # We need to send an fcU to put the client in SYNCING state.
+          var randomHeadBlock: Hash256
+          doAssert nimcrypto.randomBytes(randomHeadBlock.data) == 32
+
+          let latestHeaderHash = clMock.latestHeader.blockHash
+          let fcU = ForkchoiceStateV1(
+            headBlockHash:      Web3BlockHash randomHeadBlock.data,
+            safeBlockHash:      Web3BlockHash latestHeaderHash.data,
+            finalizedBlockHash: Web3BlockHash latestHeaderHash.data
+          )
+
+          let r = client.forkchoiceUpdatedV1(fcU)
+          if r.isErr:
+            return false
+          let z = r.get()
+          if z.payloadStatus.status != PayloadExecutionStatus.syncing:
+            return false
+
+          when sidechain:
+            # Syncing and sidechain, the caonincal head is an unknown payload to us,
+            # but this specific bad hash payload is in theory part of a side chain.
+            # Therefore the parent we use is the head hash.
+            alteredPayload.parentHash = Web3BlockHash latestHeaderHash.data
+          else:
+            # The invalid bad-hash payload points to the unknown head, but we know it is
+            # indeed canonical because the head was set using forkchoiceUpdated.
+            alteredPayload.parentHash = Web3BlockHash randomHeadBlock.data
+
         let res = client.newPayloadV1(alteredPayload)
         # Execution specification::
         # - {status: INVALID_BLOCK_HASH, latestValidHash: null, validationError: null} if the blockHash validation has failed
         if res.isErr:
           return false
         let s = res.get()
-      s.status == PayloadExecutionStatus.invalid_block_hash
+        if s.status != PayloadExecutionStatus.invalid_block_hash:
+          return false
+        s.latestValidHash.isNone
     ))
     testCond produceSingleBlockRes
 
@@ -294,6 +514,11 @@ proc badHashOnExecPayload(t: TestEnv): TestStatus =
     ))
     testCond produceSingleBlockRes
 
+badHashOnNewPayloadGen(badHashOnNewPayload1, false, false)
+badHashOnNewPayloadGen(badHashOnNewPayload2, true, false)
+badHashOnNewPayloadGen(badHashOnNewPayload3, false, true)
+badHashOnNewPayloadGen(badHashOnNewPayload4, true, true)
+
 proc parentHashOnExecPayload(t: TestEnv): TestStatus =
   result = TestStatus.OK
 
@@ -328,20 +553,35 @@ proc invalidPayloadTestCaseGen(payloadField: string): proc (t: TestEnv): TestSta
     result = TestStatus.SKIPPED
 
 # Test to verify Block information available at the Eth RPC after NewPayload
-proc blockStatusExecPayload(t: TestEnv): TestStatus =
+template blockStatusExecPayloadGen(procName: untyped, transitionBlock: bool) =
+  proc procName(t: TestEnv): TestStatus =
     result = TestStatus.OK
 
     # Wait until TTD is reached by this client
     let ok = waitFor t.clMock.waitForTTD()
     testCond ok
 
-  # Produce blocks before starting the test
+    # Produce blocks before starting the test, only if we are not testing the transition block
+    when not transitionBlock:
       let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
       testCond produce5BlockRes
 
     let clMock = t.clMock
     let client = t.rpcClient
+    let shadow = Shadow()
+
     var produceSingleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
+      onPayloadProducerSelected: proc(): bool =
+        var address: EthAddress
+        let tx = t.makeNextTransaction(address, 1.u256)
+        let res = client.sendTransaction(tx)
+        if res.isErr:
+          error "Unable to send transaction"
+          return false
+
+        shadow.hash = rlpHash(tx)
+        return true
+      ,
       onNewPayloadBroadcast: proc(): bool =
         # TODO: Ideally, we would need to testCond that the newPayload returned VALID
         var lastHeader: EthBlockHeader
@@ -352,10 +592,7 @@ proc blockStatusExecPayload(t: TestEnv): TestStatus =
 
         let lastHash = BlockHash lastHeader.blockHash.data
         # Latest block header available via Eth RPC should not have changed at this point
-      if lastHash == clMock.latestExecutedPayload.blockHash or
-        lastHash != clMock.latestForkchoice.headBlockHash or
-        lastHash != clMock.latestForkchoice.safeBlockHash or
-        lastHash != clMock.latestForkchoice.finalizedBlockHash:
+        if lastHash!= clMock.latestForkchoice.headBlockHash:
           error "latest block header incorrect after newPayload", hash=lastHash.toHex
           return false
 
@@ -366,32 +603,56 @@ proc blockStatusExecPayload(t: TestEnv): TestStatus =
 
         # Latest block number available via Eth RPC should not have changed at this point
         let latestNumber = nRes.get
-      if latestNumber != clMock.latestFinalizedNumber:
+        if latestNumber != clMock.latestHeadNumber:
           error "latest block number incorrect after newPayload",
-          expected=clMock.latestFinalizedNumber,
+            expected=clMock.latestHeadNumber,
             get=latestNumber
           return false
 
+        # Check that the receipt for the transaction we just sent is still not available
+        let rr = client.txReceipt(shadow.hash)
+        if rr.isOk:
+          error "not expecting receipt"
+          return false
+
         return true
     ))
     testCond produceSingleBlockRes
 
-proc blockStatusHeadBlock(t: TestEnv): TestStatus =
+blockStatusExecPayloadGen(blockStatusExecPayload1, false)
+blockStatusExecPayloadGen(blockStatusExecPayload2, true)
+
+template blockStatusHeadBlockGen(procName: untyped, transitionBlock: bool) =
+  proc procName(t: TestEnv): TestStatus =
     result = TestStatus.OK
 
     # Wait until TTD is reached by this client
     let ok = waitFor t.clMock.waitForTTD()
     testCond ok
 
-  # Produce blocks before starting the test
+    # Produce blocks before starting the test, only if we are not testing the transition block
+    when not transitionBlock:
       let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
       testCond produce5BlockRes
 
     let clMock = t.clMock
     let client = t.rpcClient
+    let shadow = Shadow()
+
     var produceSingleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
+      onPayloadProducerSelected: proc(): bool =
+        var address: EthAddress
+        let tx = t.makeNextTransaction(address, 1.u256)
+        let res = client.sendTransaction(tx)
+        if res.isErr:
+          error "Unable to send transaction"
+          return false
+
+        shadow.hash = rlpHash(tx)
+        return true
+      ,
       # Run test after a forkchoice with new HeadBlockHash has been broadcasted
-    onHeadBlockForkchoiceBroadcast: proc(): bool =
+      onForkchoiceBroadcast: proc(): bool =
         var lastHeader: EthBlockHeader
         var hRes = client.latestHeader(lastHeader)
         if hRes.isErr:
@@ -399,31 +660,53 @@ proc blockStatusHeadBlock(t: TestEnv): TestStatus =
           return false
 
         let lastHash = BlockHash lastHeader.blockHash.data
-      if lastHash != clMock.latestForkchoice.headBlockHash or
-         lastHash == clMock.latestForkchoice.safeBlockHash or
-         lastHash == clMock.latestForkchoice.finalizedBlockHash:
+        if lastHash != clMock.latestForkchoice.headBlockHash:
           error "latest block header doesn't match HeadBlock hash", hash=lastHash.toHex
           return false
+
+        let rr = client.txReceipt(shadow.hash)
+        if rr.isErr:
+          error "unable to get transaction receipt"
+          return false
+
         return true
     ))
     testCond produceSingleBlockRes
 
-proc blockStatusSafeBlock(t: TestEnv): TestStatus =
+blockStatusHeadBlockGen(blockStatusHeadBlock1, false)
+blockStatusHeadBlockGen(blockStatusHeadBlock2, true)
+
+template blockStatusSafeBlockGen(procName: untyped, transitionBlock: bool) =
+  proc procName(t: TestEnv): TestStatus =
     result = TestStatus.OK
 
     # Wait until TTD is reached by this client
     let ok = waitFor t.clMock.waitForTTD()
     testCond ok
 
-  # Produce blocks before starting the test
+    # Produce blocks before starting the test, only if we are not testing the transition block
+    when not transitionBlock:
       let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
       testCond produce5BlockRes
 
     let clMock = t.clMock
     let client = t.rpcClient
+    let shadow = Shadow()
+
     var produceSingleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
+      onPayloadProducerSelected: proc(): bool =
+        var address: EthAddress
+        let tx = t.makeNextTransaction(address, 1.u256)
+        let res = client.sendTransaction(tx)
+        if res.isErr:
+          error "Unable to send transaction"
+          return false
+
+        shadow.hash = rlpHash(tx)
+        return true
+      ,
       # Run test after a forkchoice with new HeadBlockHash has been broadcasted
-    onSafeBlockForkchoiceBroadcast: proc(): bool =
+      onSafeBlockChange: proc(): bool =
         var lastHeader: EthBlockHeader
         var hRes = client.latestHeader(lastHeader)
         if hRes.isErr:
@@ -431,31 +714,52 @@ proc blockStatusSafeBlock(t: TestEnv): TestStatus =
           return false
 
         let lastHash = BlockHash lastHeader.blockHash.data
-      if lastHash != clMock.latestForkchoice.headBlockHash or
-         lastHash != clMock.latestForkchoice.safeBlockHash or
-         lastHash == clMock.latestForkchoice.finalizedBlockHash:
+        if lastHash != clMock.latestForkchoice.headBlockHash:
           error "latest block header doesn't match SafeBlock hash", hash=lastHash.toHex
           return false
+
+        let rr = client.txReceipt(shadow.hash)
+        if rr.isErr:
+          error "unable to get transaction receipt"
+          return false
         return true
     ))
     testCond produceSingleBlockRes
 
-proc blockStatusFinalizedBlock(t: TestEnv): TestStatus =
+blockStatusSafeBlockGen(blockStatusSafeBlock1, false)
+blockStatusSafeBlockGen(blockStatusSafeBlock2, true)
+
+template blockStatusFinalizedBlockGen(procName: untyped, transitionBlock: bool) =
+  proc procName(t: TestEnv): TestStatus =
     result = TestStatus.OK
 
     # Wait until TTD is reached by this client
     let ok = waitFor t.clMock.waitForTTD()
     testCond ok
 
-  # Produce blocks before starting the test
+    # Produce blocks before starting the test, only if we are not testing the transition block
+    when not transitionBlock:
       let produce5BlockRes = t.clMock.produceBlocks(5, BlockProcessCallbacks())
       testCond produce5BlockRes
 
     let clMock = t.clMock
     let client = t.rpcClient
+    let shadow = Shadow()
+
     var produceSingleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
+      onPayloadProducerSelected: proc(): bool =
+        var address: EthAddress
+        let tx = t.makeNextTransaction(address, 1.u256)
+        let res = client.sendTransaction(tx)
+        if res.isErr:
+          error "Unable to send transaction"
+          return false
+
+        shadow.hash = rlpHash(tx)
+        return true
+      ,
       # Run test after a forkchoice with new HeadBlockHash has been broadcasted
-    onFinalizedBlockForkchoiceBroadcast: proc(): bool =
+      onFinalizedBlockChange: proc(): bool =
         var lastHeader: EthBlockHeader
         var hRes = client.latestHeader(lastHeader)
         if hRes.isErr:
@@ -463,15 +767,21 @@ proc blockStatusFinalizedBlock(t: TestEnv): TestStatus =
           return false
 
         let lastHash = BlockHash lastHeader.blockHash.data
-      if lastHash != clMock.latestForkchoice.headBlockHash or
-         lastHash != clMock.latestForkchoice.safeBlockHash or
-         lastHash != clMock.latestForkchoice.finalizedBlockHash:
+        if lastHash != clMock.latestForkchoice.headBlockHash:
           error "latest block header doesn't match FinalizedBlock hash", hash=lastHash.toHex
           return false
+
+        let rr = client.txReceipt(shadow.hash)
+        if rr.isErr:
+          error "unable to get transaction receipt"
+          return false
         return true
     ))
     testCond produceSingleBlockRes
 
+blockStatusFinalizedBlockGen(blockStatusFinalizedBlock1, false)
+blockStatusFinalizedBlockGen(blockStatusFinalizedBlock2, true)
+
 proc blockStatusReorg(t: TestEnv): TestStatus =
   result = TestStatus.OK
 
@@ -487,7 +797,7 @@ proc blockStatusReorg(t: TestEnv): TestStatus =
   let client = t.rpcClient
   var produceSingleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
     # Run test after a forkchoice with new HeadBlockHash has been broadcasted
-    onHeadBlockForkchoiceBroadcast: proc(): bool =
+    onForkchoiceBroadcast: proc(): bool =
       # Verify the client is serving the latest HeadBlock
       var currHeader: EthBlockHeader
       var hRes = client.latestHeader(currHeader)
@@ -649,7 +959,7 @@ proc multipleNewCanonicalPayloads(t: TestEnv): TestStatus =
           return false
       return true
   ))
-  # At the end the CLMocker continues to try to execute fcU with the original payload, which should not fail
+  # At the end the clMocker continues to try to execute fcU with the original payload, which should not fail
   testCond produceSingleBlockRes
 
 proc outOfOrderPayloads(t: TestEnv): TestStatus =
@@ -674,7 +984,7 @@ proc outOfOrderPayloads(t: TestEnv): TestStatus =
   let clMock = t.clMock
   let client = t.rpcClient
   var produceBlockRes = clMock.produceBlocks(payloadCount, BlockProcessCallbacks(
-    # We send the transactions after we got the Payload ID, before the CLMocker gets the prepared Payload
+    # We send the transactions after we got the Payload ID, before the clMocker gets the prepared Payload
     onPayloadProducerSelected: proc(): bool =
       for i in 0..<txPerPayload:
         let tx = t.makeNextTransaction(recipient, amountPerTx)
@@ -698,6 +1008,121 @@ proc outOfOrderPayloads(t: TestEnv): TestStatus =
 
   # TODO: this section need multiple client
 
+# Test that performing a re-org back into a previous block of the canonical chain does not produce errors and the chain
+# is still capable of progressing.
+proc reorgBack(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
+  # Wait until TTD is reached by this client
+  let ok = waitFor t.clMock.waitForTTD()
+  testCond ok
+
+  let clMock = t.clMock
+  let client = t.rpcClient
+
+  let r1 = clMock.produceSingleBlock(BlockProcessCallbacks())
+  testCond r1
+
+  # We are going to reorg back to this previous hash several times
+  let previousHash = clMock.latestForkchoice.headBlockHash
+
+  # Produce blocks before starting the test (So we don't try to reorg back to the genesis block)
+  let r2 = clMock.produceBlocks(5, BlockProcessCallbacks(
+    onForkchoiceBroadcast: proc(): bool =
+      # Send a fcU with the HeadBlockHash pointing back to the previous block
+      let forkchoiceUpdatedBack = ForkchoiceStateV1(
+        headBlockHash:      previousHash,
+        safeBlockHash:      previousHash,
+        finalizedBlockHash: previousHash,
+      )
+
+      # It is only expected that the client does not produce an error and the CL Mocker is able to progress after the re-org
+      let r = client.forkchoiceUpdatedV1(forkchoiceUpdatedBack)
+      r.isOk
+  ))
+
+  testCond r2
+
+  # Verify that the client is pointing to the latest payload sent
+  var header: EthBlockHeader
+  let r = client.latestHeader(header)
+  testCond r.isOk
+  let blockHash = hash256(clMock.latestPayloadBuilt.blockHash)
+  testCond blockHash == header.blockHash
+
+# Test that performs a re-org back to the canonical chain after re-org to syncing/unavailable chain.
+type
+  SideChainList = ref object
+    sidechainPayloads: seq[ExecutionPayloadV1]
+
+proc reorgBackFromSyncing(t: TestEnv): TestStatus =
+  result = TestStatus.OK
+
+  # Wait until TTD is reached by this client
+  let ok = waitFor t.clMock.waitForTTD()
+  testCond ok
+
+  # Produce an alternative chain
+  let pList = SideChainList()
+  let clMock = t.clMock
+  let client = t.rpcClient
+
+  let r1 = clMock.produceBlocks(10, BlockProcessCallbacks(
+    onGetPayload: proc(): bool =
+      # Generate an alternative payload by simply adding extraData to the block
+      var altParentHash = clMock.latestPayloadBuilt.parentHash
+
+      if pList.sidechainPayloads.len > 0:
+        altParentHash = pList.sidechainPayloads[^1].blockHash
+
+      let executableData = toExecutableData(clMock.latestPayloadBuilt)
+      let altPayload = customizePayload(executableData,
+        CustomPayload(
+          parentHash: some(altParentHash.hash256),
+          extraData:  some(@[0x01.byte]),
+        ))
+
+      pList.sidechainPayloads.add(altPayload)
+      return true
+  ))
+
+  testCond r1
+
+
+  # Produce blocks before starting the test (So we don't try to reorg back to the genesis block)
+  let r2= clMock.produceSingleBlock(BlockProcessCallbacks(
+    onGetPayload: proc(): bool =
+      let r = client.newPayloadV1(pList.sidechainPayloads[^1])
+      if r.isErr:
+        return false
+      let s = r.get()
+      if s.status notin {PayloadExecutionStatus.syncing, PayloadExecutionStatus.accepted}:
+        return false
+
+      # We are going to send one of the alternative payloads and fcU to it
+      let len = pList.sidechainPayloads.len
+      let forkchoiceUpdatedBack = ForkchoiceStateV1(
+        headBlockHash:      pList.sidechainPayloads[len-1].blockHash,
+        safeBlockHash:      pList.sidechainPayloads[len-2].blockHash,
+        finalizedBlockHash: pList.sidechainPayloads[len-3].blockHash,
+      )
+
+      # It is only expected that the client does not produce an error and the CL Mocker is able to progress after the re-org
+      let res = client.forkchoiceUpdatedV1(forkchoiceUpdatedBack)
+      if res.isErr:
+        return false
+
+      let rs = res.get()
+      if rs.payloadStatus.status != PayloadExecutionStatus.syncing:
+        return false
+
+      rs.payloadStatus.latestValidHash.isNone
+      # After this, the clMocker will continue and try to re-org to canonical chain once again
+      # clMocker will fail the test if this is not possible, so nothing left to do.
+  ))
+
+  testCond r2
+
 proc transactionReorg(t: TestEnv): TestStatus =
   result = TestStatus.OK
 
@@ -846,12 +1271,12 @@ proc sidechainReorg(t: TestEnv): TestStatus =
 
   let singleBlockRes = clMock.produceSingleBlock(BlockProcessCallbacks(
     onNewPayloadBroadcast: proc(): bool =
-      # At this point the CLMocker has a payload that will result in a specific outcome,
+      # At this point the clMocker has a payload that will result in a specific outcome,
       # we can produce an alternative payload, send it, fcU to it, and verify the changes
       var alternativePrevRandao: Hash256
       doAssert nimcrypto.randomBytes(alternativePrevRandao.data) == 32
 
-      let timestamp = Quantity toUnix(clMock.latestFinalizedHeader.timestamp + 1.seconds)
+      let timestamp = Quantity toUnix(clMock.latestHeader.timestamp + 1.seconds)
       let payloadAttributes = PayloadAttributesV1(
         timestamp:             timestamp,
         prevRandao:            FixedBytes[32] alternativePrevRandao.data,
@@ -910,9 +1335,9 @@ proc sidechainReorg(t: TestEnv): TestStatus =
   ))
 
   testCond singleBlockRes
-  # The reorg actually happens after the CLMocker continues,
+  # The reorg actually happens after the clMocker continues,
   # verify here that the reorg was successful
-  let latestBlockNum = cLMock.latestFinalizedNumber.uint64
+  let latestBlockNum = clMock.latestHeadNumber.uint64
   testCond testCondPrevRandaoValue(t, clMock.prevRandaoHistory[latestBlockNum], latestBlockNum)
 
 proc suggestedFeeRecipient(t: TestEnv): TestStatus =
@@ -1054,7 +1479,8 @@ proc postMergeSync(t: TestEnv): TestStatus =
   # TODO: need multiple client
 
 const engineTestList* = [
-  #[TestSpec(
+  # Engine API Negative Test Cases
+  TestSpec(
     name: "Invalid Terminal Block in ForkchoiceUpdated",
     run: invalidTerminalBlockForkchoiceUpdated,
     ttd: 1000000
@@ -1069,6 +1495,18 @@ const engineTestList* = [
     run:  invalidTerminalBlockNewPayload,
     ttd:  1000000,
   ),
+  TestSpec(
+    name: "Inconsistent Head in ForkchoiceState",
+    run:  inconsistentForkchoiceState1,
+  ),
+  TestSpec(
+    name: "Inconsistent Safe in ForkchoiceState",
+    run:  inconsistentForkchoiceState2,
+  ),
+  TestSpec(
+    name: "Inconsistent Finalized in ForkchoiceState",
+    run:  inconsistentForkchoiceState3,
+  ),
   TestSpec(
     name: "Unknown HeadBlockHash",
     run:  unknownHeadBlockHash,
@@ -1081,14 +1519,35 @@ const engineTestList* = [
     name: "Unknown FinalizedBlockHash",
     run:  unknownFinalizedBlockHash,
   ),
+  TestSpec(
+    name: "ForkchoiceUpdated Invalid Payload Attributes",
+    run:  invalidPayloadAttributes1,
+  ),
+  TestSpec(
+    name: "ForkchoiceUpdated Invalid Payload Attributes (Syncing)",
+    run:  invalidPayloadAttributes2,
+  ),
   TestSpec(
     name: "Pre-TTD ForkchoiceUpdated After PoS Switch",
     run:  preTTDFinalizedBlockHash,
     ttd:  2,
   ),
+  # Invalid Payload Tests
   TestSpec(
     name: "Bad Hash on NewPayload",
-    run:  badHashOnExecPayload,
+    run:  badHashOnNewPayload1,
+  ),
+  TestSpec(
+    name: "Bad Hash on NewPayload Syncing",
+    run:  badHashOnNewPayload2,
+  ),
+  TestSpec(
+    name: "Bad Hash on NewPayload Side Chain",
+    run:  badHashOnNewPayload3,
+  ),
+  TestSpec(
+    name: "Bad Hash on NewPayload Side Chain Syncing",
+    run:  badHashOnNewPayload4,
   ),
   TestSpec(
     name: "ParentHash==BlockHash on NewPayload",
@@ -1152,21 +1611,42 @@ const engineTestList* = [
   ),
 
   # Eth RPC Status on ForkchoiceUpdated Events
+
   TestSpec(
     name: "Latest Block after NewPayload",
-    run:  blockStatusExecPayload,
+    run:  blockStatusExecPayload1,
+  ),
+  TestSpec(
+    name: "Latest Block after NewPayload (Transition Block)",
+    run:  blockStatusExecPayload2,
+    ttd:  5,
   ),
   TestSpec(
     name: "Latest Block after New HeadBlock",
-    run:  blockStatusHeadBlock,
+    run:  blockStatusHeadBlock1,
+  ),
+  TestSpec(
+    name: "Latest Block after New HeadBlock (Transition Block)",
+    run:  blockStatusHeadBlock2,
+    ttd:  5,
   ),
   TestSpec(
     name: "Latest Block after New SafeBlock",
-    run:  blockStatusSafeBlock,
+    run:  blockStatusSafeBlock1,
+  ),
+  TestSpec(
+    name: "Latest Block after New SafeBlock (Transition Block)",
+    run:  blockStatusSafeBlock2,
+    ttd:  5,
   ),
   TestSpec(
     name: "Latest Block after New FinalizedBlock",
-    run:  blockStatusFinalizedBlock,
+    run:  blockStatusFinalizedBlock1,
+  ),
+  TestSpec(
+    name: "Latest Block after New FinalizedBlock (Transition Block)",
+    run:  blockStatusFinalizedBlock2,
+    ttd:  5,
   ),
   TestSpec(
     name: "Latest Block after Reorg",
@@ -1196,12 +1676,20 @@ const engineTestList* = [
     name: "Sidechain Reorg",
     run:  sidechainReorg,
   ),
+  TestSpec(
+    name: "Re-Org Back into Canonical Chain",
+    run:  reorgBack,
+  ),
+  TestSpec(
+    name: "Re-Org Back to Canonical Chain From Syncing Chain",
+    run:  reorgBackFromSyncing,
+  ),
 
   # Suggested Fee Recipient in Payload creation
   TestSpec(
     name: "Suggested Fee Recipient Test",
     run:  suggestedFeeRecipient,
-  ),]#
+  ),
 
   # TODO: debug and fix
   # PrevRandao opcode tests

nimbus/rpc/engine_api.nim

@@ -300,7 +300,7 @@ proc setupEngineAPI*(
 
       if res.isErr:
         error "Failed to create sealing payload", err = res.error
-        return simpleFCU(PayloadExecutionStatus.invalid, res.error)
+        raise (ref InvalidRequest)(code: engineApiInvalidPayloadAttributes, msg: res.error)
 
       let id = computePayloadId(blockHash, payloadAttrs)
       api.put(id, payload)