Fix engine simulator and improve logging (#2188)

* Fix engine simulator and improve logging

* Fix engine simulator genesis loader

hive_integration/nodocker/engine/cancun/customizer.nim

@@ -540,7 +540,7 @@ func scramble(data: Web3Hash): Option[common.Hash256] =
 
 func scramble(data: common.Hash256): Option[common.Hash256] =
   var h = data
-  h.data[^1] = byte(255 - h.data[^1])
+  h.data[0] = byte(255 - h.data[0])
   some(h)
 
 # This function generates an invalid payload by taking a base payload and modifying the specified field such that it ends up being invalid.

hive_integration/nodocker/engine/cancun_tests.nim

@@ -1937,15 +1937,20 @@ proc makeCancunTest(): seq[EngineSpec] =
     txType  : some(TxEIP4844),
   )
 
+proc getGenesisProc(cs: BaseSpec, param: NetworkParams) =
+  getGenesis(param)
+
 proc filCancunTests(): seq[TestDesc] =
   result.add cancunTestListA
 
   let list = makeCancunTest()
   for x in list:
+    var z = x
+    z.getGenesisFn = getGenesisProc
     result.add TestDesc(
       name: x.getName(),
       run: executeEngineSpec,
-      spec: x,
+      spec: z,
     )
 
 let cancunTestList* = filCancunTests()

hive_integration/nodocker/engine/clmock.nim

@@ -68,7 +68,7 @@ type
     latestBlobsBundle*      : Option[BlobsBundleV1]
     latestShouldOverrideBuilder*: Option[bool]
     latestPayloadAttributes*: PayloadAttributes
-    latestExecutedPayload*  : ExecutionPayload
+    latestExecutedPayload*  : ExecutableData
     latestForkchoice*       : ForkchoiceStateV1
 
     # Merge related
@@ -139,9 +139,11 @@ proc newClMocker*(eng: EngineEnv, com: CommonRef): CLMocker =
 
 proc addEngine*(cl: CLMocker, eng: EngineEnv) =
   cl.clients.add eng
+  echo "CLMocker: Adding engine client ", eng.ID()
 
 proc removeEngine*(cl: CLMocker, eng: EngineEnv) =
   cl.clients.remove eng
+  echo "CLMocker: Removing engine client ", eng.ID()
 
 proc waitForTTD*(cl: CLMocker): Future[bool] {.async.} =
   let ttd = cl.com.ttd()
@@ -151,6 +153,8 @@ proc waitForTTD*(cl: CLMocker): Future[bool] {.async.} =
     error "CLMocker: timeout while waiting for TTD"
     return false
 
+  echo "CLMocker: TTD has been reached at block ", header.blockNumber
+
   cl.latestHeader = header
   cl.headerHistory[header.blockNumber.truncate(uint64)] = header
   cl.ttdReached = true
@@ -242,6 +246,8 @@ proc pickNextPayloadProducer(cl: CLMocker): bool =
     let id = (cl.latestHeadNumber.int + i) mod cl.clients.len
     cl.nextBlockProducer = cl.clients[id]
 
+    echo "CLMocker: Selected payload producer: ", cl.nextBlockProducer.ID()
+
     # Get latest header. Number and hash must coincide with our view of the chain,
     # and only then we can build on top of this client's chain
     let res = cl.nextBlockProducer.client.latestHeader()
@@ -401,6 +407,9 @@ proc broadcastNextNewPayload(cl: CLMocker): bool =
       return false
 
     let s = res.get()
+    echo "CLMocker: Executed payload on ", eng.ID(),
+      " ", s.status, " ", s.latestValidHash
+
     if s.status == PayloadExecutionStatus.valid:
       # The client is synced and the payload was immediately validated
       # https:#github.com/ethereum/execution-apis/blob/main/src/engine/specification.md:
@@ -437,7 +446,10 @@ proc broadcastNextNewPayload(cl: CLMocker): bool =
         msg=s.validationError.get("NO MSG")
       return false
 
-  cl.latestExecutedPayload = cl.latestPayloadBuilt
+  # warning: although latestExecutedPayload is taken from
+  # latestPayloadBuilt, but during the next round, it can be different
+
+  cl.latestExecutedPayload = cl.latestExecutableData()
   let number = uint64 cl.latestPayloadBuilt.blockNumber
   cl.executedPayloadHistory[number] = cl.latestPayloadBuilt
   return true
@@ -663,6 +675,9 @@ proc produceSingleBlock*(cl: CLMocker, cb: BlockProcessCallbacks): bool {.gcsafe
   cl.latestHeader = newHeader
   cl.headerHistory[cl.latestHeadNumber] = cl.latestHeader
 
+  echo "CLMocker: New block produced: number=", newHeader.blockNumber,
+    " hash=", newHeader.blockHash
+
   return true
 
 # Loop produce PoS blocks by using the Engine API

hive_integration/nodocker/engine/engine/invalid_payload.nim

@@ -41,7 +41,7 @@ method withMainFork(cs: InvalidPayloadTestCase, fork: EngineFork): BaseSpec =
   return res
 
 method getName(cs: InvalidPayloadTestCase): string =
-  var name = "Invalid " & $cs.invalidField & " NewPayload"
+  var name = "Invalid NewPayload, " & $cs.invalidField
   if cs.syncing:
     name.add " - syncing"
 
@@ -185,11 +185,12 @@ method execute(cs: InvalidPayloadTestCase, env: TestEnv): bool =
         s.expectPayloadStatus(PayloadExecutionStatus.syncing)
 
         # When we send the previous payload, the client must now be capable of determining that the invalid payload is actually invalid
-        let p = env.engine.client.newPayload(env.clMock.latestExecutedPayload)
+        let version = env.engine.version(env.clMock.latestExecutedPayload.timestamp)
+        let p = env.engine.client.newPayload(version, env.clMock.latestExecutedPayload)
+
         p.expectStatus(PayloadExecutionStatus.valid)
         p.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)
 
-
         # Another option here could be to send an fcU to the previous payload,
         # but this does not seem like something the CL would do.
         #s = env.engine.client.forkchoiceUpdated(ForkchoiceStateV1(
@@ -199,7 +200,6 @@ method execute(cs: InvalidPayloadTestCase, env: TestEnv): bool =
         #), nil)
         #s.expectPayloadStatus(Valid)
 
-
         let q = env.engine.client.newPayload(version, shadow.alteredPayload)
         if cs.invalidField == InvalidParentHash:
           # There is no invalid parentHash, if this value is incorrect,
@@ -238,11 +238,11 @@ method execute(cs: InvalidPayloadTestCase, env: TestEnv): bool =
 
   if cs.syncing:
     # Send the valid payload and its corresponding forkchoiceUpdated
-    let r = env.engine.client.newPayload(env.clMock.latestExecutedPayload)
+    let version = env.engine.version(env.clMock.latestExecutedPayload.timestamp)
+    let r = env.engine.client.newPayload(version, env.clMock.latestExecutedPayload)
     r.expectStatus(PayloadExecutionStatus.valid)
     r.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)
 
-    let version = env.engine.version(env.clMock.latestExecutedPayload.timestamp)
     let s = env.engine.client.forkchoiceUpdated(version, env.clMock.latestForkchoice)
     s.expectPayloadStatus(PayloadExecutionStatus.valid)
     s.expectLatestValidHash(env.clMock.latestExecutedPayload.blockHash)

hive_integration/nodocker/engine/engine_client.nim

@@ -229,6 +229,24 @@ proc newPayload*(client: RpcClient,
       versionedHashes,
       w3Hash beaconRoot.get)
 
+proc newPayload*(client: RpcClient,
+                 version: Version,
+                 payload: ExecutionPayload,
+                 beaconRoot = none(common.Hash256)): Result[PayloadStatusV1, string] =
+  case version
+  of Version.V1: return client.newPayloadV1(payload)
+  of Version.V2: return client.newPayloadV2(payload)
+  of Version.V3:    
+    let versionedHashes = collectBlobHashes(payload.transactions)
+    return client.newPayloadV3(payload,
+      some(versionedHashes),
+      w3Hash beaconRoot)
+  of Version.V4:
+    let versionedHashes = collectBlobHashes(payload.transactions)
+    return client.newPayloadV4(payload,
+      some(versionedHashes),
+      w3Hash beaconRoot)
+      
 proc newPayload*(client: RpcClient,
                  version: Version,
                  payload: ExecutableData): Result[PayloadStatusV1, string] =

hive_integration/nodocker/engine/engine_env.nim

@@ -190,7 +190,8 @@ proc connect*(env: EngineEnv, node: ENode) =
   waitFor env.node.connectToNode(node)
 
 func ID*(env: EngineEnv): string =
-  $env.node.listeningAddress
+  # $env.node.listeningAddress
+  $env.conf.httpPort
 
 proc peer*(env: EngineEnv): Peer =
   doAssert(env.node.numPeers > 0)

hive_integration/nodocker/engine/engine_tests.nim

@@ -54,7 +54,11 @@ proc executeEngineSpec*(ws: BaseSpec): bool =
     return true
 
   let conf = envConfig(forkConfig)
+  if ws.getGenesisFn.isNil.not:
+    ws.getGenesisFn(ws, conf.networkParams)
+  else:
     cs.getGenesis(conf.networkParams)
+    
   let env  = TestEnv.new(conf)
   env.engine.setRealTTD()
   env.setupCLMock()

hive_integration/nodocker/engine/types.nim

@@ -11,7 +11,7 @@
 import
   std/[options, typetraits, strutils],
   eth/common,
-  nimcrypto/sysrand,
+  nimcrypto/[sysrand, sha2],
   stew/[byteutils, endians2],
   web3/eth_api_types,
   web3/engine_api_types,
@@ -19,6 +19,8 @@ import
   ../../../nimbus/beacon/web3_eth_conv,
   ../../../nimbus/utils/utils
 
+from ../../../nimbus/common/chain_config import NetworkParams
+
 export
   execution_types,
   web3_eth_conv
@@ -44,6 +46,7 @@ type
     forkHeight*: int
     forkTime*: uint64
     previousForkTime*: uint64
+    getGenesisFn*: proc(cs: BaseSpec, param: NetworkParams)
 
   TestDesc* = object
     name* : string
@@ -83,7 +86,7 @@ func toHash*(x: UInt256): common.Hash256 =
 
 func timestampToBeaconRoot*(timestamp: Quantity): FixedBytes[32] =
   # Generates a deterministic hash from the timestamp
-  let h = keccakHash(timestamp.uint64.toBytesBE)
+  let h = sha2.sha256.digest(timestamp.uint64.toBytesBE)
   FixedBytes[32](h.data)
 
 proc randomBytes*(_: type common.Hash256): common.Hash256 =
@@ -316,6 +319,12 @@ func blockNumber*(x: ExecutableData): auto =
 func stateRoot*(x: ExecutableData): auto =
   x.basePayload.stateRoot
 
+func version*(x: ExecutableData): auto =
+  x.basePayload.version
+
+func V1V2*(x: ExecutableData): auto =
+  x.basePayload.V1V2
+
 proc `parentHash=`*(x: var ExecutableData, val: auto) =
   x.basePayload.parentHash = val
 

nimbus/beacon/api_handler/api_newpayload.nim

@@ -42,7 +42,7 @@ template validateVersion(com, timestamp, version, apiVersion) =
   if com.isPragueOrLater(timestamp):
     if version != Version.V4:
       raise invalidParams("if timestamp is Prague or later, " &
-        "payload must be ExecutionPayloadV4")
+        "payload must be ExecutionPayloadV4, got ExecutionPayload" & $version)
 
   if apiVersion == Version.V3:
     if not com.isCancunOrLater(timestamp):
@@ -51,12 +51,12 @@ template validateVersion(com, timestamp, version, apiVersion) =
   if com.isCancunOrLater(timestamp):
     if version != Version.V3:
       raise invalidParams("if timestamp is Cancun or later, " &
-        "payload must be ExecutionPayloadV3")
+        "payload must be ExecutionPayloadV3, got ExecutionPayload" & $version)
 
   elif com.isShanghaiOrLater(timestamp):
     if version != Version.V2:
       raise invalidParams("if timestamp is Shanghai or later, " &
-        "payload must be ExecutionPayloadV2")
+        "payload must be ExecutionPayloadV2, got ExecutionPayload" & $version)
 
   elif version != Version.V1:
     if com.syncReqRelaxV2:
@@ -64,7 +64,7 @@ template validateVersion(com, timestamp, version, apiVersion) =
       discard
     else:
       raise invalidParams("if timestamp is earlier than Shanghai, " &
-        "payload must be ExecutionPayloadV1")
+        "payload must be ExecutionPayloadV1, got ExecutionPayload" & $version)
 
   if apiVersion >= Version.V3:
     if version != apiVersion: