Engine api simulator: fix payload customizer (#2143)

* Engine api simulator: fix payload customizer

* Move versionedHashes validation before blockHash validation in newPayload

* More descriptive error message

* Fix Web3Hash to eth.Hash256 conversion

hive_integration/nodocker/engine/cancun/customizer.nim

@@ -336,8 +336,7 @@ func getTimestamp*(cust: CustomPayloadData, basePayload: ExecutionPayload): uint
 # Construct a customized payload by taking an existing payload as base and mixing it CustomPayloadData
 # blockHash is calculated automatically.
 proc customizePayload*(cust: CustomPayloadData, data: ExecutableData): ExecutableData {.gcsafe.} =
-  var customHeader = blockHeader(data.basePayload, data.beaconRoot)
-
+  var customHeader = blockHeader(data.basePayload, removeBlobs = false, beaconRoot = data.beaconRoot)
   if cust.transactions.isSome:
     customHeader.txRoot = calcTxRoot(cust.transactions.get)
 

hive_integration/nodocker/engine/clmock.nim

@@ -333,7 +333,7 @@ proc getNextPayload(cl: CLMocker): bool =
   cl.latestShouldOverrideBuilder = x.shouldOverrideBuilder
 
   let beaconRoot = ethHash cl.latestPayloadAttributes.parentBeaconblockRoot
-  let header = blockHeader(cl.latestPayloadBuilt, beaconRoot)
+  let header = blockHeader(cl.latestPayloadBuilt, removeBlobs = true, beaconRoot = beaconRoot)
   let blockHash = w3Hash header.blockHash
   if blockHash != cl.latestPayloadBuilt.blockHash:
     error "CLMocker: getNextPayload blockHash mismatch",
@@ -412,7 +412,7 @@ proc broadcastNextNewPayload(cl: CLMocker): bool =
         return false
 
       let latestValidHash = s.latestValidHash.get()
-      if latestValidHash != BlockHash(blockHash):
+      if latestValidHash != blockHash:
         error "CLMocker: NewPayload returned VALID status with incorrect LatestValidHash",
           get=latestValidHash.toHex, expected=blockHash.toHex
         return false
@@ -424,7 +424,7 @@ proc broadcastNextNewPayload(cl: CLMocker): bool =
       # the blockHash of the payload is valid
       # the payload doesn't extend the canonical chain
       # the payload hasn't been fully validated.
-      let nullHash = BlockHash common.Hash256().data
+      let nullHash = w3Hash common.Hash256()
       let latestValidHash = s.latestValidHash.get(nullHash)
       if s.latestValidHash.isSome and latestValidHash != nullHash:
         error "CLMocker: NewPayload returned ACCEPTED status with incorrect LatestValidHash",
@@ -433,7 +433,8 @@ proc broadcastNextNewPayload(cl: CLMocker): bool =
 
     else:
       error "CLMocker: broadcastNewPayload Response",
-        status=s.status
+        status=s.status,
+        msg=s.validationError.get("NO MSG")
       return false
 
   cl.latestExecutedPayload = cl.latestPayloadBuilt
@@ -460,7 +461,8 @@ proc broadcastForkchoiceUpdated*(cl: CLMocker,
     let s = res.get()
     if s.payloadStatus.status != PayloadExecutionStatus.valid:
       error "CLMocker: broadcastForkchoiceUpdated Response",
-        status=s.payloadStatus.status
+        status=s.payloadStatus.status,
+        msg=s.payloadStatus.validationError.get("NO MSG")
       return false
 
     if s.payloadStatus.latestValidHash.get != cl.latestForkchoice.headBlockHash:

hive_integration/nodocker/engine/engine/invalid_ancestor.nim

@@ -192,7 +192,7 @@ method getName(cs: InvalidMissingAncestorReOrgSyncTest): string =
     $cs.invalidField, $cs.emptyTransactions, $cs.reOrgFromCanonical, $cs.invalidIndex]
 
 proc executableDataToBlock(ex: ExecutableData): EthBlock =
-  ethBlock(ex.basePayload, true, ex.beaconRoot)
+  ethBlock(ex.basePayload, removeBlobs = true, beaconRoot = ex.beaconRoot)
 
 method execute(cs: InvalidMissingAncestorReOrgSyncTest, env: TestEnv): bool =
   var sec = env.addEngine(true, cs.reOrgFromCanonical)

hive_integration/nodocker/engine/types.nim

@@ -158,7 +158,8 @@ template expectLatestValidHash*(res: untyped, expectedHash: Web3Hash) =
   let s = res.get
   when s is PayloadStatusV1:
     testCond s.latestValidHash.isSome:
-      error "Expect latest valid hash isSome"
+      error "Expect latest valid hash isSome",
+        msg=s.validationError.get("NO MSG")
     testCond s.latestValidHash.get == expectedHash:
       error "latest valid hash mismatch",
         expect=expectedHash.short,
@@ -198,10 +199,13 @@ template expectStatusEither*(res: untyped, cond: openArray[PayloadExecutionStatu
   let s = res.get()
   when s is PayloadStatusV1:
     testCond s.status in cond:
-      error "Unexpected expectStatusEither status", expect=cond, get=s.status
+      error "Unexpected expectStatusEither status",
+        expect=cond, get=s.status, msg=s.validationError.get("NO MSG")
   else:
     testCond s.payloadStatus.status in cond:
-      error "Unexpected expectStatusEither status", expect=cond, get=s.payloadStatus.status
+      error "Unexpected expectStatusEither status",
+        expect=cond, get=s.payloadStatus.status,
+        msg=s.payloadStatus.validationError.get("NO MSG")
 
 template expectNoValidationError*(res: untyped) =
   testCond res.isOk:

nimbus/beacon/api_handler/api_newpayload.nim

@@ -78,7 +78,7 @@ template validatePayload(apiVersion, version, payload) =
       raise invalidParams("newPayload" & $apiVersion &
         "withdrawals is expected from execution payload")
 
-  if version >= Version.V3:
+  if apiVersion >= Version.V3 or version >= Version.V3:
     if payload.blobGasUsed.isNone:
       raise invalidParams("newPayload" & $apiVersion &
         "blobGasUsed is expected from execution payload")
@@ -86,7 +86,7 @@ template validatePayload(apiVersion, version, payload) =
       raise invalidParams("newPayload" & $apiVersion &
         "excessBlobGas is expected from execution payload")
 
-  if version >= Version.V4:
+  if apiVersion >= Version.V4 or version >= Version.V4:
     if payload.depositReceipts.isNone:
       raise invalidParams("newPayload" & $apiVersion &
         "depositReceipts is expected from execution payload")
@@ -119,18 +119,19 @@ proc newPayload*(ben: BeaconEngineRef,
   validateVersion(com, timestamp, version, apiVersion)
   validatePayload(apiVersion, version, payload)
 
-  var header = blockHeader(payload, ethHash beaconRoot)
-  let blockHash = ethHash payload.blockHash
-  header.validateBlockHash(blockHash, version).isOkOr:
-    return error
-
+  var header = blockHeader(payload, removeBlobs = true, beaconRoot = ethHash beaconRoot)
+  
   if apiVersion >= Version.V3:
     if versionedHashes.isNone:
       raise invalidParams("newPayload" & $apiVersion &
         " expect blobVersionedHashes but got none")
     if not validateVersionedHashed(payload, versionedHashes.get):
       return invalidStatus(header.parentHash, "invalid blob versionedHashes")
-
+    
+  let blockHash = ethHash payload.blockHash
+  header.validateBlockHash(blockHash, version).isOkOr:
+    return error
+        
   # If we already have the block locally, ignore the entire execution and just
   # return a fake success.
   if db.getBlockHeader(blockHash, header):

nimbus/beacon/payload_conv.nim

@@ -28,10 +28,10 @@ func wdRoot(x: Option[seq[WithdrawalV1]]): Option[common.Hash256]
     if x.isNone: none(common.Hash256)
     else: some(wdRoot x.get)
 
-func txRoot(list: openArray[Web3Tx]): common.Hash256
+func txRoot(list: openArray[Web3Tx], removeBlobs: bool): common.Hash256
              {.gcsafe, raises:[RlpError].} =
   {.noSideEffect.}:
-    calcTxRoot(ethTxs(list, removeBlobs = true))
+    calcTxRoot(ethTxs(list, removeBlobs))
 
 # ------------------------------------------------------------------------------
 # Public functions
@@ -80,14 +80,15 @@ func executionPayloadV1V2*(blk: EthBlock): ExecutionPayloadV1OrV2 =
   )
 
 func blockHeader*(p: ExecutionPayload,
-                     beaconRoot: Option[common.Hash256]):
+                  removeBlobs: bool,
+                  beaconRoot: Option[common.Hash256]):
                        common.BlockHeader {.gcsafe, raises:[CatchableError].} =
   common.BlockHeader(
     parentHash     : ethHash p.parentHash,
     ommersHash     : EMPTY_UNCLE_HASH,
     coinbase       : ethAddr p.feeRecipient,
     stateRoot      : ethHash p.stateRoot,
-    txRoot         : txRoot p.transactions,
+    txRoot         : txRoot(p.transactions, removeBlobs),
     receiptRoot    : ethHash p.receiptsRoot,
     bloom          : ethBloom p.logsBloom,
     difficulty     : 0.u256,
@@ -118,7 +119,7 @@ func ethBlock*(p: ExecutionPayload,
                beaconRoot: Option[common.Hash256]):
                  common.EthBlock {.gcsafe, raises:[CatchableError].} =
   common.Ethblock(
-    header     : blockHeader(p, beaconRoot),
+    header     : blockHeader(p, removeBlobs, beaconRoot),
     uncles     : @[],
     txs        : ethTxs(p.transactions, removeBlobs),
     withdrawals: ethWithdrawals p.withdrawals,

nimbus/beacon/web3_eth_conv.nim

@@ -191,7 +191,7 @@ func w3Hashes*(z: Option[seq[common.Hash256]]): Option[seq[Web3Hash]] =
   if z.isNone: none(seq[Web3Hash])
   else:
     let list = z.get
-    var v = newSeq[Web3Hash](list.len)
+    var v = newSeqOfCap[Web3Hash](list.len)
     for x in list:
       v.add Web3Hash x.data
     some(v)