Fix test_blockchain_json and pyspec simulator for Cancun (#1805)

* Fix test_blockchain_json and pyspec simulator for Cancun

* Preserve applyDeletes comments

* Fix redefinition error

* Move test_macro to tests folder

hive_integration/nodocker/engine/engine_client.nim

@@ -143,15 +143,6 @@ proc newPayloadV3*(client: RpcClient,
   wrapTrySimpleRes:
     client.engine_newPayloadV3(payload, versionedHashes, parentBeaconBlockRoot)
 
-#proc newPayload*(client: RpcClient,
-#                 payload: ExecutionPayload,
-#                 version: Version):
-#                   Result[PayloadStatusV1, string] =
-#  if version == Version.V1:
-#    client.newPayloadV1(payload.V1)
-#  else:
-#    client.newPayloadV2(payload.V2)
-
 proc collectBlobHashes(list: openArray[Web3Tx]): seq[Web3Hash] =
   for w3tx in list:
     let tx = ethTx(w3Tx)
@@ -165,6 +156,9 @@ proc newPayload*(client: RpcClient,
   of Version.V1: return client.newPayloadV1(payload.V1)
   of Version.V2: return client.newPayloadV2(payload.V2)
   of Version.V3:
+    if beaconRoot.isNone:
+      # fallback
+      return client.newPayloadV2(payload.V2)
     let versionedHashes = collectBlobHashes(payload.transactions)
     return client.newPayloadV3(payload.V3,
       versionedHashes,

hive_integration/nodocker/pyspec/pyspec_sim.nim

@@ -25,8 +25,6 @@ import
 
 const
   baseFolder = "hive_integration/nodocker/pyspec"
-  #caseFolder = "tests/fixtures/eth_tests/EIPTests/Pyspecs/cancun"
-  caseFolder = baseFolder & "/testcases"
   supportedNetwork = [
     "Merge",
     "Shanghai",
@@ -37,17 +35,24 @@ const
 
 type
   Payload = object
+    badBlock: bool
     payload: ExecutionPayload
     beaconRoot: Option[common.Hash256]
 
-proc getPayload(node: JsonNode): Payload =
+proc getPayload(node: JsonNode): Payload  =
-  let
+  try:
-    rlpBytes = hexToSeqByte(node.getStr)
+    let
-    blk = rlp.decode(rlpBytes, EthBlock)
+      rlpBytes = hexToSeqByte(node.getStr)
-  Payload(
+      blk = rlp.decode(rlpBytes, EthBlock)
-    payload: executionPayload(blk),
+    Payload(
-    beaconRoot: blk.header.parentBeaconBlockRoot,
+      badBlock: false,
-  )
+      payload: executionPayload(blk),
+      beaconRoot: blk.header.parentBeaconBlockRoot,
+    )
+  except RlpError:
+    Payload(
+      badBlock: true,
+    )
 
 proc validatePostState(node: JsonNode, t: TestEnv): bool =
   # check nonce, balance & storage of accounts in final block against fixture values
@@ -125,8 +130,19 @@ proc runTest(node: JsonNode, network: string): TestStatus =
                            PayloadExecutionStatus.invalid
                          else:
                            PayloadExecutionStatus.valid
-    let payload = getPayload(blkNode["rlp"])
+    let
+      badBlock = blkNode.hasKey("expectException")
+      payload = getPayload(blkNode["rlp"])
+
+    if badBlock == payload.badBlock and badBlock == true:
+      # It could be the rlp decoding succeed, but the actual
+      # block validation is failed in engine api
+      # So, we skip newPayload call only if decoding is also
+      # failed
+      break
+
     latestVersion = payload.payload.version
+
     let res = t.rpcClient.newPayload(payload.payload, payload.beaconRoot)
     if res.isErr:
       result = TestStatus.Failed
@@ -165,6 +181,22 @@ proc runTest(node: JsonNode, network: string): TestStatus =
 
   t.stopELClient()
 
+const
+  skipName = [
+    "beacon_root_contract_timestamps.json",
+    "beacon_root_equal_to_timestamp.json",
+  ]
+
+  caseFolderCancun   = "tests/fixtures/eth_tests/EIPTests/Pyspecs/cancun"
+  caseFolderShanghai = baseFolder & "/testcases"
+
+proc collectTestVectors(): seq[string] =
+  for fileName in walkDirRec(caseFolderCancun):
+    result.add fileName
+
+  for fileName in walkDirRec(caseFolderShanghai):
+    result.add fileName
+
 proc main() =
   var stat: SimStat
   let start = getTime()
@@ -174,10 +206,18 @@ proc main() =
     echo "FATAL: ", res.error
     quit(QuitFailure)
 
-  for fileName in walkDirRec(caseFolder):
+  let testVectors = collectTestVectors()
+  for fileName in testVectors:
     if not fileName.endsWith(".json"):
       continue
 
+    let suspect = splitPath(fileName)
+    if suspect.tail in skipName:
+      let fixtureTests = json.parseFile(fileName)
+      for name, fixture in fixtureTests:
+        stat.inc(name, TestStatus.Skipped)
+      continue
+
     let fixtureTests = json.parseFile(fileName)
     for name, fixture in fixtureTests:
       let network = fixture["network"].getStr

hive_integration/nodocker/pyspec/test_env.nim

@@ -45,7 +45,7 @@ proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) =
       conf,
       t.conf.pruneMode == PruneMode.Full
     )
-  t.chainRef = newChain(t.com)
+  t.chainRef = newChain(t.com, extraValidation = true)
   let
     stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.pruneMode == PruneMode.Full)
     genesisHeader = node.genesisHeader

newBlockchainTests.md

@@ -496,11 +496,12 @@ OK: 23/23 Fail: 0/23 Skip: 0/23
 + gasLimitTooHigh2.json                                           OK
 + gasPrice0.json                                                  OK
 + log1_correct.json                                               OK
++ reentrencySuicide.json                                          OK
 + timeDiff12.json                                                 OK
 + timeDiff13.json                                                 OK
 + timeDiff14.json                                                 OK
 ```
-OK: 19/19 Fail: 0/19 Skip: 0/19
+OK: 20/20 Fail: 0/20 Skip: 0/20
 ## bcWalletTest
 ```diff
 + wallet2outOf3txs.json                                           OK
@@ -510,23 +511,65 @@ OK: 19/19 Fail: 0/19 Skip: 0/19
 + walletReorganizeOwners.json                                     OK
 ```
 OK: 5/5 Fail: 0/5 Skip: 0/5
-## eips
+## eip1344_chainid
+```diff
++ chainid.json                                                    OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
+## eip2930_access_list
+```diff
++ access_list.json                                                OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
+## eip3651_warm_coinbase
 ```diff
-+ initcode_limit_contract_creating_tx.json                        OK
-+ initcode_limit_contract_creating_tx_gas_usage.json              OK
-+ initcode_limit_create2_opcode.json                              OK
-+ initcode_limit_create_opcode.json                               OK
-+ push0.json                                                      OK
 + warm_coinbase_call_out_of_gas.json                              OK
 + warm_coinbase_gas_usage.json                                    OK
 ```
-OK: 7/7 Fail: 0/7 Skip: 0/7
-## example
-```diff
-+ access_list.json                                                OK
-+ yul.json                                                        OK
-```
 OK: 2/2 Fail: 0/2 Skip: 0/2
+## eip3855_push0
+```diff
++ push0_before_jumpdest.json                                      OK
++ push0_during_staticcall.json                                    OK
++ push0_fill_stack.json                                           OK
++ push0_gas_cost.json                                             OK
++ push0_key_sstore.json                                           OK
++ push0_stack_overflow.json                                       OK
++ push0_storage_overwrite.json                                    OK
+```
+OK: 7/7 Fail: 0/7 Skip: 0/7
+## eip3860_initcode
+```diff
++ contract_creating_tx.json                                       OK
++ create_opcode_initcode.json                                     OK
++ gas_usage.json                                                  OK
+```
+OK: 3/3 Fail: 0/3 Skip: 0/3
+## eip4895_withdrawals
+```diff
++ balance_within_block.json                                       OK
++ large_amount.json                                               OK
++ many_withdrawals.json                                           OK
++ multiple_withdrawals_same_address.json                          OK
++ newly_created_contract.json                                     OK
++ no_evm_execution.json                                           OK
++ self_destructing_account.json                                   OK
++ use_value_in_contract.json                                      OK
++ use_value_in_tx.json                                            OK
++ withdrawing_to_precompiles.json                                 OK
++ zero_amount.json                                                OK
+```
+OK: 11/11 Fail: 0/11 Skip: 0/11
+## opcodes
+```diff
++ dup.json                                                        OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
+## security
+```diff
++ tx_selfdestruct_balance_bug.json                                OK
+```
+OK: 1/1 Fail: 0/1 Skip: 0/1
 ## stArgsZeroOneBalance
 ```diff
 + addNonConst.json                                                OK
@@ -2921,6 +2964,10 @@ OK: 12/14 Fail: 0/14 Skip: 2/14
 + InternalCallHittingGasLimitSuccess.json                         OK
 + InternlCallStoreClearsOOG.json                                  OK
 + InternlCallStoreClearsSucces.json                               OK
++ NoSrcAccount.json                                               OK
++ NoSrcAccount1559.json                                           OK
++ NoSrcAccountCreate.json                                         OK
++ NoSrcAccountCreate1559.json                                     OK
 + Opcodes_TransactionInit.json                                    OK
 + OverflowGasRequire2.json                                        OK
 + PointAtInfinityECRecover.json                                   OK
@@ -2940,7 +2987,7 @@ OK: 12/14 Fail: 0/14 Skip: 2/14
 + TransactionToItself.json                                        OK
 + ValueOverflow.json                                              OK
 ```
-OK: 31/31 Fail: 0/31 Skip: 0/31
+OK: 35/35 Fail: 0/35 Skip: 0/35
 ## stTransitionTest
 ```diff
 + createNameRegistratorPerTxsAfter.json                           OK
@@ -3316,12 +3363,6 @@ OK: 133/133 Fail: 0/133 Skip: 0/133
 + ecmul_1-2_2_21000_96.json                                       OK
 ```
 OK: 130/130 Fail: 0/130 Skip: 0/130
-## vm
-```diff
-+ chain_id.json                                                   OK
-+ dup.json                                                        OK
-```
-OK: 2/2 Fail: 0/2 Skip: 0/2
 ## vmArithmeticTest
 ```diff
 + add.json                                                        OK
@@ -3410,20 +3451,11 @@ OK: 0/3 Fail: 0/3 Skip: 3/3
 + swap.json                                                       OK
 ```
 OK: 11/11 Fail: 0/11 Skip: 0/11
-## withdrawals
+## yul
 ```diff
-+ balance_within_block.json                                       OK
++ yul.json                                                        OK
-+ large_amount.json                                               OK
-+ many_withdrawals.json                                           OK
-+ multiple_withdrawals_same_address.json                          OK
-+ newly_created_contract.json                                     OK
-+ no_evm_execution.json                                           OK
-+ self_destructing_account.json                                   OK
-+ use_value_in_contract.json                                      OK
-+ use_value_in_tx.json                                            OK
-+ zero_amount.json                                                OK
 ```
-OK: 10/10 Fail: 0/10 Skip: 0/10
+OK: 1/1 Fail: 0/1 Skip: 0/1
 
 ---TOTAL---
-OK: 2934/3040 Fail: 0/3040 Skip: 106/3040
+OK: 2946/3052 Fail: 0/3052 Skip: 106/3052

nimbus/beacon/api_handler/api_newpayload.nim

@@ -8,7 +8,7 @@
 # those terms.
 
 import
-  std/[typetraits, times],
+  std/[times],
   eth/common,
   stew/results,
   ../web3_eth_conv,

nimbus/beacon/api_handler/api_utils.nim

@@ -163,6 +163,8 @@ proc latestValidHash*(db: CoreDbRef,
                       parent: common.BlockHeader,
                       ttd: DifficultyInt): common.Hash256
                        {.gcsafe, raises: [RlpError].} =
+  if parent.isGenesis:
+    return common.Hash256()
   let ptd = db.getScore(parent.parentHash)
   if ptd >= ttd:
     parent.blockHash

nimbus/beacon/payload_conv.nim

@@ -18,18 +18,18 @@ import
 # ------------------------------------------------------------------------------
 
 func wdRoot(list: openArray[WithdrawalV1]): common.Hash256
-             {.gcsafe, raises:[CatchableError].} =
+             {.gcsafe, raises:[].} =
   {.nosideEffect.}:
     calcWithdrawalsRoot(ethWithdrawals list)
 
 func wdRoot(x: Option[seq[WithdrawalV1]]): Option[common.Hash256]
-             {.gcsafe, raises:[CatchableError].} =
+             {.gcsafe, raises:[].} =
   {.nosideEffect.}:
     if x.isNone: none(common.Hash256)
     else: some(wdRoot x.get)
 
 func txRoot(list: openArray[Web3Tx]): common.Hash256
-             {.gcsafe, raises:[CatchableError].} =
+             {.gcsafe, raises:[RlpError].} =
   {.nosideEffect.}:
     calcTxRoot(ethTxs list)
 

nimbus/beacon/web3_eth_conv.nim

@@ -200,7 +200,7 @@ func w3Withdrawals*(x: Option[seq[common.Withdrawal]]):
   else: some(w3Withdrawals x.get)
 
 func w3Tx*(tx: common.Transaction): Web3Tx =
-  Web3Tx rlp.encode(tx.removeNetworkPayload)
+  Web3Tx rlp.encode(tx)
 
 func w3Txs*(list: openArray[common.Transaction]): seq[Web3Tx] =
   result = newSeqOfCap[Web3Tx](list.len)

nimbus/core/chain/chain_desc.nim

@@ -13,6 +13,7 @@
 import
   ../../common/common,
   ../../utils/utils,
+  ../../vm_types,
   ../pow,
   ../clique
 
@@ -37,39 +38,45 @@ type
       ## First block to when `extraValidation` will be applied (only
       ## effective if `extraValidation` is true.)
 
-# ------------------------------------------------------------------------------
+    vmState: BaseVMState
-# Private constructor helper
+      ## If it's not nil, block validation will use this
-# ------------------------------------------------------------------------------
+      ## If it's nil, a new vmState state will be created.
-
-proc initChain(c: ChainRef; com: CommonRef; extraValidation: bool) =
-  ## Constructor for the `Chain` descriptor object.
-  c.com = com
-
-  c.validateBlock = true
-  c.extraValidation = extraValidation
 
 # ------------------------------------------------------------------------------
 # Public constructors
 # ------------------------------------------------------------------------------
 
-proc newChain*(com: CommonRef, extraValidation: bool): ChainRef =
+proc newChain*(com: CommonRef,
+               extraValidation: bool, vmState = BaseVMState(nil)): ChainRef =
   ## Constructor for the `Chain` descriptor object.
   ## The argument `extraValidation` enables extra block
   ## chain validation if set `true`.
-  new result
+  ChainRef(
-  result.initChain(com, extraValidation)
+    com: com,
+    validateBlock: true,
+    extraValidation: extraValidation,
+    vmState: vmState,
+  )
 
 proc newChain*(com: CommonRef): ChainRef =
   ## Constructor for the `Chain` descriptor object. All sub-object descriptors
   ## are initialised with defaults. So is extra block chain validation
   ##  * `enabled` for PoA networks (such as Goerli)
   ##  * `disabled` for non-PaA networks
-  new result
+  let extraValidation = com.consensus in {ConsensusType.POA, ConsensusType.POS}
-  result.initChain(com, com.consensus == ConsensusType.POA)
+  ChainRef(
+    com: com,
+    validateBlock: true,
+    extraValidation: extraValidation,
+  )
 
 # ------------------------------------------------------------------------------
 # Public `Chain` getters
 # ------------------------------------------------------------------------------
+proc vmState*(c: ChainRef): BaseVMState =
+  ## Getter
+  c.vmState
+
 proc clique*(c: ChainRef): Clique =
   ## Getter
   c.com.poa

nimbus/core/chain/persist_blocks.nim

@@ -39,14 +39,27 @@ type
 # Private
 # ------------------------------------------------------------------------------
 
+proc getVmState(c: ChainRef, header: BlockHeader):
+                 Result[BaseVMState, void]
+                  {.gcsafe, raises: [CatchableError].} =
+  if c.vmState.isNil.not:
+    return ok(c.vmState)
+
+  let vmState = BaseVMState()
+  if not vmState.init(header, c.com):
+    debug "Cannot initialise VmState",
+      number = header.blockNumber
+    return err()
+  return ok(vmState)
+
 proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
                        bodies: openArray[BlockBody],
                        flags: PersistBlockFlags = {}): ValidationResult
                           # wildcard exception, wrapped below in public section
                           {.inline, raises: [CatchableError].} =
 
-  let transaction = c.db.beginTransaction()
+  let dbTx = c.db.beginTransaction()
-  defer: transaction.dispose()
+  defer: dbTx.dispose()
 
   var cliqueState = c.clique.cliqueSave
   defer: c.clique.cliqueRestore(cliqueState)
@@ -54,11 +67,7 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
   c.com.hardForkTransition(headers[0])
 
   # Note that `0 < headers.len`, assured when called from `persistBlocks()`
-  let vmState = BaseVMState()
+  let vmState = c.getVmState(headers[0]).valueOr:
-  if not vmState.init(headers[0], c.com):
-    debug "Cannot initialise VmState",
-      fromBlock = headers[0].blockNumber,
-      toBlock = headers[^1].blockNumber
     return ValidationResult.Error
 
   trace "Persisting blocks",
@@ -77,9 +86,22 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
         item = i
       return ValidationResult.Error
 
+    if c.validateBlock and c.extraValidation and
+       c.verifyFrom <= header.blockNumber:
+
+      if c.com.consensus != ConsensusType.POA:
+        let res = c.com.validateHeaderAndKinship(
+          header,
+          body,
+          checkSealOK = false) # TODO: how to checkseal from here
+        if res.isErr:
+          debug "block validation error",
+            msg = res.error
+          return ValidationResult.Error
+
     let
       validationResult = if c.validateBlock:
-                           vmState.processBlock(c.clique, header, body)
+                           vmState.processBlock(header, body)
                          else:
                            ValidationResult.OK
     when not defined(release):
@@ -105,15 +127,6 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
             blockNumber = header.blockNumber,
             msg = $rc.error
           return ValidationResult.Error
-      else:
-        let res = c.com.validateHeaderAndKinship(
-          header,
-          body,
-          checkSealOK = false) # TODO: how to checkseal from here
-        if res.isErr:
-          debug "block validation error",
-            msg = res.error
-          return ValidationResult.Error
 
     if NoPersistHeader notin flags:
       discard c.db.persistHeaderToDb(
@@ -133,7 +146,7 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
     # between eth_blockNumber and eth_syncing
     c.com.syncCurrent = header.blockNumber
 
-  transaction.commit()
+  dbTx.commit()
 
 # ------------------------------------------------------------------------------
 # Public `ChainDB` methods

nimbus/core/executor/process_block.nim

@@ -110,7 +110,7 @@ proc procBlkPreamble(vmState: BaseVMState;
 
 proc procBlkEpilogue(vmState: BaseVMState;
                      header: BlockHeader; body: BlockBody): bool
-    {.gcsafe, raises: [CatchableError].} =
+    {.gcsafe, raises: [].} =
   # Reward beneficiary
   vmState.mutateStateDB:
     if vmState.generateWitness:
@@ -147,18 +147,19 @@ proc procBlkEpilogue(vmState: BaseVMState;
 # Public functions
 # ------------------------------------------------------------------------------
 
-proc processBlockNotPoA*(
+proc processBlock*(
-    vmState: BaseVMState; ## Parent environment of header/body block
+    vmState: BaseVMState;  ## Parent environment of header/body block
-    header:  BlockHeader; ## Header/body block to add to the blockchain
+    header:  BlockHeader;  ## Header/body block to add to the blockchain
     body:    BlockBody): ValidationResult
     {.gcsafe, raises: [CatchableError].} =
-  ## Processes `(header,body)` pair for a non-PoA network, only. This function
+  ## Generalised function to processes `(header,body)` pair for any network,
-  ## will fail when applied to a PoA network like `Goerli`.
+  ## regardless of PoA or not.
-  if vmState.com.consensus == ConsensusType.POA:
+  ##
-    # PoA consensus engine unsupported, see the other version of
+  ## Rather than calculating the PoA state change here, it is done with the
-    # processBlock() below
+  ## verification in the `chain/persist_blocks.persistBlocks()` method. So
-    debug "Unsupported PoA request"
+  ## the `poa` descriptor is currently unused and only provided for later
-    return ValidationResult.Error
+  ## implementations (but can be savely removed, as well.)
+  ## variant of `processBlock()` where the `header` argument is explicitely set.
 
   var dbTx = vmState.com.db.beginTransaction()
   defer: dbTx.dispose()
@@ -182,47 +183,6 @@ proc processBlockNotPoA*(
 
   ValidationResult.OK
 
-
-proc processBlock*(
-    vmState: BaseVMState;  ## Parent environment of header/body block
-    poa:     Clique;       ## PoA descriptor (if needed, at all)
-    header:  BlockHeader;  ## Header/body block to add to the blockchain
-    body:    BlockBody): ValidationResult
-    {.gcsafe, raises: [CatchableError].} =
-  ## Generalised function to processes `(header,body)` pair for any network,
-  ## regardless of PoA or not. Currently there is no mining support so this
-  ## function is mostly the same as `processBlockNotPoA()`.
-  ##
-  ## Rather than calculating the PoA state change here, it is done with the
-  ## verification in the `chain/persist_blocks.persistBlocks()` method. So
-  ## the `poa` descriptor is currently unused and only provided for later
-  ## implementations (but can be savely removed, as well.)
-  ## variant of `processBlock()` where the `header` argument is explicitely set.
-  ##
-  # # Process PoA state transition first so there is no need to re-wind on
-  # # an error.
-  # if vmState.chainDB.config.poaEngine and
-  #    not poa.updatePoaState(header, body):
-  #   debug "PoA update failed"
-  #   return ValidationResult.Error
-
-  var dbTx = vmState.com.db.beginTransaction()
-  defer: dbTx.dispose()
-
-  if not vmState.procBlkPreamble(header, body):
-    return ValidationResult.Error
-
-  # EIP-3675: no reward for miner in POA/POS
-  if vmState.com.consensus == ConsensusType.POW:
-    vmState.calculateReward(header, body)
-
-  if not vmState.procBlkEpilogue(header, body):
-    return ValidationResult.Error
-
-  dbTx.commit(applyDeletes = false)
-
-  ValidationResult.OK
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/core/validate.nim

@@ -13,10 +13,11 @@ import
   ../db/accounts_cache,
   ".."/[transaction, common/common],
   ".."/[errors],
+  ../utils/utils,
   "."/[dao, eip4844, gaslimit, withdrawals],
   ./pow/[difficulty, header],
   ./pow,
-  nimcrypto/utils,
+  nimcrypto/utils as cryptoutils,
   stew/[objects, results]
 
 from stew/byteutils
@@ -31,14 +32,6 @@ const
   daoForkBlockExtraData* =
     byteutils.hexToByteArray[13](DAOForkBlockExtra).toSeq
 
-# ------------------------------------------------------------------------------
-# Private Helpers
-# ------------------------------------------------------------------------------
-
-func isGenesis(header: BlockHeader): bool =
-  header.blockNumber == 0.u256 and
-    header.parentHash == GENESIS_PARENT_HASH
-
 # ------------------------------------------------------------------------------
 # Pivate validator functions
 # ------------------------------------------------------------------------------
@@ -76,7 +69,7 @@ proc validateHeader(
     body: BlockBody;
     checkSealOK: bool;
       ): Result[void,string]
-      {.gcsafe, raises: [CatchableError].} =
+      {.gcsafe, raises: [].} =
 
   template inDAOExtraRange(blockNumber: BlockNumber): bool =
     # EIP-799
@@ -129,29 +122,14 @@ proc validateHeader(
 
   ? com.validateWithdrawals(header, body)
   ? com.validateEip4844Header(header, parentHeader, body.transactions)
+  ? com.validateGasLimitOrBaseFee(header, parentHeader)
 
   ok()
 
-func validateUncle(currBlock, uncle, uncleParent: BlockHeader):
-                                               Result[void,string] =
-  if uncle.blockNumber >= currBlock.blockNumber:
-    return err("uncle block number larger than current block number")
-
-  if uncle.blockNumber != uncleParent.blockNumber + 1:
-    return err("Uncle number is not one above ancestor's number")
-
-  if uncle.timestamp.toUnix < uncleParent.timestamp.toUnix:
-    return err("Uncle timestamp is before ancestor's timestamp")
-
-  if uncle.gasUsed > uncle.gasLimit:
-    return err("Uncle's gas usage is above the limit")
-
-  result = ok()
-
-
 proc validateUncles(com: CommonRef; header: BlockHeader;
                     uncles: openArray[BlockHeader];
-                    checkSealOK: bool): Result[void,string] =
+                    checkSealOK: bool): Result[void,string]
+                      {.gcsafe, raises: [].} =
   let hasUncles = uncles.len > 0
   let shouldHaveUncles = header.ommersHash != EMPTY_UNCLE_HASH
 
@@ -206,6 +184,9 @@ proc validateUncles(com: CommonRef; header: BlockHeader;
        (uncle.parentHash == header.parentHash):
       return err("Uncle's parent is not an ancestor")
 
+    if uncle.blockNumber >= header.blockNumber:
+      return err("uncle block number larger than current block number")
+
     # check uncle against own parent
     var parent: BlockHeader
     if not chainDB.getBlockHeader(uncle.parentHash,parent):
@@ -224,11 +205,8 @@ proc validateUncles(com: CommonRef; header: BlockHeader;
     except BlockNotFound:
       return err("Uncle parent not found")
 
-    result = validateUncle(header, uncle, uncleParent)
+    result = com.validateHeader(uncle, uncleParent,
-    if result.isErr:
+                                BlockBody(), checkSealOK)
-      return
-
-    result = com.validateGasLimitOrBaseFee(uncle, uncleParent)
     if result.isErr:
       return
 
@@ -287,6 +265,9 @@ proc validateTxBasic*(
             "index=$1, len=$2" % [$i, $acl.storageKeys.len])
 
     if tx.txType >= TxEip4844:
+      if tx.networkPayload.isNil.not:
+        return err("invalid tx: network payload should not appear in block validation")
+
       if tx.to.isNone:
         return err("invalid tx: destination must be not empty")
 
@@ -399,7 +380,7 @@ proc validateHeaderAndKinship*(
     body: BlockBody;
     checkSealOK: bool;
       ): Result[void, string]
-      {.gcsafe, raises: [CatchableError].} =
+      {.gcsafe, raises: [].} =
   if header.isGenesis:
     if header.extraData.len > 32:
       return err("BlockHeader.extraData larger than 32 bytes")
@@ -419,15 +400,9 @@ proc validateHeaderAndKinship*(
   if body.uncles.len > MAX_UNCLES:
     return err("Number of uncles exceed limit.")
 
-  if not chainDB.exists(header.stateRoot):
-    return err("`state_root` was not found in the db.")
-
   if com.consensus != ConsensusType.POS:
     result = com.validateUncles(header, body.uncles, checkSealOK)
 
-  if result.isOk:
-    result = com.validateGasLimitOrBaseFee(header, parent)
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/core/withdrawals.nim

@@ -20,7 +20,7 @@ proc validateWithdrawals*(
     header: BlockHeader,
     body: BlockBody
       ): Result[void, string]
-      {.gcsafe, raises: [CatchableError].} =
+      {.gcsafe, raises: [].} =
 
   if com.forkGTE(Shanghai):
     if header.withdrawalsRoot.isNone:

nimbus/stateless_runner.nim

@@ -48,7 +48,7 @@ proc statelesslyRunBlock*(asyncDataSource: AsyncDataSource, com: CommonRef, head
     info("statelessly running block", blockNumber=header.blockNumber, blockHash=blockHash, parentHash=header.parentHash, parentStateRoot=parentHeader.stateRoot, desiredNewStateRoot=header.stateRoot)
 
     let vmState = createVmStateForStatelessMode(com, header, body, parentHeader, asyncFactory).get
-    let vres = processBlockNotPoA(vmState, header, body)
+    let vres = processBlock(vmState, header, body)
 
     let elapsedTime = now() - t0
 

nimbus/tracer.nim

@@ -182,7 +182,7 @@ proc dumpBlockState*(com: CommonRef, header: BlockHeader, body: BlockBody, dumpS
   for idx, uncle in body.uncles:
     before.captureAccount(stateBefore, uncle.coinbase, uncleName & $idx)
 
-  discard vmState.processBlockNotPoA(header, body)
+  discard vmState.processBlock(header, body)
 
   var stateAfter = vmState.stateDB
 

nimbus/utils/debug.nim

@@ -143,6 +143,17 @@ proc debug*(tx: Transaction): string =
   result.add "value         : " & $tx.value          & "\n"
   result.add "payload       : " & $tx.payload        & "\n"
   result.add "accessList    : " & $tx.accessList     & "\n"
+  result.add "maxFeePerBlobGas: " & $tx.maxFeePerBlobGas & "\n"
+  result.add "versionedHashes.len: " & $tx.versionedHashes.len & "\n"
+  
+  if tx.networkPayload.isNil:
+    result.add "networkPaylod : nil\n" 
+  else:
+    result.add "networkPaylod : \n"
+    result.add " - blobs       : " & $tx.networkPayload.blobs.len & "\n"
+    result.add " - commitments : " & $tx.networkPayload.commitments.len & "\n"
+    result.add " - proofs      : " & $tx.networkPayload.proofs.len & "\n"
+       
   result.add "V             : " & $tx.V              & "\n"
   result.add "R             : " & $tx.R              & "\n"
   result.add "S             : " & $tx.S              & "\n"

nimbus/utils/utils.nim

@@ -3,14 +3,14 @@ import
   eth/[rlp, common/eth_types_rlp],
   stew/byteutils,
   nimcrypto,
-  ../db/core_db
+  ../db/core_db,
+  ../constants
 
 export eth_types_rlp
 
 {.push raises: [].}
 
-proc calcRootHash[T](items: openArray[T]): Hash256
+proc calcRootHash[T](items: openArray[T]): Hash256 {.gcsafe.} =
-    {.gcsafe, raises: [CatchableError]} =
   var tr = newCoreDbRef(LegacyDbMemory).mptPrune
   for i, t in items:
     tr.put(rlp.encode(i), rlp.encode(t))
@@ -33,7 +33,7 @@ func sumHash*(hashes: varargs[Hash256]): Hash256 =
   ctx.finish result.data
   ctx.clear()
 
-proc sumHash*(body: BlockBody): Hash256 {.gcsafe, raises: [CatchableError]} =
+proc sumHash*(body: BlockBody): Hash256 {.gcsafe, raises: []} =
   let txRoot = calcTxRoot(body.transactions)
   let ommersHash = keccakHash(rlp.encode(body.uncles))
   let wdRoot = if body.withdrawals.isSome:
@@ -125,3 +125,7 @@ func gwei*(n: uint64): GasInt =
 # Helper types to convert gwei into wei more easily
 func weiAmount*(w: Withdrawal): UInt256 =
   w.amount.u256 * (10'u64 ^ 9'u64).u256
+
+func isGenesis*(header: BlockHeader): bool =
+  header.blockNumber == 0.u256 and
+    header.parentHash == GENESIS_PARENT_HASH

premix/debug.nim

@@ -29,7 +29,7 @@ proc executeBlock(blockEnv: JsonNode, memoryDB: CoreDbRef, blockNumber: UInt256)
 
   let
     vmState = BaseVMState.new(parent, header, com)
-    validationResult = vmState.processBlockNotPoA(header, body)
+    validationResult = vmState.processBlock(header, body)
 
   if validationResult != ValidationResult.OK:
     error "block validation error", validationResult

premix/dumper.nim

@@ -30,7 +30,7 @@ proc dumpDebug(com: CommonRef, blockNumber: UInt256) =
     vmState = BaseVMState.new(parent, header, captureCom)
 
   discard captureCom.db.setHead(parent, true)
-  discard vmState.processBlockNotPoA(header, body)
+  discard vmState.processBlock(header, body)
 
   transaction.rollback()
   dumpDebuggingMetaData(captureCom, header, body, vmState, false)

premix/hunter.nim

@@ -102,7 +102,7 @@ proc huntProblematicBlock(blockNumber: UInt256): ValidationResult =
   defer: transaction.dispose()
   let
     vmState = HunterVMState.new(parentBlock.header, thisBlock.header, com)
-    validationResult = vmState.processBlockNotPoA(thisBlock.header, thisBlock.body)
+    validationResult = vmState.processBlock(thisBlock.header, thisBlock.body)
 
   if validationResult != ValidationResult.OK:
     transaction.rollback()

premix/regress.nim

@@ -30,7 +30,7 @@ proc validateBlock(com: CommonRef, blockNumber: BlockNumber): BlockNumber =
 
     let
       vmState = BaseVMState.new(parent, headers[i], com)
-      validationResult = vmState.processBlockNotPoA(headers[i], bodies[i])
+      validationResult = vmState.processBlock(headers[i], bodies[i])
 
     if validationResult != ValidationResult.OK:
       error "block validation error", validationResult, blockNumber = blockNumber + i.u256

tests/all_tests.nim

@@ -5,7 +5,7 @@
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-import ../test_macro
+import ./all_tests_macro
 
 {. warning[UnusedImport]:off .}
 

tests/all_tests_macro.nim

@@ -6,7 +6,7 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import std/times
-import ./nimbus/vm_compile_info
+import ../nimbus/vm_compile_info
 import macros, strutils, os, unittest2, osproc
 import threadpool
 

tests/test_blockchain_json.nim

@@ -20,7 +20,7 @@ import
   ../nimbus/utils/[utils, debug],
   ../nimbus/evm/tracer/legacy_tracer,
   ../nimbus/evm/tracer/json_tracer,
-  ../nimbus/core/[executor, validate, pow/header],
+  ../nimbus/core/[validate, chain, pow/header],
   ../stateless/[tree_from_witness, witness_types],
   ../tools/common/helpers as chp,
   ../tools/evmstate/helpers,
@@ -53,9 +53,6 @@ type
     postStateHash: Hash256
     json         : bool
 
-var
-  trustedSetupLoaded = false
-
 proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = false, trace = false)
 
 func normalizeNumber(n: JsonNode): JsonNode =
@@ -209,53 +206,32 @@ proc setupTracer(ctx: TestCtx): TracerRef =
     TracerRef()
 
 proc importBlock(ctx: var TestCtx, com: CommonRef,
-                 tb: TestBlock, checkSeal, validation: bool) =
+                 tb: TestBlock, checkSeal: bool) =
-
-  let parentHeader = com.db.getBlockHeader(tb.header.parentHash)
-  let td = some(com.db.getScore(tb.header.parentHash))
-  com.hardForkTransition(tb.header.blockNumber, td, some(tb.header.timestamp))
-
-  if com.isCancunOrLater(tb.header.timestamp):
-    if not trustedSetupLoaded:
-      let res = loadKzgTrustedSetup()
-      if res.isErr:
-        echo "FATAL: ", res.error
-        quit(QuitFailure)
-      trustedSetupLoaded = true
-
   if ctx.vmState.isNil or ctx.vmState.stateDB.isTopLevelClean.not:
-    let tracerInst = ctx.setupTracer()
+    let
+      parentHeader = com.db.getBlockHeader(tb.header.parentHash)
+      tracerInst = ctx.setupTracer()
     ctx.vmState = BaseVMState.new(
       parentHeader,
       tb.header,
       com,
       tracerInst,
     )
-  else:
-    doAssert(ctx.vmState.reinit(parentHeader, tb.header))
 
-  if validation:
+  let
-    let rc = com.validateHeaderAndKinship(
+    chain = newChain(com, extraValidation = true, ctx.vmState)
-      tb.header, tb.body, checkSeal)
+    res = chain.persistBlocks([tb.header], [tb.body])
-    if rc.isErr:
-      raise newException(
-        ValidationError, "validateHeaderAndKinship: " & rc.error)
 
-  let res = ctx.vmState.processBlockNotPoA(tb.header, tb.body)
   if res == ValidationResult.Error:
-    if not (tb.hasException or (not tb.goodBlock)):
+    raise newException(ValidationError, "persistBlocks validation")
-      raise newException(ValidationError, "process block validation")
   else:
     if ctx.vmState.generateWitness():
-     blockWitness(ctx.vmState, com.db)
+      blockWitness(ctx.vmState, com.db)
-
-  discard com.db.persistHeaderToDb(tb.header,
-    com.consensus == ConsensusType.POS)
 
 proc applyFixtureBlockToChain(ctx: var TestCtx, tb: var TestBlock,
-                              com: CommonRef, checkSeal, validation: bool) =
+                              com: CommonRef, checkSeal: bool) =
   decompose(tb.blockRLP, tb.header, tb.body)
-  ctx.importBlock(com, tb, checkSeal, validation)
+  ctx.importBlock(com, tb, checkSeal)
 
 func shouldCheckSeal(ctx: TestCtx): bool =
   if ctx.sealEngine.isSome:
@@ -285,19 +261,9 @@ proc runTestCtx(ctx: var TestCtx, com: CommonRef, testStatusIMPL: var TestStatus
   for idx, tb in ctx.blocks:
     if tb.goodBlock:
       try:
-        ctx.applyFixtureBlockToChain(
-          ctx.blocks[idx], com, checkSeal, validation = false)
 
-        # manually validating
+        ctx.applyFixtureBlockToChain(
-        let res = com.validateHeaderAndKinship(
+          ctx.blocks[idx], com, checkSeal)
-                    tb.header, tb.body, checkSeal)
-        check res.isOk
-        when defined(noisy):
-          if res.isErr:
-            debugEcho "blockNumber  : ", tb.header.blockNumber
-            debugEcho "fork         : ", com.toHardFork(tb.header.blockNumber)
-            debugEcho "error message: ", res.error
-            debugEcho "consensusType: ", com.consensus
 
       except CatchableError as ex:
         debugEcho "FATAL ERROR(WE HAVE BUG): ", ex.msg
@@ -306,7 +272,7 @@ proc runTestCtx(ctx: var TestCtx, com: CommonRef, testStatusIMPL: var TestStatus
       var noError = true
       try:
         ctx.applyFixtureBlockToChain(ctx.blocks[idx],
-          com, checkSeal, validation = true)
+          com, checkSeal)
       except ValueError, ValidationError, BlockNotFound, RlpError:
         # failure is expected on this bad block
         check (tb.hasException or (not tb.goodBlock))
@@ -399,7 +365,7 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal
       elif lastBlockHash == ctx.lastBlockHash:
         # multiple chain, we are using the last valid canonical
         # state root to test against 'postState'
-        let stateDB = AccountsCache.init(memDB, header.stateRoot, pruneTrie)
+        let stateDB = AccountsCache.init(com.db, header.stateRoot, pruneTrie)
         verifyStateDB(fixture["postState"], ReadOnlyStateDB(stateDB))
 
       success = lastBlockHash == ctx.lastBlockHash
@@ -425,6 +391,11 @@ proc blockchainJsonMain*(debugMode = false) =
     #newFolder = "eth_tests/EIPTests/BlockchainTests"
     #newFolder = "eth_tests/EIPTests/Pyspecs/cancun"
 
+  let res = loadKzgTrustedSetup()
+  if res.isErr:
+    echo "FATAL: ", res.error
+    quit(QuitFailure)
+
   let config = test_config.getConfiguration()
   if config.testSubject == "" or not debugMode:
     # run all test fixtures

tests/test_txpool.nim

@@ -814,7 +814,6 @@ proc runTxPackerTests(noisy = true) =
             " size=", mostlySize + blk.txs[n].gasLimit - blk.header.gasUsed
 
         let
-          poa = bcCom.poa
           bdy = BlockBody(transactions: blk.txs, withdrawals: blk.withdrawals)
           hdr = block:
             var rc = blk.header
@@ -831,13 +830,13 @@ proc runTxPackerTests(noisy = true) =
         # Test low-level function for adding the new block to the database
         #xq.chain.maxMode = (packItemsMaxGasLimit in xq.flags)
         xq.chain.clearAccounts
-        check xq.chain.vmState.processBlock(poa, hdr, bdy).isOK
+        check xq.chain.vmState.processBlock(hdr, bdy).isOK
 
         setErrorLevel()
 
         # Re-allocate using VM environment from `persistBlocks()`
         let vmstate2 = BaseVMState.new(hdr, bcCom)
-        check vmstate2.processBlock(poa, hdr, bdy).isOK
+        check vmstate2.processBlock(hdr, bdy).isOK
 
         # This should not have changed
         check canonicalHead == xq.chain.com.db.getCanonicalHead

tests/test_txpool2.nim

@@ -173,10 +173,10 @@ proc runTxPoolCliqueTest*() =
 
     test "Store generated block in block chain database":
       xp.chain.clearAccounts
-      check xp.chain.vmState.processBlock(com.poa, blk.header, body).isOK
+      check xp.chain.vmState.processBlock(blk.header, body).isOK
 
       let vmstate2 = BaseVMState.new(blk.header, com)
-      check vmstate2.processBlock(com.poa, blk.header, body).isOK
+      check vmstate2.processBlock(blk.header, body).isOK
 
     test "Clique persistBlocks":
       let rr = chain.persistBlocks([blk.header], [body])
@@ -210,7 +210,7 @@ proc runTxPoolCliqueTest*() =
       os.sleep(com.cliquePeriod * 1000)
 
       xp.chain.clearAccounts
-      check xp.chain.vmState.processBlock(com.poa, blk.header, body).isOK
+      check xp.chain.vmState.processBlock(blk.header, body).isOK
       let rr = chain.persistBlocks([blk.header], [body])
       check rr == ValidationResult.OK