Prepare state test and blockchain test for Cancun (#1772)

premix/parser.nim

@@ -117,6 +117,9 @@ proc parseBlockHeader*(n: JsonNode): BlockHeader =
   n.fromJson "nonce", result.nonce
   n.fromJson "baseFeePerGas", result.fee
   n.fromJson "withdrawalsRoot", result.withdrawalsRoot
+  n.fromJson "blobGasUsed", result.blobGasUsed
+  n.fromJson "excessBlobGas", result.excessBlobGas
+  n.fromJson "parentBeaconBlockRoot", result.parentBeaconBlockRoot
 
   if result.baseFee == 0.u256:
     # probably geth bug

tests/test_blockchain_json.nim

@@ -9,7 +9,7 @@
 # according to those terms.
 
 import
-  std/[json, os, tables, strutils, options],
+  std/[json, os, tables, strutils, options, streams],
   unittest2,
   eth/rlp, eth/trie/trie_defs, eth/common/eth_types_rlp,
   stew/byteutils,
@@ -19,11 +19,13 @@ import
   ../nimbus/db/accounts_cache,
   ../nimbus/utils/[utils, debug],
   ../nimbus/evm/tracer/legacy_tracer,
+  ../nimbus/evm/tracer/json_tracer,
   ../nimbus/core/[executor, validate, pow/header],
   ../stateless/[tree_from_witness, witness_types],
   ../tools/common/helpers as chp,
   ../tools/evmstate/helpers,
-  ../nimbus/common/common
+  ../nimbus/common/common,
+  ../nimbus/core/eip4844
 
 type
   SealEngine = enum
@@ -38,7 +40,7 @@ type
     hasException: bool
     withdrawals: Option[seq[Withdrawal]]
 
-  Tester = object
+  TestCtx = object
     lastBlockHash: Hash256
     genesisHeader: BlockHeader
     blocks       : seq[TestBlock]
@@ -49,6 +51,10 @@ type
     debugData    : JsonNode
     network      : string
     postStateHash: Hash256
+    json         : bool
+
+var
+  trustedSetupLoaded = false
 
 proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = false, trace = false)
 
@@ -142,7 +148,7 @@ proc parseBlocks(blocks: JsonNode): seq[TestBlock] =
 
     result.add t
 
-proc parseTester(fixture: JsonNode, testStatusIMPL: var TestStatus): Tester =
+proc parseTestCtx(fixture: JsonNode, testStatusIMPL: var TestStatus): TestCtx =
   result.blocks = parseBlocks(fixture["blocks"])
 
   fixture.fromJson "lastblockhash", result.lastBlockHash
@@ -185,23 +191,48 @@ proc blockWitness(vmState: BaseVMState, chainDB: CoreDbRef) =
   if root != rootHash:
     raise newException(ValidationError, "Invalid trie generated from block witness")
 
-proc importBlock(tester: var Tester, com: CommonRef,
+proc setupTracer(ctx: TestCtx): TracerRef =
+  if ctx.trace:
+    if ctx.json:
+      var tracerFlags = {
+        TracerFlags.DisableMemory,
+        TracerFlags.DisableStorage,
+        TracerFlags.DisableState,
+        TracerFlags.DisableStateDiff,
+        TracerFlags.DisableReturnData
+      }
+      let stream = newFileStream(stdout)
+      newJsonTracer(stream, tracerFlags, false)
+    else:
+      newLegacyTracer({})
+  else:
+    TracerRef()
+
+proc importBlock(ctx: var TestCtx, com: CommonRef,
                  tb: TestBlock, checkSeal, validation: bool) =
 
   let parentHeader = com.db.getBlockHeader(tb.header.parentHash)
   let td = some(com.db.getScore(tb.header.parentHash))
   com.hardForkTransition(tb.header.blockNumber, td, some(tb.header.timestamp))
-  let tracerInst = if tester.trace:
-                 newLegacyTracer({})
-               else:
-                 LegacyTracer(nil)
 
-  tester.vmState = BaseVMState.new(
+  if com.isCancunOrLater(tb.header.timestamp):
+    if not trustedSetupLoaded:
+      let res = loadKzgTrustedSetup()
+      if res.isErr:
+        echo "FATAL: ", res.error
+        quit(QuitFailure)
+      trustedSetupLoaded = true
+
+  if ctx.vmState.isNil or ctx.vmState.stateDB.isTopLevelClean.not:
+    let tracerInst = ctx.setupTracer()
+    ctx.vmState = BaseVMState.new(
       parentHeader,
       tb.header,
       com,
       tracerInst,
     )
+  else:
+    doAssert(ctx.vmState.reinit(parentHeader, tb.header))
 
   if validation:
     let rc = com.validateHeaderAndKinship(
@@ -210,52 +241,52 @@ proc importBlock(tester: var Tester, com: CommonRef,
       raise newException(
         ValidationError, "validateHeaderAndKinship: " & rc.error)
 
-  let res = tester.vmState.processBlockNotPoA(tb.header, tb.body)
+  let res = ctx.vmState.processBlockNotPoA(tb.header, tb.body)
   if res == ValidationResult.Error:
     if not (tb.hasException or (not tb.goodBlock)):
       raise newException(ValidationError, "process block validation")
   else:
-    if tester.vmState.generateWitness():
+    if ctx.vmState.generateWitness():
-     blockWitness(tester.vmState, com.db)
+     blockWitness(ctx.vmState, com.db)
 
   discard com.db.persistHeaderToDb(tb.header,
     com.consensus == ConsensusType.POS)
 
-proc applyFixtureBlockToChain(tester: var Tester, tb: var TestBlock,
+proc applyFixtureBlockToChain(ctx: var TestCtx, tb: var TestBlock,
                               com: CommonRef, checkSeal, validation: bool) =
   decompose(tb.blockRLP, tb.header, tb.body)
-  tester.importBlock(com, tb, checkSeal, validation)
+  ctx.importBlock(com, tb, checkSeal, validation)
 
-func shouldCheckSeal(tester: Tester): bool =
+func shouldCheckSeal(ctx: TestCtx): bool =
-  if tester.sealEngine.isSome:
+  if ctx.sealEngine.isSome:
-    result = tester.sealEngine.get() != NoProof
+    result = ctx.sealEngine.get() != NoProof
 
-proc collectDebugData(tester: var Tester) =
+proc collectDebugData(ctx: var TestCtx) =
-  if tester.vmState.isNil:
+  if ctx.vmState.isNil:
     return
 
-  let vmState = tester.vmState
+  let vmState = ctx.vmState
   let tracerInst = LegacyTracer(vmState.tracer)
-  let tracingResult = if tester.trace: tracerInst.getTracingResult() else: %[]
+  let tracingResult = if ctx.trace: tracerInst.getTracingResult() else: %[]
-  tester.debugData.add %{
+  ctx.debugData.add %{
     "blockNumber": %($vmState.blockNumber),
     "structLogs": tracingResult,
   }
 
-proc runTester(tester: var Tester, com: CommonRef, testStatusIMPL: var TestStatus) =
+proc runTestCtx(ctx: var TestCtx, com: CommonRef, testStatusIMPL: var TestStatus) =
-  discard com.db.persistHeaderToDb(tester.genesisHeader,
+  discard com.db.persistHeaderToDb(ctx.genesisHeader,
     com.consensus == ConsensusType.POS)
-  check com.db.getCanonicalHead().blockHash == tester.genesisHeader.blockHash
+  check com.db.getCanonicalHead().blockHash == ctx.genesisHeader.blockHash
-  let checkSeal = tester.shouldCheckSeal
+  let checkSeal = ctx.shouldCheckSeal
 
-  if tester.debugMode:
+  if ctx.debugMode:
-    tester.debugData = newJArray()
+    ctx.debugData = newJArray()
 
-  for idx, tb in tester.blocks:
+  for idx, tb in ctx.blocks:
     if tb.goodBlock:
       try:
-        tester.applyFixtureBlockToChain(
+        ctx.applyFixtureBlockToChain(
-          tester.blocks[idx], com, checkSeal, validation = false)
+          ctx.blocks[idx], com, checkSeal, validation = false)
 
         # manually validating
         let res = com.validateHeaderAndKinship(
@@ -274,14 +305,14 @@ proc runTester(tester: var Tester, com: CommonRef, testStatusIMPL: var TestStatu
     else:
       var noError = true
       try:
-        tester.applyFixtureBlockToChain(tester.blocks[idx],
+        ctx.applyFixtureBlockToChain(ctx.blocks[idx],
           com, checkSeal, validation = true)
       except ValueError, ValidationError, BlockNotFound, RlpError:
         # failure is expected on this bad block
         check (tb.hasException or (not tb.goodBlock))
         noError = false
-        if tester.debugMode:
+        if ctx.debugMode:
-          tester.debugData.add %{
+          ctx.debugData.add %{
             "exception": %($getCurrentException().name),
             "msg": %getCurrentExceptionMsg()
           }
@@ -289,32 +320,32 @@ proc runTester(tester: var Tester, com: CommonRef, testStatusIMPL: var TestStatu
       # Block should have caused a validation error
       check noError == false
 
-    if tester.debugMode:
+    if ctx.debugMode and not ctx.json:
-      tester.collectDebugData()
+      ctx.collectDebugData()
 
-proc debugDataFromAccountList(tester: Tester): JsonNode =
+proc debugDataFromAccountList(ctx: TestCtx): JsonNode =
-  let vmState = tester.vmState
+  let vmState = ctx.vmState
-  result = %{"debugData": tester.debugData}
+  result = %{"debugData": ctx.debugData}
   if not vmState.isNil:
     result["accounts"] = vmState.dumpAccounts()
 
-proc debugDataFromPostStateHash(tester: Tester): JsonNode =
+proc debugDataFromPostStateHash(ctx: TestCtx): JsonNode =
-  let vmState = tester.vmState
+  let vmState = ctx.vmState
   %{
-    "debugData": tester.debugData,
+    "debugData": ctx.debugData,
     "postStateHash": %($vmState.readOnlyStateDB.rootHash),
-    "expectedStateHash": %($tester.postStateHash),
+    "expectedStateHash": %($ctx.postStateHash),
     "accounts": vmState.dumpAccounts()
   }
 
-proc dumpDebugData(tester: Tester, fixtureName: string, fixtureIndex: int, success: bool) =
+proc dumpDebugData(ctx: TestCtx, fixtureName: string, fixtureIndex: int, success: bool) =
-  let debugData = if tester.postStateHash != Hash256():
+  let debugData = if ctx.postStateHash != Hash256():
-                    debugDataFromPostStateHash(tester)
+                    debugDataFromPostStateHash(ctx)
                   else:
-                    debugDataFromAccountList(tester)
+                    debugDataFromAccountList(ctx)
 
   let status = if success: "_success" else: "_failed"
-  let name = fixtureName.replace('/', '-')
+  let name = fixtureName.replace('/', '-').replace(':', '-')
   writeFile("debug_" & name & "_" & $fixtureIndex & status & ".json", debugData.pretty())
 
 proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = false, trace = false) =
@@ -333,50 +364,51 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal
     if specifyIndex > 0 and fixtureIndex != specifyIndex:
       continue
 
-    var tester = parseTester(fixture, testStatusIMPL)
+    var ctx = parseTestCtx(fixture, testStatusIMPL)
 
     let
       pruneTrie = test_config.getConfiguration().pruning
       memDB     = newCoreDbRef LegacyDbMemory
       stateDB   = AccountsCache.init(memDB, emptyRlpHash, pruneTrie)
-      config    = getChainConfig(tester.network)
+      config    = getChainConfig(ctx.network)
       com       = CommonRef.new(memDB, config, pruneTrie)
 
     setupStateDB(fixture["pre"], stateDB)
     stateDB.persist()
 
-    check stateDB.rootHash == tester.genesisHeader.stateRoot
+    check stateDB.rootHash == ctx.genesisHeader.stateRoot
 
-    tester.debugMode = debugMode
+    ctx.debugMode = debugMode
-    tester.trace = trace
+    ctx.trace = trace
+    ctx.json = test_config.getConfiguration().json
 
     var success = true
     try:
-      tester.runTester(com, testStatusIMPL)
+      ctx.runTestCtx(com, testStatusIMPL)
       let header = com.db.getCanonicalHead()
       let lastBlockHash = header.blockHash
-      check lastBlockHash == tester.lastBlockHash
+      check lastBlockHash == ctx.lastBlockHash
-      success = lastBlockHash == tester.lastBlockHash
+      success = lastBlockHash == ctx.lastBlockHash
-      if tester.postStateHash != Hash256():
+      if ctx.postStateHash != Hash256():
-        let rootHash = tester.vmState.stateDB.rootHash
+        let rootHash = ctx.vmState.stateDB.rootHash
-        if tester.postStateHash != rootHash:
+        if ctx.postStateHash != rootHash:
           raise newException(ValidationError, "incorrect postStateHash, expect=" &
             $rootHash & ", get=" &
-            $tester.postStateHash
+            $ctx.postStateHash
           )
-      elif lastBlockHash == tester.lastBlockHash:
+      elif lastBlockHash == ctx.lastBlockHash:
         # multiple chain, we are using the last valid canonical
         # state root to test against 'postState'
         let stateDB = AccountsCache.init(memDB, header.stateRoot, pruneTrie)
         verifyStateDB(fixture["postState"], ReadOnlyStateDB(stateDB))
 
-      success = lastBlockHash == tester.lastBlockHash
+      success = lastBlockHash == ctx.lastBlockHash
     except ValidationError as E:
       echo fixtureName, " ERROR: ", E.msg
       success = false
 
-    if tester.debugMode:
+    if ctx.debugMode:
-      tester.dumpDebugData(fixtureName, fixtureIndex, success)
+      ctx.dumpDebugData(fixtureName, fixtureIndex, success)
 
     fixtureTested = true
     check success == true
@@ -388,8 +420,10 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal
 
 proc blockchainJsonMain*(debugMode = false) =
   const
-    legacyFolder = "eth_tests" / "LegacyTests" / "Constantinople" / "BlockchainTests"
+    legacyFolder = "eth_tests/LegacyTests/Constantinople/BlockchainTests"
-    newFolder = "eth_tests" / "BlockchainTests"
+    newFolder = "eth_tests/BlockchainTests"
+    #newFolder = "eth_tests/EIPTests/BlockchainTests"
+    #newFolder = "eth_tests/EIPTests/Pyspecs/cancun"
 
   let config = test_config.getConfiguration()
   if config.testSubject == "" or not debugMode:
@@ -407,7 +441,7 @@ proc blockchainJsonMain*(debugMode = false) =
       quit(QuitFailure)
 
     let folder = if config.legacy: legacyFolder else: newFolder
-    let path = "tests" / "fixtures" / folder
+    let path = "tests/fixtures/" & folder
     let n = json.parseFile(path / config.testSubject)
     var testStatusIMPL: TestStatus
     testFixture(n, testStatusIMPL, debugMode = true, config.trace)

tests/test_config.nim

@@ -18,6 +18,7 @@ type
     trace*: bool
     legacy*: bool
     pruning*: bool
+    json*: bool
 
 var testConfig {.threadvar.}: Configuration
 
@@ -48,6 +49,7 @@ proc processArguments*(msg: var string): ConfigStatus =
       of "trace": config.trace = parseBool(value)
       of "legacy": config.legacy = parseBool(value)
       of "pruning": config.pruning = parseBool(value)
+      of "json": config.json = parseBool(value)
       else:
         msg = "Unknown option " & key
         if value.len > 0: msg = msg & " : " & value

tests/test_generalstate_json.nim

@@ -15,6 +15,7 @@ import
   ../nimbus/common/common,
   ../nimbus/utils/[utils, debug],
   ../nimbus/evm/tracer/legacy_tracer,
+  ../nimbus/core/eip4844,
   ../tools/common/helpers as chp,
   ../tools/evmstate/helpers,
   ../tools/common/state_clearing,
@@ -23,8 +24,9 @@ import
   stew/[results, byteutils]
 
 type
-  Tester = object
+  TestCtx = object
     name: string
+    parent: BlockHeader
     header: BlockHeader
     pre: JsonNode
     tx: Transaction
@@ -36,6 +38,9 @@ type
     index: int
     fork: string
 
+var
+  trustedSetupLoaded = false
+
 proc toBytes(x: string): seq[byte] =
   result = newSeq[byte](x.len)
   for i in 0..<x.len: result[i] = x[i].byte
@@ -50,39 +55,47 @@ method getAncestorHash*(vmState: BaseVMState; blockNumber: BlockNumber): Hash256
   else:
     return keccakHash(toBytes($blockNumber))
 
-proc dumpDebugData(tester: Tester, vmState: BaseVMState, gasUsed: GasInt, success: bool) =
+proc dumpDebugData(ctx: TestCtx, vmState: BaseVMState, gasUsed: GasInt, success: bool) =
   let tracerInst = LegacyTracer(vmState.tracer)
-  let tracingResult = if tester.trace: tracerInst.getTracingResult() else: %[]
+  let tracingResult = if ctx.trace: tracerInst.getTracingResult() else: %[]
   let debugData = %{
     "gasUsed": %gasUsed,
     "structLogs": tracingResult,
     "accounts": vmState.dumpAccounts()
   }
   let status = if success: "_success" else: "_failed"
-  writeFile(tester.name & "_" & tester.fork & "_" & $tester.index & status & ".json", debugData.pretty())
+  writeFile(ctx.name & "_" & ctx.fork & "_" & $ctx.index & status & ".json", debugData.pretty())
 
-proc testFixtureIndexes(tester: Tester, testStatusIMPL: var TestStatus) =
+proc testFixtureIndexes(ctx: var TestCtx, testStatusIMPL: var TestStatus) =
   let
-    com    = CommonRef.new(newCoreDbRef LegacyDbMemory, tester.chainConfig, getConfiguration().pruning)
+    com    = CommonRef.new(newCoreDbRef LegacyDbMemory, ctx.chainConfig, getConfiguration().pruning)
     parent = BlockHeader(stateRoot: emptyRlpHash)
-    tracer = if tester.trace:
+    tracer = if ctx.trace:
                newLegacyTracer({})
              else:
                LegacyTracer(nil)
 
+  if com.isCancunOrLater(ctx.header.timestamp):
+    if not trustedSetupLoaded:
+      let res = loadKzgTrustedSetup()
+      if res.isErr:
+        echo "FATAL: ", res.error
+        quit(QuitFailure)
+      trustedSetupLoaded = true
+
   let vmState = BaseVMState.new(
       parent = parent,
-      header = tester.header,
+      header = ctx.header,
       com    = com,
       tracer = tracer,
     )
 
   var gasUsed: GasInt
-  let sender = tester.tx.getSender()
+  let sender = ctx.tx.getSender()
-  let fork = com.toEVMFork(tester.header.forkDeterminationInfoForHeader)
+  let fork = com.toEVMFork(ctx.header.forkDeterminationInfoForHeader)
 
   vmState.mutateStateDB:
-    setupStateDB(tester.pre, db)
+    setupStateDB(ctx.pre, db)
 
     # this is an important step when using accounts_cache
     # it will affect the account storage's location
@@ -91,35 +104,36 @@ proc testFixtureIndexes(tester: Tester, testStatusIMPL: var TestStatus) =
 
   defer:
     let obtainedHash = vmState.readOnlyStateDB.rootHash
-    check obtainedHash == tester.expectedHash
+    check obtainedHash == ctx.expectedHash
     let logEntries = vmState.getAndClearLogEntries()
     let actualLogsHash = rlpHash(logEntries)
-    check(tester.expectedLogs == actualLogsHash)
+    check(ctx.expectedLogs == actualLogsHash)
-    if tester.debugMode:
+    if ctx.debugMode:
-      let success = tester.expectedLogs == actualLogsHash and obtainedHash == tester.expectedHash
+      let success = ctx.expectedLogs == actualLogsHash and obtainedHash == ctx.expectedHash
-      tester.dumpDebugData(vmState, gasUsed, success)
+      ctx.dumpDebugData(vmState, gasUsed, success)
 
   let rc = vmState.processTransaction(
-                tester.tx, sender, tester.header, fork)
+                ctx.tx, sender, ctx.header, fork)
   if rc.isOk:
     gasUsed = rc.value
 
-  let miner = tester.header.coinbase
+  let miner = ctx.header.coinbase
   coinbaseStateClearing(vmState, miner, fork)
 
 proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus,
                  trace = false, debugMode = false) =
-  var tester: Tester
+  var ctx: TestCtx
   var fixture: JsonNode
   for label, child in fixtures:
     fixture = child
-    tester.name = label
+    ctx.name = label
     break
 
-  tester.pre = fixture["pre"]
+  ctx.pre    = fixture["pre"]
-  tester.header = parseHeader(fixture["env"])
+  ctx.parent = parseParentHeader(fixture["env"])
-  tester.trace = trace
+  ctx.header = parseHeader(fixture["env"])
-  tester.debugMode = debugMode
+  ctx.trace  = trace
+  ctx.debugMode = debugMode
 
   let
     post   = fixture["post"]
@@ -128,51 +142,53 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus,
 
   template prepareFork(forkName: string) =
     try:
-      tester.chainConfig = getChainConfig(forkName)
+      ctx.chainConfig = getChainConfig(forkName)
     except ValueError as ex:
       debugEcho ex.msg
+      testStatusIMPL = TestStatus.Failed
       return
 
   template runSubTest(subTest: JsonNode) =
-    tester.expectedHash = Hash256.fromJson(subTest["hash"])
+    ctx.expectedHash = Hash256.fromJson(subTest["hash"])
-    tester.expectedLogs = Hash256.fromJson(subTest["logs"])
+    ctx.expectedLogs = Hash256.fromJson(subTest["logs"])
-    tester.tx = parseTx(txData, subTest["indexes"])
+    ctx.tx = parseTx(txData, subTest["indexes"])
-    tester.testFixtureIndexes(testStatusIMPL)
+    ctx.testFixtureIndexes(testStatusIMPL)
 
   if conf.fork.len > 0:
     if not post.hasKey(conf.fork):
       debugEcho "selected fork not available: " & conf.fork
       return
 
-    tester.fork = conf.fork
+    ctx.fork = conf.fork
     let forkData = post[conf.fork]
     prepareFork(conf.fork)
     if conf.index.isNone:
       for subTest in forkData:
         runSubTest(subTest)
-        inc tester.index
+        inc ctx.index
     else:
-      tester.index = conf.index.get()
+      ctx.index = conf.index.get()
-      if tester.index > forkData.len or tester.index < 0:
+      if ctx.index > forkData.len or ctx.index < 0:
         debugEcho "selected index out of range(0-$1), requested $2" %
-          [$forkData.len, $tester.index]
+          [$forkData.len, $ctx.index]
         return
 
-      let subTest = forkData[tester.index]
+      let subTest = forkData[ctx.index]
       runSubTest(subTest)
   else:
     for forkName, forkData in post:
       prepareFork(forkName)
-      tester.fork = forkName
+      ctx.fork = forkName
-      tester.index = 0
+      ctx.index = 0
       for subTest in forkData:
         runSubTest(subTest)
-        inc tester.index
+        inc ctx.index
 
 proc generalStateJsonMain*(debugMode = false) =
   const
-    legacyFolder = "eth_tests" / "LegacyTests" / "Constantinople" / "GeneralStateTests"
+    legacyFolder = "eth_tests/LegacyTests/Constantinople/GeneralStateTests"
-    newFolder = "eth_tests" / "GeneralStateTests"
+    newFolder = "eth_tests/GeneralStateTests"
+    #newFolder = "eth_tests/EIPTests/StateTests"
 
   let config = getConfiguration()
   if config.testSubject == "" or not debugMode: