nimbus-eth1/stateless/test_witness_json.nim

191 lines
5.0 KiB
Nim
Raw Normal View History

2023-11-01 03:32:09 +00:00
# Nimbus
# Copyright (c) 2020-2024 Status Research & Development GmbH
2023-11-01 03:32:09 +00:00
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
2020-05-06 05:34:54 +00:00
import
Unified database frontend integration (#1670) * Nimbus folder environment update details: * Integrated `CoreDbRef` for the sources in the `nimbus` sub-folder. * The `nimbus` program does not compile yet as it needs the updates in the parallel `stateless` sub-folder. * Stateless environment update details: * Integrated `CoreDbRef` for the sources in the `stateless` sub-folder. * The `nimbus` program compiles now. * Premix environment update details: * Integrated `CoreDbRef` for the sources in the `premix` sub-folder. * Fluffy environment update details: * Integrated `CoreDbRef` for the sources in the `fluffy` sub-folder. * Tools environment update details: * Integrated `CoreDbRef` for the sources in the `tools` sub-folder. * Nodocker environment update details: * Integrated `CoreDbRef` for the sources in the `hive_integration/nodocker` sub-folder. * Tests environment update details: * Integrated `CoreDbRef` for the sources in the `tests` sub-folder. * The unit tests compile and run cleanly now. * Generalise `CoreDbRef` to any `select_backend` supported database why: Generalisation was just missed due to overcoming some compiler oddity which was tied to rocksdb for testing. * Suppress compiler warning for `newChainDB()` why: Warning was added to this function which must be wrapped so that any `CatchableError` is re-raised as `Defect`. * Split off persistent `CoreDbRef` constructor into separate file why: This allows to compile a memory only database version without linking the backend library. * Use memory `CoreDbRef` database by default detail: Persistent DB constructor needs to import `db/core_db/persistent why: Most tests use memory DB anyway. This avoids linking `-lrocksdb` or any other backend by default. * fix `toLegacyBackend()` availability check why: got garbled after memory/persistent split. * Clarify raw access to MPT for snap sync handler why: Logically, `kvt` is not the raw access for the hexary trie (although this holds for the legacy database)
2023-08-04 11:10:09 +00:00
eth/common, json, os, unittest2,
../nimbus/db/core_db,
2020-05-19 04:34:40 +00:00
./tree_from_witness, parseopt,
2020-05-08 05:16:24 +00:00
./witness_types, stew/byteutils
2020-05-06 05:34:54 +00:00
2020-05-08 05:16:24 +00:00
type
Tester = object
rootHash: KeccakHash
error: bool
output: seq[byte]
proc write(t: var Tester, x: openArray[byte]) =
t.output.add x
proc write(t: var Tester, x: string) =
let len = (x.len - 2) div 2
var buf: array[4096, byte]
hexToByteArray(x, buf, 0, len - 1)
t.write(buf.toOpenArray(0, len - 1))
proc write(t: var Tester, x: JsonNode) =
t.write(x.getStr())
proc processBranchNode(t: var Tester, x: JsonNode) =
t.write(x["mask"])
proc processExtensionNode(t: var Tester, x: JsonNode) =
t.write(x["nibblesLen"])
t.write(x["nibbles"])
proc processNode(t: var Tester, x: JsonNode, storageMode: bool = false)
proc writeSub(t: var Tester, x: JsonNode, name: string): string =
let subName = name & "Sub"
let nodeType = x[name].getStr()
if subName in x:
let subType = x[subName].getStr()
t.write(subType)
else:
t.write(nodeType)
result = nodeType
2020-05-08 05:16:24 +00:00
proc processHashNode(t: var Tester, x: JsonNode) =
discard t.writeSub(x, "nodeType")
2020-05-08 05:16:24 +00:00
t.write(x["data"])
proc processStorage(t: var Tester, tree: JsonNode) =
for x in tree:
t.processNode(x, true)
2020-05-14 04:09:01 +00:00
proc processByteCode(t: var Tester, x: JsonNode) =
let codeType = t.writeSub(x, "codeType")
2020-05-14 04:09:01 +00:00
case codeType
of "0x00":
let codeLen = x["codeLen"].getStr()
t.write(codeLen)
if codeLen != "0x00":
2020-05-14 04:09:01 +00:00
t.write(x["code"])
of "0x01":
t.write(x["codeLen"])
t.processHashNode(x["codeHash"])
else:
raise newException(ParsingError, "wrong bytecode type")
2020-05-14 04:09:01 +00:00
2020-05-08 05:16:24 +00:00
proc processAccountNode(t: var Tester, x: JsonNode) =
let accountType = t.writeSub(x, "accountType")
2020-05-08 05:16:24 +00:00
t.write(x["address"])
t.write(x["balance"])
t.write(x["nonce"])
case accountType:
of "0x00":
discard
of "0x01":
2020-05-14 04:09:01 +00:00
t.processByteCode(x)
2020-05-08 05:16:24 +00:00
t.processStorage(x["storage"])
else:
raise newException(ParsingError, "wrong account type")
2020-05-08 05:16:24 +00:00
proc processStorageLeafNode(t: var Tester, x: JsonNode) =
t.write(x["key"])
t.write(x["value"])
proc processNode(t: var Tester, x: JsonNode, storageMode: bool = false) =
let nodeType = t.writeSub(x, "nodeType")
2020-05-08 05:16:24 +00:00
case nodeType
of "0x00": t.processBranchNode(x)
of "0x01": t.processExtensionNode(x)
of "0x02":
if storageMode:
t.processStorageLeafNode(x)
else:
t.processAccountNode(x)
of "0x03":
t.write(x["data"])
2020-05-08 05:16:24 +00:00
else:
raise newException(ParsingError, "wrong node type")
2020-05-08 05:16:24 +00:00
proc parseRootHash(x: string): KeccakHash =
result.data = hexToByteArray[32](x)
proc parseTester(t: var Tester, n: JsonNode, testStatusIMPL: var TestStatus) =
2020-05-08 05:16:24 +00:00
t.error = n["error"].getBool()
t.rootHash = parseRootHash(n["rootHash"].getStr())
t.write(n["version"])
t.write(n["metadata"])
let tree = n["tree"]
try:
for x in tree:
t.processNode(x)
except ParsingError:
check t.error == true
2020-05-08 05:16:24 +00:00
proc parseTester(filename: string, testStatusIMPL: var TestStatus): Tester =
2020-05-08 05:16:24 +00:00
let n = parseFile(filename)
parseTester(result, n, testStatusIMPL)
2020-05-08 05:16:24 +00:00
proc runTest(filePath, fileName: string) =
test fileName:
let t = parseTester(filePath, testStatusIMPL)
Unified database frontend integration (#1670) * Nimbus folder environment update details: * Integrated `CoreDbRef` for the sources in the `nimbus` sub-folder. * The `nimbus` program does not compile yet as it needs the updates in the parallel `stateless` sub-folder. * Stateless environment update details: * Integrated `CoreDbRef` for the sources in the `stateless` sub-folder. * The `nimbus` program compiles now. * Premix environment update details: * Integrated `CoreDbRef` for the sources in the `premix` sub-folder. * Fluffy environment update details: * Integrated `CoreDbRef` for the sources in the `fluffy` sub-folder. * Tools environment update details: * Integrated `CoreDbRef` for the sources in the `tools` sub-folder. * Nodocker environment update details: * Integrated `CoreDbRef` for the sources in the `hive_integration/nodocker` sub-folder. * Tests environment update details: * Integrated `CoreDbRef` for the sources in the `tests` sub-folder. * The unit tests compile and run cleanly now. * Generalise `CoreDbRef` to any `select_backend` supported database why: Generalisation was just missed due to overcoming some compiler oddity which was tied to rocksdb for testing. * Suppress compiler warning for `newChainDB()` why: Warning was added to this function which must be wrapped so that any `CatchableError` is re-raised as `Defect`. * Split off persistent `CoreDbRef` constructor into separate file why: This allows to compile a memory only database version without linking the backend library. * Use memory `CoreDbRef` database by default detail: Persistent DB constructor needs to import `db/core_db/persistent why: Most tests use memory DB anyway. This avoids linking `-lrocksdb` or any other backend by default. * fix `toLegacyBackend()` availability check why: got garbled after memory/persistent split. * Clarify raw access to MPT for snap sync handler why: Logically, `kvt` is not the raw access for the hexary trie (although this holds for the legacy database)
2023-08-04 11:10:09 +00:00
var db = newCoreDbRef(LegacyDbMemory)
try:
var tb = initTreeBuilder(t.output, db, {wfNoFlag})
let root = tb.buildTree()
if t.error:
check root != t.rootHash
else:
check root == t.rootHash
check t.error == false
except ParsingError, ContractCodeError:
2020-07-21 06:15:06 +00:00
# echo "Exception detected ", getCurrentExceptionMsg()
check t.error == true
2020-05-08 05:16:24 +00:00
2020-05-19 04:34:40 +00:00
proc writeFuzzData(filePath, fileName: string) =
var testStatusIMPL: TestStatus
let t = parseTester(filePath, testStatusIMPL)
2020-07-21 06:15:06 +00:00
# this block below check the parsed json
Unified database frontend integration (#1670) * Nimbus folder environment update details: * Integrated `CoreDbRef` for the sources in the `nimbus` sub-folder. * The `nimbus` program does not compile yet as it needs the updates in the parallel `stateless` sub-folder. * Stateless environment update details: * Integrated `CoreDbRef` for the sources in the `stateless` sub-folder. * The `nimbus` program compiles now. * Premix environment update details: * Integrated `CoreDbRef` for the sources in the `premix` sub-folder. * Fluffy environment update details: * Integrated `CoreDbRef` for the sources in the `fluffy` sub-folder. * Tools environment update details: * Integrated `CoreDbRef` for the sources in the `tools` sub-folder. * Nodocker environment update details: * Integrated `CoreDbRef` for the sources in the `hive_integration/nodocker` sub-folder. * Tests environment update details: * Integrated `CoreDbRef` for the sources in the `tests` sub-folder. * The unit tests compile and run cleanly now. * Generalise `CoreDbRef` to any `select_backend` supported database why: Generalisation was just missed due to overcoming some compiler oddity which was tied to rocksdb for testing. * Suppress compiler warning for `newChainDB()` why: Warning was added to this function which must be wrapped so that any `CatchableError` is re-raised as `Defect`. * Split off persistent `CoreDbRef` constructor into separate file why: This allows to compile a memory only database version without linking the backend library. * Use memory `CoreDbRef` database by default detail: Persistent DB constructor needs to import `db/core_db/persistent why: Most tests use memory DB anyway. This avoids linking `-lrocksdb` or any other backend by default. * fix `toLegacyBackend()` availability check why: got garbled after memory/persistent split. * Clarify raw access to MPT for snap sync handler why: Logically, `kvt` is not the raw access for the hexary trie (although this holds for the legacy database)
2023-08-04 11:10:09 +00:00
var db = newCoreDbRef(LegacyDbMemory)
var tb = initTreeBuilder(t.output, db, {wfNoFlag})
2020-07-21 06:15:06 +00:00
discard tb.buildTree()
writeFile(fileName, t.output)
2020-05-19 04:34:40 +00:00
2020-07-21 06:15:06 +00:00
proc fuzzTool(): bool {.used.} =
2020-05-19 04:34:40 +00:00
var filename: string
var numArg = 0
for kind, key, val in getopt():
case kind
of cmdArgument:
inc numArg
case numArg
of 1:
if key != "fuzz":
quit(1)
of 2:
filename = key
else:
discard
of cmdLongOption, cmdShortOption:
discard
of cmdEnd: assert(false) # cannot happen
if filename != "":
echo "generate fuzz data"
writeFuzzData(filename, "fuzz.data")
return true
2020-05-19 04:34:40 +00:00
proc witnessJsonMain*() =
suite "test tree builder against json fixtures":
for x in walkDirRec("stateless" / "fixtures"):
let y = splitPath(x)
runTest(x, y.tail)
2020-05-08 05:16:24 +00:00
when isMainModule:
if not fuzzTool():
witnessJsonMain()