Premix persist tool improvements (#2000)

* Add premix persist tool to Makefile to build as separate binary.

* Fix rootBytes.len > 0 assertion defect in HexaryTrie by adding call to com.db.compensateLegacySetup().

* Run hardForkTransition before executing transactions in getBlockWitness.

* Improve logging and add help message.

* Retry requests for block data.

Makefile

@@ -58,9 +58,11 @@ EXCLUDED_NIM_PACKAGES := 	\
 
 # debugging tools + testing tools
 TOOLS := \
-	test_tools_build
+	test_tools_build \
+	persist
 TOOLS_DIRS := \
-	tests
+	tests \
+	premix
 # comma-separated values for the "clean" target
 TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 

nimbus/rpc/experimental.nim

@@ -45,7 +45,9 @@ proc getBlockWitness*(
     # before trying to initialize the VM as we do here.
     vmState = BaseVMState.new(blockHeader, com)
     flags = if vmState.fork >= FKSpurious: {wfEIP170} else: {}
+
   vmState.generateWitness = true # Enable saving witness data
+  vmState.com.hardForkTransition(blockHeader)
 
   var dbTx = vmState.com.db.beginTransaction()
   defer: dbTx.dispose()

premix/configuration.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2020-2023 Status Research & Development GmbH
+# Copyright (c) 2020-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -10,7 +10,10 @@
 
 import
   std/[os, parseopt, strutils],
-  eth/common, stint, ../nimbus/config
+  eth/common,
+  stint,
+  chronicles,
+  ../nimbus/config
 
 from ../nimbus/common/chain_config import
   MainNet,
@@ -101,10 +104,12 @@ proc processArguments*(msg: var string): ConfigStatus =
   for kind, key, value in opt.getopt():
     case kind
     of cmdArgument:
-      discard
+      return EmptyOption
     of cmdLongOption, cmdShortOption:
       inc(length)
       case key.toLowerAscii()
+      of "help":
+        return EmptyOption
       of "datadir": config.dataDir = value
       of "maxblocks":
         checkArgument(processInteger, config.maxBlocks, value)
@@ -118,9 +123,14 @@ proc processArguments*(msg: var string): ConfigStatus =
       else:
         msg = "Unknown option " & key
         if value.len > 0: msg = msg & " : " & value
-        result = ErrorUnknownOption
-        break
+        return ErrorUnknownOption
     of cmdEnd:
       msg = "Error processing option [" & key & "]"
-      result = ErrorParseOption
-      break
+      return ErrorParseOption
+
+  info "Using configuration parameters: ",
+      datadir = config.dataDir,
+      maxblocks = config.maxBlocks,
+      head = config.head,
+      numcommits = config.numCommits,
+      netid = config.netId

premix/persist.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2020-2023 Status Research & Development GmbH
+# Copyright (c) 2020-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -11,6 +11,7 @@
 # use this module to quickly populate db with data from geth/parity
 
 import
+  std/os,
   chronicles,
   ../nimbus/errors,
   ../nimbus/core/chain,
@@ -37,7 +38,7 @@ else:
 
 template persistToDb(db: CoreDbRef, body: untyped) =
   block: body
- 
+
 proc main() {.used.} =
   # 97 block with uncles
   # 46147 block with first transaction
@@ -58,12 +59,13 @@ proc main() {.used.} =
 
   # move head to block number ...
   if conf.head != 0.u256:
-    var parentBlock = requestBlock(conf.head)
+    var parentBlock = requestBlock(conf.head, { DownloadAndValidate })
     discard com.db.setHead(parentBlock.header)
 
   if canonicalHeadHashKey().toOpenArray notin com.db.kvt:
     persistToDb(com.db):
       com.initializeEmptyDb()
+      com.db.compensateLegacySetup()
     doAssert(canonicalHeadHashKey().toOpenArray in com.db.kvt)
 
   var head = com.db.getCanonicalHead()
@@ -78,9 +80,21 @@ proc main() {.used.} =
 
   var numBlocks = 0
   var counter = 0
+  var retryCount = 0
 
   while true:
-    var thisBlock = requestBlock(blockNumber)
+
+    var thisBlock: Block
+    try:
+      thisBlock = requestBlock(blockNumber, { DownloadAndValidate })
+    except CatchableError as e:
+      if retryCount < 3:
+        warn "Unable to get block data via JSON-RPC API", error = e.msg
+        inc retryCount
+        sleep(1000)
+        continue
+      else:
+        raise e
 
     headers.add thisBlock.header
     bodies.add thisBlock.body
@@ -111,7 +125,9 @@ when isMainModule:
 
   ## Processing command line arguments
   if configuration.processArguments(message) != Success:
-    echo message
+    if len(message) > 0:
+      echo message
+    echo "Usage: persist --datadir=<DATA_DIR> --maxblocks=<MAX_BLOCKS> --head=<HEAD> --numcommits=<NUM_COMMITS> --netid=<NETWORK_ID>"
     quit(QuitFailure)
   else:
     if len(message) > 0: