Storage of block witnesses (#1986)

* Added procs to get and store block witness in db and add generate-witness cli flag.

* Completed initial implementation of block witness storage.

* Added test to verify witness is persisted to db after call to persistBlock.

* Update getBlockWitness to return witness using Result type.

* Make generate witness parameter hidden.

nimbus/config.nim

@@ -200,6 +200,12 @@ type
       defaultValueDesc: ""
       name: "verify-from" }: Option[uint64]
 
+    generateWitness* {.
+      hidden
+      desc: "Enable experimental generation and storage of block witnesses"
+      defaultValue: false
+      name: "generate-witness" }: bool
+
     evm* {.
       desc: "Load alternative EVM from EVMC-compatible shared library" & sharedLibText
       defaultValue: ""

nimbus/core/chain/chain_desc.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -34,6 +34,10 @@ type
       ## Trigger extra validation, currently within `persistBlocks()`
       ## function only.
 
+    generateWitness: bool ##\
+      ## Enable generation of block witness, currently within `persistBlocks()`
+      ## function only.
+
     verifyFrom: BlockNumber ##\
       ## First block to when `extraValidation` will be applied (only
       ## effective if `extraValidation` is true.)
@@ -101,6 +105,10 @@ proc extraValidation*(c: ChainRef): bool =
   ## Getter
   c.extraValidation
 
+proc generateWitness*(c: ChainRef): bool =
+  ## Getter
+  c.generateWitness
+
 proc verifyFrom*(c: ChainRef): BlockNumber =
   ## Getter
   c.verifyFrom
@@ -125,6 +133,11 @@ proc `extraValidation=`*(c: ChainRef; extraValidation: bool) =
   ## extra block chain validation.
   c.extraValidation = extraValidation
 
+proc `generateWitness=`*(c: ChainRef; generateWitness: bool) =
+  ## Setter. If set `true`, the assignment value `generateWitness` enables
+  ## block witness generation.
+  c.generateWitness = generateWitness
+
 proc `verifyFrom=`*(c: ChainRef; verifyFrom: BlockNumber) =
   ## Setter. The  assignment value `verifyFrom` defines the first block where
   ## validation should start if the `Clique` field `extraValidation` was set

nimbus/core/chain/persist_blocks.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -100,8 +100,11 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
             msg = res.error
           return ValidationResult.Error
 
+    if c.generateWitness:
+      vmState.generateWitness = true
+
     let
-      validationResult = if c.validateBlock:
+      validationResult = if c.validateBlock or c.generateWitness:
                            vmState.processBlock(header, body)
                          else:
                            ValidationResult.OK
@@ -132,6 +135,21 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
             msg = $rc.error
           return ValidationResult.Error
 
+    if c.generateWitness:
+      let dbTx = c.db.beginTransaction()
+      defer: dbTx.dispose()
+
+      let
+        mkeys = vmState.stateDB.makeMultiKeys()
+        # Reset state to what it was before executing the block of transactions
+        initialState = BaseVMState.new(header, c.com)
+        witness = initialState.buildWitness(mkeys)
+
+      dbTx.rollback()
+
+      c.db.setBlockWitness(header.blockHash(), witness)
+
+
     if NoPersistHeader notin flags:
       discard c.db.persistHeaderToDb(
         header, c.com.consensus == ConsensusType.POS, c.com.startOfHistory)

nimbus/db/core_db/core_apps_legacy.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -734,6 +734,12 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool =
   # see if stateRoot exists
   db.exists(header.stateRoot)
 
+proc getBlockWitness*(db: CoreDbRef, blockHash: Hash256): seq[byte] {.gcsafe.} =
+  db.kvt.get(blockHashToBlockWitnessKey(blockHash).toOpenArray)
+
+proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) =
+  db.kvt.put(blockHashToBlockWitnessKey(blockHash).toOpenArray, witness)
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/db/core_db/core_apps_newapi.nim

@@ -924,6 +924,21 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool =
   # see if stateRoot exists
   db.exists(header.stateRoot)
 
+proc getBlockWitness*(
+    db: CoreDbRef, blockHash: Hash256): Result[seq[byte], string] {.gcsafe.} =
+  let res = db.newKvt(Shared)
+      .get(blockHashToBlockWitnessKey(blockHash).toOpenArray)
+  if res.isErr():
+    err("Failed to get block witness from database: " & $res.error.error)
+  else:
+    ok(res.value())
+
+proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) =
+  let witnessKey = blockHashToBlockWitnessKey(blockHash)
+  db.newKvt.put(witnessKey.toOpenArray, witness).isOkOr:
+    warn logTxt "setBlockWitness()", witnessKey, action="put()", error=($$error)
+    return
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/db/storage_types.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2023 Status Research & Development GmbH
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -31,6 +31,7 @@ type
     snapSyncAccount
     snapSyncStorageSlot
     snapSyncStateRoot
+    blockHashToBlockWitness
 
   DbKey* = object
     # The first byte stores the key type. The rest are key-specific values
@@ -129,6 +130,11 @@ proc snapSyncStateRootKey*(h: openArray[byte]): DbKey {.inline.} =
   result.data[1 .. 32] = h
   result.dataEndPos = uint8 sizeof(h)
 
+proc blockHashToBlockWitnessKey*(h: Hash256): DbKey {.inline.} =
+  result.data[0] = byte ord(blockHashToBlockWitness)
+  result.data[1 .. 32] = h.data
+  result.dataEndPos = uint8 32
+
 template toOpenArray*(k: DbKey): openArray[byte] =
   k.data.toOpenArray(0, int(k.dataEndPos))
 

nimbus/nimbus.nim

@@ -67,6 +67,7 @@ proc basicServices(nimbus: NimbusNode,
     nimbus.chainRef.extraValidation = 0 < verifyFrom
     nimbus.chainRef.verifyFrom = verifyFrom
 
+  nimbus.chainRef.generateWitness = conf.generateWitness
   nimbus.beaconEngine = BeaconEngineRef.new(nimbus.txPool, nimbus.chainRef)
 
 proc manageAccounts(nimbus: NimbusNode, conf: NimbusConf) =

nimbus/rpc/experimental.nim

@@ -47,7 +47,7 @@ proc getBlockWitness*(
   vmState.generateWitness = true # Enable saving witness data
   vmState.com.hardForkTransition(blockHeader)
 
-  var dbTx = vmState.com.db.beginTransaction()
+  let dbTx = vmState.com.db.beginTransaction()
   defer: dbTx.dispose()
 
   # Execute the block of transactions and collect the keys of the touched account state

tests/all_tests.nim

@@ -54,4 +54,5 @@ cliBuilder:
           ./test_beacon/test_skeleton,
           ./test_overflow,
           ./test_getproof_json,
-          ./test_rpc_experimental_json
+          ./test_rpc_experimental_json,
+          ./test_persistblock_witness_json

tests/test_configuration.nim

@@ -335,5 +335,21 @@ proc configurationMain*() =
       check conf.dataDir.string == defaultDataDir()
       check conf.keyStore.string == "banana"
 
+    test "generate-witness default":
+      let conf = makeTestConfig()
+      check conf.generateWitness == false
+
+    test "generate-witness enabled":
+      let conf = makeConfig(@["--generate-witness"])
+      check conf.generateWitness == true
+
+    test "generate-witness equals true":
+      let conf = makeConfig(@["--generate-witness=true"])
+      check conf.generateWitness == true
+
+    test "generate-witness equals false":
+      let conf = makeConfig(@["--generate-witness=false"])
+      check conf.generateWitness == false
+
 when isMainModule:
   configurationMain()

tests/test_persistblock_witness_json.nim

@@ -0,0 +1,69 @@
+# Nimbus
+# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  std/[json, os, tables, strutils],
+  unittest2,
+  stew/byteutils,
+  ./test_helpers,
+  ../nimbus/core/chain,
+  ../nimbus/common/common,
+  ../stateless/[witness_verification, witness_types]
+
+# use tracerTestGen.nim to generate additional test data
+proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) =
+  var
+    blockNumber = UInt256.fromHex(node["blockNumber"].getStr())
+    memoryDB    = newCoreDbRef LegacyDbMemory
+    config      = chainConfigForNetwork(MainNet)
+    com         = CommonRef.new(memoryDB, config, pruneTrie = false)
+    state       = node["state"]
+
+  for k, v in state:
+    let key = hexToSeqByte(k)
+    let value = hexToSeqByte(v.getStr())
+    memoryDB.kvt.put(key, value)
+
+  let
+    parentNumber = blockNumber - 1
+    parent = com.db.getBlockHeader(parentNumber)
+    header = com.db.getBlockHeader(blockNumber)
+    headerHash = header.blockHash
+    blockBody = com.db.getBlockBody(headerHash)
+    chain = newChain(com)
+    headers = @[header]
+    bodies = @[blockBody]
+
+  chain.generateWitness = true # Enable code to generate and store witness in the db
+
+  # it's ok if setHead fails here because of missing ancestors
+  discard com.db.setHead(parent, true)
+  let validationResult = chain.persistBlocks(headers, bodies)
+  check validationResult == ValidationResult.OK
+
+  let
+    blockHash = memoryDB.getBlockHash(blockNumber)
+    witness = memoryDB.getBlockWitness(blockHash).value()
+    verifyWitnessResult = verifyWitness(parent.stateRoot, witness, {wfNoFlag})
+
+  check verifyWitnessResult.isOk()
+  let witnessData = verifyWitnessResult.value()
+
+  if blockBody.transactions.len() > 0:
+    check:
+      witness.len() > 0
+      witnessData.len() > 0
+
+proc persistBlockWitnessJsonMain*() =
+  suite "persist block json tests":
+    jsonTest("PersistBlockTests", testFixture)
+  #var testStatusIMPL: TestStatus
+  #let n = json.parseFile("tests" / "fixtures" / "PersistBlockTests" / "block420301.json")
+  #testFixture(n, testStatusIMPL)
+
+when isMainModule:
+  persistBlockWitnessJsonMain()