From 332d294e772fc51b596960b57c1ce05aff698ae3 Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Tue, 13 Feb 2024 17:49:41 +0800 Subject: [PATCH] Storage of block witnesses (#1986) * Added procs to get and store block witness in db and add generate-witness cli flag. * Completed initial implementation of block witness storage. * Added test to verify witness is persisted to db after call to persistBlock. * Update getBlockWitness to return witness using Result type. * Make generate witness parameter hidden. --- nimbus/config.nim | 6 +++ nimbus/core/chain/chain_desc.nim | 15 +++++- nimbus/core/chain/persist_blocks.nim | 22 +++++++- nimbus/db/core_db/core_apps_legacy.nim | 8 ++- nimbus/db/core_db/core_apps_newapi.nim | 15 ++++++ nimbus/db/storage_types.nim | 8 ++- nimbus/nimbus.nim | 1 + nimbus/rpc/experimental.nim | 2 +- tests/all_tests.nim | 3 +- tests/test_configuration.nim | 16 ++++++ tests/test_persistblock_witness_json.nim | 69 ++++++++++++++++++++++++ 11 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 tests/test_persistblock_witness_json.nim diff --git a/nimbus/config.nim b/nimbus/config.nim index a0b14e658..7985dbf64 100644 --- a/nimbus/config.nim +++ b/nimbus/config.nim @@ -200,6 +200,12 @@ type defaultValueDesc: "" name: "verify-from" }: Option[uint64] + generateWitness* {. + hidden + desc: "Enable experimental generation and storage of block witnesses" + defaultValue: false + name: "generate-witness" }: bool + evm* {. desc: "Load alternative EVM from EVMC-compatible shared library" & sharedLibText defaultValue: "" diff --git a/nimbus/core/chain/chain_desc.nim b/nimbus/core/chain/chain_desc.nim index 0cf25d745..0e65aa147 100644 --- a/nimbus/core/chain/chain_desc.nim +++ b/nimbus/core/chain/chain_desc.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -34,6 +34,10 @@ type ## Trigger extra validation, currently within `persistBlocks()` ## function only. + generateWitness: bool ##\ + ## Enable generation of block witness, currently within `persistBlocks()` + ## function only. + verifyFrom: BlockNumber ##\ ## First block to when `extraValidation` will be applied (only ## effective if `extraValidation` is true.) @@ -101,6 +105,10 @@ proc extraValidation*(c: ChainRef): bool = ## Getter c.extraValidation +proc generateWitness*(c: ChainRef): bool = + ## Getter + c.generateWitness + proc verifyFrom*(c: ChainRef): BlockNumber = ## Getter c.verifyFrom @@ -125,6 +133,11 @@ proc `extraValidation=`*(c: ChainRef; extraValidation: bool) = ## extra block chain validation. c.extraValidation = extraValidation +proc `generateWitness=`*(c: ChainRef; generateWitness: bool) = + ## Setter. If set `true`, the assignment value `generateWitness` enables + ## block witness generation. + c.generateWitness = generateWitness + proc `verifyFrom=`*(c: ChainRef; verifyFrom: BlockNumber) = ## Setter. The assignment value `verifyFrom` defines the first block where ## validation should start if the `Clique` field `extraValidation` was set diff --git a/nimbus/core/chain/persist_blocks.nim b/nimbus/core/chain/persist_blocks.nim index 0a71454e6..f41450893 100644 --- a/nimbus/core/chain/persist_blocks.nim +++ b/nimbus/core/chain/persist_blocks.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -100,8 +100,11 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader]; msg = res.error return ValidationResult.Error + if c.generateWitness: + vmState.generateWitness = true + let - validationResult = if c.validateBlock: + validationResult = if c.validateBlock or c.generateWitness: vmState.processBlock(header, body) else: ValidationResult.OK @@ -132,6 +135,21 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader]; msg = $rc.error return ValidationResult.Error + if c.generateWitness: + let dbTx = c.db.beginTransaction() + defer: dbTx.dispose() + + let + mkeys = vmState.stateDB.makeMultiKeys() + # Reset state to what it was before executing the block of transactions + initialState = BaseVMState.new(header, c.com) + witness = initialState.buildWitness(mkeys) + + dbTx.rollback() + + c.db.setBlockWitness(header.blockHash(), witness) + + if NoPersistHeader notin flags: discard c.db.persistHeaderToDb( header, c.com.consensus == ConsensusType.POS, c.com.startOfHistory) diff --git a/nimbus/db/core_db/core_apps_legacy.nim b/nimbus/db/core_db/core_apps_legacy.nim index f6552068c..ec8902043 100644 --- a/nimbus/db/core_db/core_apps_legacy.nim +++ b/nimbus/db/core_db/core_apps_legacy.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -734,6 +734,12 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool = # see if stateRoot exists db.exists(header.stateRoot) +proc getBlockWitness*(db: CoreDbRef, blockHash: Hash256): seq[byte] {.gcsafe.} = + db.kvt.get(blockHashToBlockWitnessKey(blockHash).toOpenArray) + +proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) = + db.kvt.put(blockHashToBlockWitnessKey(blockHash).toOpenArray, witness) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/core_apps_newapi.nim b/nimbus/db/core_db/core_apps_newapi.nim index 92da9ce4f..ef006191a 100644 --- a/nimbus/db/core_db/core_apps_newapi.nim +++ b/nimbus/db/core_db/core_apps_newapi.nim @@ -924,6 +924,21 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool = # see if stateRoot exists db.exists(header.stateRoot) +proc getBlockWitness*( + db: CoreDbRef, blockHash: Hash256): Result[seq[byte], string] {.gcsafe.} = + let res = db.newKvt(Shared) + .get(blockHashToBlockWitnessKey(blockHash).toOpenArray) + if res.isErr(): + err("Failed to get block witness from database: " & $res.error.error) + else: + ok(res.value()) + +proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) = + let witnessKey = blockHashToBlockWitnessKey(blockHash) + db.newKvt.put(witnessKey.toOpenArray, witness).isOkOr: + warn logTxt "setBlockWitness()", witnessKey, action="put()", error=($$error) + return + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/storage_types.nim b/nimbus/db/storage_types.nim index 9b8540608..1cf756e05 100644 --- a/nimbus/db/storage_types.nim +++ b/nimbus/db/storage_types.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -31,6 +31,7 @@ type snapSyncAccount snapSyncStorageSlot snapSyncStateRoot + blockHashToBlockWitness DbKey* = object # The first byte stores the key type. The rest are key-specific values @@ -129,6 +130,11 @@ proc snapSyncStateRootKey*(h: openArray[byte]): DbKey {.inline.} = result.data[1 .. 32] = h result.dataEndPos = uint8 sizeof(h) +proc blockHashToBlockWitnessKey*(h: Hash256): DbKey {.inline.} = + result.data[0] = byte ord(blockHashToBlockWitness) + result.data[1 .. 32] = h.data + result.dataEndPos = uint8 32 + template toOpenArray*(k: DbKey): openArray[byte] = k.data.toOpenArray(0, int(k.dataEndPos)) diff --git a/nimbus/nimbus.nim b/nimbus/nimbus.nim index 4d6e193a8..ed4def8c8 100644 --- a/nimbus/nimbus.nim +++ b/nimbus/nimbus.nim @@ -67,6 +67,7 @@ proc basicServices(nimbus: NimbusNode, nimbus.chainRef.extraValidation = 0 < verifyFrom nimbus.chainRef.verifyFrom = verifyFrom + nimbus.chainRef.generateWitness = conf.generateWitness nimbus.beaconEngine = BeaconEngineRef.new(nimbus.txPool, nimbus.chainRef) proc manageAccounts(nimbus: NimbusNode, conf: NimbusConf) = diff --git a/nimbus/rpc/experimental.nim b/nimbus/rpc/experimental.nim index 69b291e98..8c15b4e6b 100644 --- a/nimbus/rpc/experimental.nim +++ b/nimbus/rpc/experimental.nim @@ -47,7 +47,7 @@ proc getBlockWitness*( vmState.generateWitness = true # Enable saving witness data vmState.com.hardForkTransition(blockHeader) - var dbTx = vmState.com.db.beginTransaction() + let dbTx = vmState.com.db.beginTransaction() defer: dbTx.dispose() # Execute the block of transactions and collect the keys of the touched account state diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 73ffcc2a4..b6e64c620 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -54,4 +54,5 @@ cliBuilder: ./test_beacon/test_skeleton, ./test_overflow, ./test_getproof_json, - ./test_rpc_experimental_json + ./test_rpc_experimental_json, + ./test_persistblock_witness_json diff --git a/tests/test_configuration.nim b/tests/test_configuration.nim index 0b23445ab..fbb60dc07 100644 --- a/tests/test_configuration.nim +++ b/tests/test_configuration.nim @@ -335,5 +335,21 @@ proc configurationMain*() = check conf.dataDir.string == defaultDataDir() check conf.keyStore.string == "banana" + test "generate-witness default": + let conf = makeTestConfig() + check conf.generateWitness == false + + test "generate-witness enabled": + let conf = makeConfig(@["--generate-witness"]) + check conf.generateWitness == true + + test "generate-witness equals true": + let conf = makeConfig(@["--generate-witness=true"]) + check conf.generateWitness == true + + test "generate-witness equals false": + let conf = makeConfig(@["--generate-witness=false"]) + check conf.generateWitness == false + when isMainModule: configurationMain() diff --git a/tests/test_persistblock_witness_json.nim b/tests/test_persistblock_witness_json.nim new file mode 100644 index 000000000..80d36d89b --- /dev/null +++ b/tests/test_persistblock_witness_json.nim @@ -0,0 +1,69 @@ +# Nimbus +# Copyright (c) 2018-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + std/[json, os, tables, strutils], + unittest2, + stew/byteutils, + ./test_helpers, + ../nimbus/core/chain, + ../nimbus/common/common, + ../stateless/[witness_verification, witness_types] + +# use tracerTestGen.nim to generate additional test data +proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) = + var + blockNumber = UInt256.fromHex(node["blockNumber"].getStr()) + memoryDB = newCoreDbRef LegacyDbMemory + config = chainConfigForNetwork(MainNet) + com = CommonRef.new(memoryDB, config, pruneTrie = false) + state = node["state"] + + for k, v in state: + let key = hexToSeqByte(k) + let value = hexToSeqByte(v.getStr()) + memoryDB.kvt.put(key, value) + + let + parentNumber = blockNumber - 1 + parent = com.db.getBlockHeader(parentNumber) + header = com.db.getBlockHeader(blockNumber) + headerHash = header.blockHash + blockBody = com.db.getBlockBody(headerHash) + chain = newChain(com) + headers = @[header] + bodies = @[blockBody] + + chain.generateWitness = true # Enable code to generate and store witness in the db + + # it's ok if setHead fails here because of missing ancestors + discard com.db.setHead(parent, true) + let validationResult = chain.persistBlocks(headers, bodies) + check validationResult == ValidationResult.OK + + let + blockHash = memoryDB.getBlockHash(blockNumber) + witness = memoryDB.getBlockWitness(blockHash).value() + verifyWitnessResult = verifyWitness(parent.stateRoot, witness, {wfNoFlag}) + + check verifyWitnessResult.isOk() + let witnessData = verifyWitnessResult.value() + + if blockBody.transactions.len() > 0: + check: + witness.len() > 0 + witnessData.len() > 0 + +proc persistBlockWitnessJsonMain*() = + suite "persist block json tests": + jsonTest("PersistBlockTests", testFixture) + #var testStatusIMPL: TestStatus + #let n = json.parseFile("tests" / "fixtures" / "PersistBlockTests" / "block420301.json") + #testFixture(n, testStatusIMPL) + +when isMainModule: + persistBlockWitnessJsonMain()