From e8eb3268f5632e0b9ec1abf415c3d9ceb7ecefc5 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Wed, 17 Apr 2024 18:09:55 +0000 Subject: [PATCH] Generalise prune mode option 4 different db models (#2139) * Update README * Nimbus-main: replaced `PruneMode` options by `ChainDbMode` options details: For the legacy database, this changes the phrase - `conf.pruneMode == PruneMode.Full` to the expression + `conf.chainDbMode == ChainDbMode.Prune`. * Fix issues moaned about by NIM compiler * Fix copyright year --- README.md | 9 ++++++--- fluffy/docs/the_fluffy_book/mkdocs.yml | 2 +- .../nodocker/engine/engine_env.nim | 2 +- hive_integration/nodocker/pyspec/test_env.nim | 4 ++-- hive_integration/nodocker/rpc/test_env.nim | 4 ++-- nimbus/config.nim | 20 ++++++++++++------- nimbus/db/aristo/aristo_init/rocks_db.nim | 4 ---- .../aristo/aristo_init/rocks_db/rdb_put.nim | 7 ------- .../backend/aristo_db/handlers_trace.nim | 5 +---- nimbus/db/kvstore_rocksdb.nim | 6 ++++-- nimbus/db/kvt/kvt_init/rocks_db/rdb_desc.nim | 2 +- nimbus/db/kvt/kvt_init/rocks_db/rdb_put.nim | 12 ++++++----- nimbus/db/ledger/distinct_ledgers.nim | 1 - nimbus/nimbus.nim | 14 ++++++++----- tests/test_beacon/setup_env.nim | 4 ++-- tests/test_configuration.nim | 17 +++++++++------- tests/test_merge.nim | 2 +- tests/test_rpc.nim | 2 +- tests/test_txpool/helpers.nim | 14 ++++++++----- tests/test_txpool2.nim | 2 +- 20 files changed, 71 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index eda1f5193..2607cc116 100644 --- a/README.md +++ b/README.md @@ -230,9 +230,6 @@ available.) Disable legacy chunked RLPx messages which are enabled by default for synchronising against `Nethermind` nodes - * ENABLE_ETH_VERSION=66
- Enable legacy protocol `eth66` (or whatever other protocol version.) - * ENABLE_EVMC=1
Enable mostly EVMC compliant wrapper around the native Nim VM @@ -244,6 +241,12 @@ available.) For these variables, using <variable>=0 is ignored and <variable>=2 has the same effect as <variable>=1 (ditto for other numbers.) +Other settings where the non-zero value matters: + + * ENABLE_ETH_VERSION=66
+ Enable legacy protocol `eth66` (or another available protocol version.) + + ### Development tips Interesting Make variables and targets are documented in the [nimbus-build-system](https://github.com/status-im/nimbus-build-system) repo. diff --git a/fluffy/docs/the_fluffy_book/mkdocs.yml b/fluffy/docs/the_fluffy_book/mkdocs.yml index f85b7a55c..a2efec277 100644 --- a/fluffy/docs/the_fluffy_book/mkdocs.yml +++ b/fluffy/docs/the_fluffy_book/mkdocs.yml @@ -1,5 +1,5 @@ # Nimbus Fluffy book -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). diff --git a/hive_integration/nodocker/engine/engine_env.nim b/hive_integration/nodocker/engine/engine_env.nim index d453b74f3..8cd27f67c 100644 --- a/hive_integration/nodocker/engine/engine_env.nim +++ b/hive_integration/nodocker/engine/engine_env.nim @@ -59,7 +59,7 @@ const proc makeCom*(conf: NimbusConf): CommonRef = CommonRef.new( newCoreDbRef LegacyDbMemory, - conf.pruneMode == PruneMode.Full, + conf.chainDbMode == ChainDbMode.Prune, conf.networkId, conf.networkParams ) diff --git a/hive_integration/nodocker/pyspec/test_env.nim b/hive_integration/nodocker/pyspec/test_env.nim index 7a298057d..3e48465a2 100644 --- a/hive_integration/nodocker/pyspec/test_env.nim +++ b/hive_integration/nodocker/pyspec/test_env.nim @@ -53,11 +53,11 @@ proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) = t.com = CommonRef.new( memDB, conf, - t.conf.pruneMode == PruneMode.Full + t.conf.chainDbMode == ChainDbMode.Prune ) t.chainRef = newChain(t.com, extraValidation = true) let - stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.pruneMode == PruneMode.Full) + stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.chainDbMode == ChainDbMode.Prune) genesisHeader = node.genesisHeader setupStateDB(node["pre"], stateDB) diff --git a/hive_integration/nodocker/rpc/test_env.nim b/hive_integration/nodocker/rpc/test_env.nim index 3505a741a..d601c43eb 100644 --- a/hive_integration/nodocker/rpc/test_env.nim +++ b/hive_integration/nodocker/rpc/test_env.nim @@ -61,7 +61,7 @@ proc stopRpcHttpServer(srv: RpcServer) = proc setupEnv*(): TestEnv = let conf = makeConfig(@[ - "--prune-mode:archive", + "--chaindb:archive", # "--nat:extip:0.0.0.0", "--network:7", "--import-key:" & initPath / "private-key", @@ -77,7 +77,7 @@ proc setupEnv*(): TestEnv = ethCtx = newEthContext() ethNode = setupEthNode(conf, ethCtx, eth) com = CommonRef.new(newCoreDbRef LegacyDbMemory, - conf.pruneMode == PruneMode.Full, + conf.chainDbMode == ChainDbMode.Prune, conf.networkId, conf.networkParams ) diff --git a/nimbus/config.nim b/nimbus/config.nim index 7985dbf64..93421f633 100644 --- a/nimbus/config.nim +++ b/nimbus/config.nim @@ -105,9 +105,10 @@ const sharedLibText = if defined(linux): " (*.so, *.so.N)" else: "" type - PruneMode* {.pure.} = enum - Full + ChainDbMode* {.pure.} = enum + Prune Archive + Aristo NimbusCmd* {.pure.} = enum noCommand @@ -154,12 +155,17 @@ type abbr: "k" name: "key-store" }: OutDir - pruneMode* {. - desc: "Blockchain prune mode (Full or Archive)" - defaultValue: PruneMode.Full - defaultValueDesc: $PruneMode.Full + chainDbMode* {. + desc: "Blockchain database" + longDesc: + "- Prune -- Legacy/reference database, full pruning\n" & + "- Archive -- Legacy/reference database without pruning\n" & + "- Aristo -- Experimental single state DB\n" & + "" + defaultValue: ChainDbMode.Prune + defaultValueDesc: $ChainDbMode.Prune abbr : "p" - name: "prune-mode" }: PruneMode + name: "chaindb" }: ChainDbMode syncMode* {. desc: "Specify particular blockchain sync mode." diff --git a/nimbus/db/aristo/aristo_init/rocks_db.nim b/nimbus/db/aristo/aristo_init/rocks_db.nim index 7a061cb47..9021a22cd 100644 --- a/nimbus/db/aristo/aristo_init/rocks_db.nim +++ b/nimbus/db/aristo/aristo_init/rocks_db.nim @@ -60,10 +60,6 @@ when extraTraceMessages: # Private helpers # ------------------------------------------------------------------------------ -template logTxt(info: static[string]): static[string] = - "RocksDB " & info - - proc newSession(db: RdbBackendRef): RdbPutHdlRef = new result result.TypedPutHdlRef.beginSession db diff --git a/nimbus/db/aristo/aristo_init/rocks_db/rdb_put.nim b/nimbus/db/aristo/aristo_init/rocks_db/rdb_put.nim index 2a6807764..d5f7467a2 100644 --- a/nimbus/db/aristo/aristo_init/rocks_db/rdb_put.nim +++ b/nimbus/db/aristo/aristo_init/rocks_db/rdb_put.nim @@ -15,19 +15,12 @@ import eth/common, - rocksdb/lib/librocksdb, rocksdb, results, ../../aristo_desc, ../init_common, ./rdb_desc -type - RdbPutSession = object - writer: ptr rocksdb_sstfilewriter_t - sstPath: string - nRecords: int - const extraTraceMessages = false ## Enable additional logging noise diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_trace.nim b/nimbus/db/core_db/backend/aristo_db/handlers_trace.nim index b2f89abe0..6805c8d6b 100644 --- a/nimbus/db/core_db/backend/aristo_db/handlers_trace.nim +++ b/nimbus/db/core_db/backend/aristo_db/handlers_trace.nim @@ -141,15 +141,12 @@ when EnableDebugLog: # ------------------------- -func getOrVoid(tab: TableRef[Blob,Blob]; w: openArray[byte]): Blob = - tab.getOrDefault(@w, EmptyBlob) - func leafTie( root: VertexID; path: openArray[byte]; ): Result[LeafTie,(VertexID,AristoError)] = let tag = path.pathToTag.valueOr: - return err((VertexID(root), error)) + return err((root, error)) ok LeafTie(root: root, path: tag) proc blobify( diff --git a/nimbus/db/kvstore_rocksdb.nim b/nimbus/db/kvstore_rocksdb.nim index 4318d3174..35bd41a38 100644 --- a/nimbus/db/kvstore_rocksdb.nim +++ b/nimbus/db/kvstore_rocksdb.nim @@ -54,8 +54,10 @@ proc contains*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] = proc del*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] = - let exists = ? store.db.keyExists(key) - if not exists: + let rc = store.db.keyExists(key) + if rc.isErr: + return rc + if not rc.value: return ok(false) let res = store.db.delete(key) diff --git a/nimbus/db/kvt/kvt_init/rocks_db/rdb_desc.nim b/nimbus/db/kvt/kvt_init/rocks_db/rdb_desc.nim index 2b66b0f13..252ab391d 100644 --- a/nimbus/db/kvt/kvt_init/rocks_db/rdb_desc.nim +++ b/nimbus/db/kvt/kvt_init/rocks_db/rdb_desc.nim @@ -39,7 +39,7 @@ func dataDir*(rdb: RdbInst): string = rdb.baseDir / DataFolder -template logTxt(info: static[string]): static[string] = +template logTxt*(info: static[string]): static[string] = "RocksDB/" & info # ------------------------------------------------------------------------------ diff --git a/nimbus/db/kvt/kvt_init/rocks_db/rdb_put.nim b/nimbus/db/kvt/kvt_init/rocks_db/rdb_put.nim index f564a7091..8e1a28f5f 100644 --- a/nimbus/db/kvt/kvt_init/rocks_db/rdb_put.nim +++ b/nimbus/db/kvt/kvt_init/rocks_db/rdb_put.nim @@ -15,7 +15,6 @@ import eth/common, - stew/byteutils, rocksdb, results, ../../kvt_desc, @@ -26,7 +25,9 @@ const ## Enable additional logging noise when extraTraceMessages: - import chronicles + import + chronicles, + stew/byteutils logScope: topics = "kvt-rocksdb" @@ -39,9 +40,10 @@ proc disposeSession(rdb: var RdbInst) = rdb.session.close() rdb.session = WriteBatchRef(nil) -proc `$`(a: Blob): string = - a.toHex - +when extraTraceMessages: + proc `$`(a: Blob): string = + a.toHex + # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ diff --git a/nimbus/db/ledger/distinct_ledgers.nim b/nimbus/db/ledger/distinct_ledgers.nim index f72869e2b..5bbe5074c 100644 --- a/nimbus/db/ledger/distinct_ledgers.nim +++ b/nimbus/db/ledger/distinct_ledgers.nim @@ -206,7 +206,6 @@ proc init*( ## https://github.com/status-im/nimbus-eth1/issues/932.) const info = "StorageLedger/init(): " - noisy = true let db = al.distinctBase.parent stt = account.stoTrie diff --git a/nimbus/nimbus.nim b/nimbus/nimbus.nim index ed4def8c8..cbdd02948 100644 --- a/nimbus/nimbus.nim +++ b/nimbus/nimbus.nim @@ -280,12 +280,16 @@ proc start(nimbus: NimbusNode, conf: NimbusConf) = evmcSetLibraryPath(conf.evm) createDir(string conf.dataDir) + let coreDB = + # Resolve statically for database type + case conf.chainDbMode: + of Prune,Archive: LegacyDbPersistent.newCoreDbRef(string conf.dataDir) + of Aristo: AristoDbRocks.newCoreDbRef(string conf.dataDir) let com = CommonRef.new( - newCoreDbRef(LegacyDbPersistent, string conf.dataDir), - conf.pruneMode == PruneMode.Full, - conf.networkId, - conf.networkParams - ) + db = coreDB, + pruneTrie = (conf.chainDbMode == ChainDbMode.Prune), + networkId = conf.networkId, + params = conf.networkParams) com.initializeEmptyDb() com.db.compensateLegacySetup() diff --git a/tests/test_beacon/setup_env.nim b/tests/test_beacon/setup_env.nim index 8f7e18573..1e60a4778 100644 --- a/tests/test_beacon/setup_env.nim +++ b/tests/test_beacon/setup_env.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at # https://opensource.org/licenses/MIT). @@ -64,7 +64,7 @@ proc setupEnv*(extraValidation: bool = false, ccm: CCModify = nil): TestEnv = let com = CommonRef.new( newCoreDbRef LegacyDbMemory, - conf.pruneMode == PruneMode.Full, + conf.chainDbMode == ChainDbMode.Prune, conf.networkId, conf.networkParams ) diff --git a/tests/test_configuration.nim b/tests/test_configuration.nim index 08f32bece..70b78eb77 100644 --- a/tests/test_configuration.nim +++ b/tests/test_configuration.nim @@ -39,18 +39,21 @@ proc configurationMain*() = check dd.dataDir.string == "apple\\bin" check dd.keyStore.string == "banana/bin" - test "prune-mode": + test "chaindb-mode": let aa = makeTestConfig() - check aa.pruneMode == PruneMode.Full + check aa.chainDbMode == ChainDbMode.Prune - let bb = makeConfig(@["--prune-mode:full"]) - check bb.pruneMode == PruneMode.Full + let bb = makeConfig(@["--chaindb:prune"]) + check bb.chainDbMode == ChainDbMode.Prune - let cc = makeConfig(@["--prune-mode:archive"]) - check cc.pruneMode == PruneMode.Archive + let cc = makeConfig(@["--chaindb:archive"]) + check cc.chainDbMode == ChainDbMode.Archive let dd = makeConfig(@["-p:archive"]) - check dd.pruneMode == PruneMode.Archive + check dd.chainDbMode == ChainDbMode.Archive + + let ee = makeConfig(@["--chaindb:aristo"]) + check ee.chainDbMode == ChainDbMode.Aristo test "import": let aa = makeTestConfig() diff --git a/tests/test_merge.nim b/tests/test_merge.nim index eb0817acd..33833b0a9 100644 --- a/tests/test_merge.nim +++ b/tests/test_merge.nim @@ -66,7 +66,7 @@ proc runTest(steps: Steps) = ethNode = setupEthNode(conf, ctx, eth) com = CommonRef.new( newCoreDbRef LegacyDbMemory, - conf.pruneMode == PruneMode.Full, + conf.chainDbMode == ChainDbMode.Prune, conf.networkId, conf.networkParams ) diff --git a/tests/test_rpc.nim b/tests/test_rpc.nim index b9780671b..a1fc34889 100644 --- a/tests/test_rpc.nim +++ b/tests/test_rpc.nim @@ -209,7 +209,7 @@ proc rpcMain*() = ethNode = setupEthNode(conf, ctx, eth) com = CommonRef.new( newCoreDbRef LegacyDbMemory, - conf.pruneMode == PruneMode.Full, + conf.chainDbMode == ChainDbMode.Prune, conf.networkId, conf.networkParams ) diff --git a/tests/test_txpool/helpers.nim b/tests/test_txpool/helpers.nim index 8903c52e8..af5667136 100644 --- a/tests/test_txpool/helpers.nim +++ b/tests/test_txpool/helpers.nim @@ -15,7 +15,7 @@ import ../replay/[pp, undump_blocks], chronicles, eth/[common, keys], - stew/[keyed_queue, sorted_set], + stew/[byteutils,keyed_queue, sorted_set], stint # Make sure that the runner can stay on public view without the need @@ -76,6 +76,10 @@ const # Helpers # ------------------------------------------------------------------------------ +proc toHex2(a: byte): string = + # Avoids compiler warning with `toHex(2)` + [a].toHex + proc joinXX(s: string): string = if s.len <= 30: return s @@ -92,13 +96,13 @@ proc toXX[T](s: T): string = s.toHex.strip(leading=true,chars={'0'}).toLowerAscii proc toXX(q: Blob): string = - q.mapIt(it.toHex(2)).join(":") + q.mapIt(it.toHex2).join(":") proc toXX(a: EthAddress): string = - a.mapIt(it.toHex(2)).joinXX + a.mapIt(it.toHex2).joinXX proc toXX(h: Hash256): string = - h.data.mapIt(it.toHex(2)).joinXX + h.data.mapIt(it.toHex2).joinXX proc toXX(v: int64; r,s: UInt256): string = v.toXX & ":" & ($r).joinXX & ":" & ($s).joinXX @@ -184,7 +188,7 @@ proc isOk*(rc: ValidationResult): bool = rc == ValidationResult.OK proc toHex*(acc: EthAddress): string = - acc.toSeq.mapIt(it.toHex(2)).join + acc.toHex proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) = if noisy: diff --git a/tests/test_txpool2.nim b/tests/test_txpool2.nim index e2f1dd008..b75900ffc 100644 --- a/tests/test_txpool2.nim +++ b/tests/test_txpool2.nim @@ -106,7 +106,7 @@ proc initEnv(envFork: HardFork): TestEnv = let com = CommonRef.new( newCoreDbRef LegacyDbMemory, - conf.pruneMode == PruneMode.Full, + conf.chainDbMode == ChainDbMode.Prune, conf.networkId, conf.networkParams )