From a441ec3cb1c7420c4ae84efbdbb31e9a95b4f8bf Mon Sep 17 00:00:00 2001 From: andri lim Date: Sun, 4 Feb 2024 21:28:20 +0700 Subject: [PATCH] Remove obsolete select_backend and fix simulators CI (#2007) * Remove obsolete select_backend * Fix copyright year --- .github/workflows/simulators.yml | 52 ++++-- hive_integration/nodocker/build_sims.sh | 2 +- nimbus/config.nim | 7 +- nimbus/core/block_import.nim | 3 +- nimbus/db/core_db/backend/legacy_rocksdb.nim | 52 +++--- nimbus/db/core_db/persistent.nim | 57 ++----- nimbus/db/select_backend.nim | 80 --------- nimbus/nimbus.nim | 1 - nimbus/stateless_runner.nim | 4 +- nimbus/sync/handlers.nim | 16 +- nimbus/sync/handlers/setup.nim | 30 ++-- nimbus/sync/handlers/snap.nim | 2 +- nimbus/sync/snap.nim | 1 - .../sync/snap/worker/db/rocky_bulk_load.nim | 157 ++++++++---------- .../sync/snap/worker/db/snapdb_accounts.nim | 18 +- .../sync/snap/worker/db/snapdb_contracts.nim | 22 +-- nimbus/sync/snap/worker/db/snapdb_desc.nim | 45 +++-- .../snap/worker/db/snapdb_storage_slots.nim | 18 +- nimbus/sync/snap/worker_desc.nim | 3 +- tests/test_sync_snap/test_accounts.nim | 3 +- tests/test_sync_snap/test_storage.nim | 3 +- 21 files changed, 228 insertions(+), 348 deletions(-) delete mode 100644 nimbus/db/select_backend.nim diff --git a/.github/workflows/simulators.yml b/.github/workflows/simulators.yml index 3c40434fc..8c9011f9f 100644 --- a/.github/workflows/simulators.yml +++ b/.github/workflows/simulators.yml @@ -27,7 +27,7 @@ jobs: id: versions run: | sudo apt-get -q update - sudo apt-get install -y libpcre3-dev + sudo apt-get install -y librocksdb-dev libpcre3-dev getHash() { git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 } @@ -66,6 +66,20 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Restore rocksdb from cache + id: rocksdb-cache + uses: actions/cache@v4 + with: + path: rocks-db-cache + key: 'rocksdb-v2' + + - name: Build and install rocksdb + run: | + HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install ccache + echo "/usr/local/opt/ccache/libexec" >> ${GITHUB_PATH} + curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_rocksdb.sh + bash build_rocksdb.sh rocks-db-cache + - name: Get latest nimbus-build-system commit hash id: versions run: | @@ -110,18 +124,37 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Restore llvm-mingw (Windows) from cache - if: runner.os == 'Windows' + - name: Restore Nim DLLs dependencies from cache + id: windows-dlls-cache + uses: actions/cache@v4 + with: + path: external/dlls + # according to docu, idle caches are kept for up to 7 days + # so change dlls# to force new cache contents (for some number #) + key: dlls0 + + - name: Install DLLs dependencies + if: steps.windows-dlls-cache.outputs.cache-hit != 'true' + run: | + ROCKSDBSUB=x64 + DLLPATH=external/dlls + mkdir -p external + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x -y external/windeps.zip -o"$DLLPATH" + # ROCKSDB + curl -L "https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/nimbus-deps.zip" -o external/nimbus-deps.zip + 7z x -y external/nimbus-deps.zip + cp "./$ROCKSDBSUB/librocksdb.dll" "$DLLPATH/librocksdb.dll" + + - name: Restore llvm-mingw from cache id: windows-mingw-cache uses: actions/cache@v4 with: path: external/mingw-amd64 key: 'mingw-llvm-17-sim' - - name: Install llvm-mingw dependency (Windows) - if: > - steps.windows-mingw-cache.outputs.cache-hit != 'true' && - runner.os == 'Windows' + - name: Install llvm-mingw dependency + if: steps.windows-mingw-cache.outputs.cache-hit != 'true' run: | mkdir -p external MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905" @@ -130,11 +163,10 @@ jobs: 7z x -y "external/mingw-amd64.zip" -oexternal/mingw-amd64/ mv external/mingw-amd64/**/* ./external/mingw-amd64 - - name: Path to cached dependencies (Windows) - if: > - runner.os == 'Windows' + - name: Path to cached dependencies run: | echo '${{ github.workspace }}'"/external/mingw-amd64/bin" >> $GITHUB_PATH + echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH - name: Get latest nimbus-build-system commit hash id: versions diff --git a/hive_integration/nodocker/build_sims.sh b/hive_integration/nodocker/build_sims.sh index 6bf65d050..64eb1ff62 100644 --- a/hive_integration/nodocker/build_sims.sh +++ b/hive_integration/nodocker/build_sims.sh @@ -17,7 +17,7 @@ USE_SYSTEM_NIM=1 ENV_SCRIPT="vendor/nimbus-build-system/scripts/env.sh" # nimbus_db_backend:none -> we only use memory db in simulators -NIM_FLAGS="c -d:release -d:nimbus_db_backend:none" +NIM_FLAGS="c -d:release" ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/engine/engine_sim ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/consensus/consensus_sim diff --git a/nimbus/config.nim b/nimbus/config.nim index 1f1d3a6ee..0c4f87d1c 100644 --- a/nimbus/config.nim +++ b/nimbus/config.nim @@ -25,9 +25,7 @@ import confutils/std/net ], eth/[common, net/utils, net/nat, p2p/bootnodes, p2p/enode, p2p/discoveryv5/enr], - "."/[db/select_backend, - constants, vm_compile_info, version - ], + "."/[constants, vm_compile_info, version], common/chain_config export net @@ -35,12 +33,11 @@ export net const # TODO: fix this agent-string format to match other # eth clients format - NimbusIdent* = "$# v$# [$#: $#, $#, $#, $#]" % [ + NimbusIdent* = "$# v$# [$#: $#, $#, $#]" % [ NimbusName, NimbusVersion, hostOS, hostCPU, - nimbus_db_backend, VmName, GitRevision ] diff --git a/nimbus/core/block_import.nim b/nimbus/core/block_import.nim index 8e28650fc..260ffbb10 100644 --- a/nimbus/core/block_import.nim +++ b/nimbus/core/block_import.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2021 Status Research & Development GmbH +# Copyright (c) 2021-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) # * MIT license ([LICENSE-MIT](LICENSE-MIT)) @@ -11,7 +11,6 @@ import chronicles, eth/rlp, stew/io2, ./chain, - ../db/select_backend, ../common/common, ../utils/utils diff --git a/nimbus/db/core_db/backend/legacy_rocksdb.nim b/nimbus/db/core_db/backend/legacy_rocksdb.nim index afa2e7266..c4a4a9075 100644 --- a/nimbus/db/core_db/backend/legacy_rocksdb.nim +++ b/nimbus/db/core_db/backend/legacy_rocksdb.nim @@ -12,39 +12,51 @@ import eth/trie/db, + eth/db/kvstore, rocksdb, - "../.."/select_backend, ../base, - ./legacy_db + ./legacy_db, + ../../kvstore_rocksdb type LegaPersDbRef = ref object of LegacyDbRef rdb: RocksStoreRef # for backend access with legacy mode -# No other backend supported -doAssert nimbus_db_backend == "rocksdb" + ChainDB = ref object of RootObj + kv: KvStoreRef + rdb: RocksStoreRef + +# TODO KvStore is a virtual interface and TrieDB is a virtual interface - one +# will be enough eventually - unless the TrieDB interface gains operations +# that are not typical to KvStores +proc get(db: ChainDB, key: openArray[byte]): seq[byte] = + var res: seq[byte] + proc onData(data: openArray[byte]) = res = @data + if db.kv.get(key, onData).expect("working database"): + return res + +proc put(db: ChainDB, key, value: openArray[byte]) = + db.kv.put(key, value).expect("working database") + +proc contains(db: ChainDB, key: openArray[byte]): bool = + db.kv.contains(key).expect("working database") + +proc del(db: ChainDB, key: openArray[byte]): bool = + db.kv.del(key).expect("working database") + +proc newChainDB(path: string): KvResult[ChainDB] = + let rdb = RocksStoreRef.init(path, "nimbus").valueOr: + return err(error) + ok(ChainDB(kv: kvStore rdb, rdb: rdb)) # ------------------------------------------------------------------------------ # Public constructor and low level data retrieval, storage & transation frame # ------------------------------------------------------------------------------ proc newLegacyPersistentCoreDbRef*(path: string): CoreDbRef = - # Kludge: Compiler bails out on `results.tryGet()` with - # :: - # fatal.nim(54) sysFatal - # Error: unhandled exception: types.nim(1251, 10) \ - # `b.kind in {tyObject} + skipPtrs` [AssertionDefect] - # - # when running `select_backend.newChainDB(path)`. The culprit seems to be - # the `ResultError` exception (or any other `CatchableError`). So this is - # converted to a `Defect`. - var backend: ChainDB - try: - {.push warning[Deprecated]: off.} - backend = newChainDB path - {.pop.} - except CatchableError as e: - let msg = "DB initialisation error(" & $e.name & "): " & e.msg + # when running `newChainDB(path)`. converted to a `Defect`. + let backend = newChainDB(path).valueOr: + let msg = "DB initialisation : " & error raise (ref ResultDefect)(msg: msg) proc done() = diff --git a/nimbus/db/core_db/persistent.nim b/nimbus/db/core_db/persistent.nim index 3f62fa649..d91b1bd5c 100644 --- a/nimbus/db/core_db/persistent.nim +++ b/nimbus/db/core_db/persistent.nim @@ -22,31 +22,15 @@ {.push raises: [].} import - ../aristo, + ../aristo, ./memory_only, - ../select_backend + base_iterators_persistent, + ./backend/[aristo_rocksdb, legacy_rocksdb] export - memory_only - -# This file is currently inconsistent due to the `dbBackend == rocksdb` hack -# which will be removed, soon (must go to the test base where such a compile -# time flag induced mechanism might be useful.) -# -# The `Aristo` backend has no business with `dbBackend` and will be extended -# in future. -{.warning: "Inconsistent API file needs to be de-uglified".} - -# Allow hive sim to compile with dbBackend == none -when dbBackend == rocksdb: - import - base_iterators_persistent, - ./backend/[aristo_rocksdb, legacy_rocksdb] - - export - base_iterators_persistent, - toRocksStoreRef - + memory_only, + base_iterators_persistent, + toRocksStoreRef proc newCoreDbRef*( dbType: static[CoreDbType]; # Database type symbol @@ -56,17 +40,14 @@ proc newCoreDbRef*( ## ## Note: Using legacy notation `newCoreDbRef()` rather than ## `CoreDbRef.init()` because of compiler coughing. - when dbBackend == rocksdb: - when dbType == LegacyDbPersistent: - newLegacyPersistentCoreDbRef path - - elif dbType == AristoDbRocks: - newAristoRocksDbCoreDbRef path - - else: - {.error: "Unsupported dbType for persistent newCoreDbRef()".} + when dbType == LegacyDbPersistent: + newLegacyPersistentCoreDbRef path + + elif dbType == AristoDbRocks: + newAristoRocksDbCoreDbRef path + else: - {.error: "Unsupported dbBackend setting for persistent newCoreDbRef()".} + {.error: "Unsupported dbType for persistent newCoreDbRef()".} proc newCoreDbRef*( dbType: static[CoreDbType]; # Database type symbol @@ -77,15 +58,11 @@ proc newCoreDbRef*( ## ## Note: Using legacy notation `newCoreDbRef()` rather than ## `CoreDbRef.init()` because of compiler coughing. - when dbBackend == rocksdb: - when dbType == AristoDbRocks: - newAristoRocksDbCoreDbRef(path, qlr) - - else: - {.error: "Unsupported dbType for persistent newCoreDbRef()" & - " with qidLayout argument".} + when dbType == AristoDbRocks: + newAristoRocksDbCoreDbRef(path, qlr) + else: - {.error: "Unsupported dbBackend setting for persistent newCoreDbRef()" & + {.error: "Unsupported dbType for persistent newCoreDbRef()" & " with qidLayout argument".} # End diff --git a/nimbus/db/select_backend.nim b/nimbus/db/select_backend.nim deleted file mode 100644 index 7188c2608..000000000 --- a/nimbus/db/select_backend.nim +++ /dev/null @@ -1,80 +0,0 @@ -# Nimbus -# Copyright (c) 2019-2023 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -import strutils, eth/db/kvstore - -export kvstore - -# Database access layer that turns errors in the database into Defects as the -# code that uses it isn't equipped to handle errors of that kind - this should -# be reconsidered when making more changes here. - -type DbBackend* = enum - none, - sqlite, - rocksdb, - lmdb - -const - nimbus_db_backend* {.strdefine.} = "rocksdb" - dbBackend* = parseEnum[DbBackend](nimbus_db_backend) - -when dbBackend == sqlite: - import eth/db/kvstore_sqlite3 as database_backend -elif dbBackend == rocksdb: - import ./kvstore_rocksdb as database_backend - -type - ChainDB* = ref object of RootObj - kv*: KvStoreRef - when dbBackend == rocksdb: - rdb*: RocksStoreRef - -# TODO KvStore is a virtual interface and TrieDB is a virtual interface - one -# will be enough eventually - unless the TrieDB interface gains operations -# that are not typical to KvStores -proc get*(db: ChainDB, key: openArray[byte]): seq[byte] = - var res: seq[byte] - proc onData(data: openArray[byte]) = res = @data - if db.kv.get(key, onData).expect("working database"): - return res - -proc put*(db: ChainDB, key, value: openArray[byte]) = - db.kv.put(key, value).expect("working database") - -proc contains*(db: ChainDB, key: openArray[byte]): bool = - db.kv.contains(key).expect("working database") - -proc del*(db: ChainDB, key: openArray[byte]): bool = - db.kv.del(key).expect("working database") - -when dbBackend == sqlite: - proc newChainDB*(path: string): ChainDB = - let db = SqStoreRef.init(path, "nimbus").expect("working database") - ChainDB(kv: kvStore db.openKvStore().expect("working database")) -elif dbBackend == rocksdb: - proc newChainDB*(path: string): ChainDB - {.gcsafe, deprecated: "use newCoreDbRef(LegacyDbPersistent,)".} = - let rdb = RocksStoreRef.init(path, "nimbus").tryGet() - ChainDB(kv: kvStore rdb, rdb: rdb) -elif dbBackend == lmdb: - # TODO This implementation has several issues on restricted platforms, possibly - # due to mmap restrictions - see: - # https://github.com/status-im/nim-beacon-chain/issues/732 - # https://github.com/status-im/nim-beacon-chain/issues/688 - # It also has other issues, including exception safety: - # https://github.com/status-im/nim-beacon-chain/pull/809 - - {.error: "lmdb deprecated, needs reimplementing".} -elif dbBackend == none: - discard - -when dbBackend != none: - export database_backend diff --git a/nimbus/nimbus.nim b/nimbus/nimbus.nim index 4ad29ee81..21fe39bb9 100644 --- a/nimbus/nimbus.nim +++ b/nimbus/nimbus.nim @@ -24,7 +24,6 @@ import ./nimbus_desc, ./core/eip4844, ./core/block_import, - ./db/select_backend, ./db/core_db/persistent, ./core/clique/clique_desc, ./core/clique/clique_sealer, diff --git a/nimbus/stateless_runner.nim b/nimbus/stateless_runner.nim index 27eb6bbd2..0655a2444 100644 --- a/nimbus/stateless_runner.nim +++ b/nimbus/stateless_runner.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -18,7 +18,7 @@ import eth/[common/eth_types, p2p], ./core/chain/chain_desc, ./core/executor/process_block, - ./db/[core_db, select_backend, ledger], + ./db/[core_db, ledger], ./evm/async/[data_sources, operations, data_sources/json_rpc_data_source], "."/[vm_state, vm_types] diff --git a/nimbus/sync/handlers.nim b/nimbus/sync/handlers.nim index 026a89258..284a35b37 100644 --- a/nimbus/sync/handlers.nim +++ b/nimbus/sync/handlers.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2021 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -9,19 +9,13 @@ # except according to those terms. import - ../db/select_backend, ./handlers/eth as handlers_eth, - ./handlers/setup as handlers_setup + ./handlers/setup as handlers_setup, + ./handlers/snap as handlers_snap export - handlers_eth, handlers_setup - -when dbBackend != select_backend.none: - import - ./handlers/snap as handlers_snap - - export - handlers_snap + handlers_eth, handlers_setup, + handlers_snap static: type diff --git a/nimbus/sync/handlers/setup.nim b/nimbus/sync/handlers/setup.nim index c2a2eef9b..ef1b0813d 100644 --- a/nimbus/sync/handlers/setup.nim +++ b/nimbus/sync/handlers/setup.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2021 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -12,7 +12,6 @@ import eth/p2p, - ../../db/select_backend, ../../core/[chain, tx_pool], ../protocol, ./eth as handlers_eth @@ -47,21 +46,20 @@ proc addEthHandlerCapability*( # Public functions: convenience mappings for `snap` # ------------------------------------------------------------------------------ -when dbBackend != select_backend.none: - import - ./snap as handlers_snap +import + ./snap as handlers_snap - proc addSnapHandlerCapability*( - node: EthereumNode; - peerPool: PeerPool; - chain = ChainRef(nil); - ) = - ## Install `snap` handlers,Passing `chein` as `nil` installs the handler - ## in minimal/outbound mode. - if chain.isNil: - node.addCapability protocol.snap - else: - node.addCapability(protocol.snap, SnapWireRef.init(chain, peerPool)) +proc addSnapHandlerCapability*( + node: EthereumNode; + peerPool: PeerPool; + chain = ChainRef(nil); + ) = + ## Install `snap` handlers,Passing `chein` as `nil` installs the handler + ## in minimal/outbound mode. + if chain.isNil: + node.addCapability protocol.snap + else: + node.addCapability(protocol.snap, SnapWireRef.init(chain, peerPool)) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/sync/handlers/snap.nim b/nimbus/sync/handlers/snap.nim index e306cd3a9..1ed614950 100644 --- a/nimbus/sync/handlers/snap.nim +++ b/nimbus/sync/handlers/snap.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2021 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) diff --git a/nimbus/sync/snap.nim b/nimbus/sync/snap.nim index b5fbbcc91..306de3224 100644 --- a/nimbus/sync/snap.nim +++ b/nimbus/sync/snap.nim @@ -14,7 +14,6 @@ import chronicles, chronos, eth/p2p, - ../db/select_backend, ../core/chain, ./snap/[worker, worker_desc], "."/[protocol, sync_sched] diff --git a/nimbus/sync/snap/worker/db/rocky_bulk_load.nim b/nimbus/sync/snap/worker/db/rocky_bulk_load.nim index 536be8cb6..b2ea6e087 100644 --- a/nimbus/sync/snap/worker/db/rocky_bulk_load.nim +++ b/nimbus/sync/snap/worker/db/rocky_bulk_load.nim @@ -1,5 +1,5 @@ # nimbus-eth1 -# Copyright (c) 2018-2021 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -14,18 +14,17 @@ import std/os, # std/[sequtils, strutils], eth/common/eth_types, rocksdb, - ../../../../db/[kvstore_rocksdb, select_backend] + ../../../../db/kvstore_rocksdb {.push raises: [].} type RockyBulkLoadRef* = ref object of RootObj - when select_backend.dbBackend == select_backend.rocksdb: - db: RocksStoreRef - envOption: rocksdb_envoptions_t - importOption: rocksdb_ingestexternalfileoptions_t - writer: rocksdb_sstfilewriter_t - filePath: string + db: RocksStoreRef + envOption: rocksdb_envoptions_t + importOption: rocksdb_ingestexternalfileoptions_t + writer: rocksdb_sstfilewriter_t + filePath: string csError: string # ------------------------------------------------------------------------------ @@ -38,16 +37,13 @@ proc init*( envOption: rocksdb_envoptions_t ): T = ## Create a new bulk load descriptor. - when select_backend.dbBackend == select_backend.rocksdb: - result = T( - db: db, - envOption: envOption, - importOption: rocksdb_ingestexternalfileoptions_create()) + result = T( + db: db, + envOption: envOption, + importOption: rocksdb_ingestexternalfileoptions_create()) - doAssert not result.importOption.isNil - doAssert not envOption.isNil - else: - T(csError: "rocksdb is unsupported") + doAssert not result.importOption.isNil + doAssert not envOption.isNil proc init*(T: type RockyBulkLoadRef; db: RocksStoreRef): T = ## Variant of `init()` @@ -57,12 +53,10 @@ proc clearCacheFile*(db: RocksStoreRef; fileName: string): bool {.gcsafe, raises: [OSError].} = ## Remove left-over cache file from an imcomplete previous session. The ## return value `true` indicated that a cache file was detected. - discard - when select_backend.dbBackend == select_backend.rocksdb: - let filePath = db.tmpDir / fileName - if filePath.fileExists: - filePath.removeFile - return true + let filePath = db.tmpDir / fileName + if filePath.fileExists: + filePath.removeFile + return true proc destroy*(rbl: RockyBulkLoadRef) {.gcsafe, raises: [OSError].} = ## Destructor, free memory resources and delete temporary file. This function @@ -73,17 +67,15 @@ proc destroy*(rbl: RockyBulkLoadRef) {.gcsafe, raises: [OSError].} = ## reset and must not be used anymore with any function (different from ## `destroy()`.) ## - discard - when select_backend.dbBackend == select_backend.rocksdb: - if not rbl.writer.isNil: - rbl.writer.rocksdb_sstfilewriter_destroy() - if not rbl.envOption.isNil: - rbl.envOption.rocksdb_envoptions_destroy() - if not rbl.importOption.isNil: - rbl.importOption.rocksdb_ingestexternalfileoptions_destroy() - if 0 < rbl.filePath.len: - rbl.filePath.removeFile - rbl[].reset + if not rbl.writer.isNil: + rbl.writer.rocksdb_sstfilewriter_destroy() + if not rbl.envOption.isNil: + rbl.envOption.rocksdb_envoptions_destroy() + if not rbl.importOption.isNil: + rbl.importOption.rocksdb_ingestexternalfileoptions_destroy() + if 0 < rbl.filePath.len: + rbl.filePath.removeFile + rbl[].reset # ------------------------------------------------------------------------------ # Public functions, getters @@ -95,17 +87,7 @@ proc lastError*(rbl: RockyBulkLoadRef): string = proc store*(rbl: RockyBulkLoadRef): RocksDBInstance = ## Provide the diecriptor for backend functions as defined in `rocksdb`. - discard - when select_backend.dbBackend == select_backend.rocksdb: - rbl.db.store - -proc rocksStoreRef*(db: ChainDb): RocksStoreRef = - ## Pull out underlying rocksdb backend descripto (if any) - # Current architecture allows only one globally defined persistent type - discard - when select_backend.dbBackend == select_backend.rocksdb: - if not db.isNil: - return db.rdb + rbl.db.store # ------------------------------------------------------------------------------ # Public functions @@ -115,24 +97,22 @@ proc begin*(rbl: RockyBulkLoadRef; fileName: string): bool = ## Begin a new bulk load session storing data into a temporary cache file ## `fileName`. When finished, this file will bi direcly imported into the ## database. - discard - when select_backend.dbBackend == select_backend.rocksdb: - rbl.writer = rocksdb_sstfilewriter_create( - rbl.envOption, rbl.db.store.options) - if rbl.writer.isNil: - rbl.csError = "Cannot create sst writer session" - return false + rbl.writer = rocksdb_sstfilewriter_create( + rbl.envOption, rbl.db.store.options) + if rbl.writer.isNil: + rbl.csError = "Cannot create sst writer session" + return false - rbl.csError = "" - let filePath = rbl.db.tmpDir / fileName - var csError: cstring - rbl.writer.rocksdb_sstfilewriter_open(fileName, addr csError) - if not csError.isNil: - rbl.csError = $csError - return false + rbl.csError = "" + let filePath = rbl.db.tmpDir / fileName + var csError: cstring + rbl.writer.rocksdb_sstfilewriter_open(fileName, addr csError) + if not csError.isNil: + rbl.csError = $csError + return false - rbl.filePath = filePath - return true + rbl.filePath = filePath + return true proc add*( rbl: RockyBulkLoadRef; @@ -145,16 +125,14 @@ proc add*( ## This function is a wrapper around `rocksdb_sstfilewriter_add()` or ## `rocksdb_sstfilewriter_put()` (stragely enough, there are two functions ## with exactly the same impementation code.) - discard - when select_backend.dbBackend == select_backend.rocksdb: - var csError: cstring - rbl.writer.rocksdb_sstfilewriter_add( - cast[cstring](unsafeAddr key[0]), csize_t(key.len), - cast[cstring](unsafeAddr val[0]), csize_t(val.len), - addr csError) - if csError.isNil: - return true - rbl.csError = $csError + var csError: cstring + rbl.writer.rocksdb_sstfilewriter_add( + cast[cstring](unsafeAddr key[0]), csize_t(key.len), + cast[cstring](unsafeAddr val[0]), csize_t(val.len), + addr csError) + if csError.isNil: + return true + rbl.csError = $csError proc finish*( rbl: RockyBulkLoadRef @@ -166,29 +144,26 @@ proc finish*( ## ## If successful, the return value is the size of the SST file used if ## that value is available. Otherwise, `0` is returned. - when select_backend.dbBackend == select_backend.rocksdb: - var csError: cstring - rbl.writer.rocksdb_sstfilewriter_finish(addr csError) + var csError: cstring + rbl.writer.rocksdb_sstfilewriter_finish(addr csError) + + if csError.isNil: + rbl.db.store.db.rocksdb_ingest_external_file( + [rbl.filePath].allocCStringArray, 1, + rbl.importOption, + addr csError) if csError.isNil: - rbl.db.store.db.rocksdb_ingest_external_file( - [rbl.filePath].allocCStringArray, 1, - rbl.importOption, - addr csError) + var + size: int64 + f: File + if f.open(rbl.filePath): + size = f.getFileSize + f.close + rbl.destroy() + return ok(size) - if csError.isNil: - var - size: int64 - f: File - if f.open(rbl.filePath): - size = f.getFileSize - f.close - rbl.destroy() - return ok(size) - - rbl.csError = $csError - - err() + rbl.csError = $csError # ------------------------------------------------------------------------------ # End diff --git a/nimbus/sync/snap/worker/db/snapdb_accounts.nim b/nimbus/sync/snap/worker/db/snapdb_accounts.nim index f84d64971..a1bddb057 100644 --- a/nimbus/sync/snap/worker/db/snapdb_accounts.nim +++ b/nimbus/sync/snap/worker/db/snapdb_accounts.nim @@ -20,9 +20,6 @@ import hexary_interpolate, hexary_inspect, hexary_paths, snapdb_desc, snapdb_persistent] -import - ../../../../db/select_backend - logScope: topics = "snap-db" @@ -66,14 +63,13 @@ proc persistentAccounts( ): Result[void,HexaryError] {.gcsafe, raises: [OSError,IOError,KeyError].} = ## Store accounts trie table on databse - when dbBackend == rocksdb: - if ps.rockDb.isNil: - let rc = db.persistentAccountsPut(ps.kvDb) - if rc.isErr: return rc - else: - let rc = db.persistentAccountsPut(ps.rockDb) - if rc.isErr: return rc - ok() + if ps.rockDb.isNil: + let rc = db.persistentAccountsPut(ps.kvDb) + if rc.isErr: return rc + else: + let rc = db.persistentAccountsPut(ps.rockDb) + if rc.isErr: return rc + ok() proc collectAccounts( diff --git a/nimbus/sync/snap/worker/db/snapdb_contracts.nim b/nimbus/sync/snap/worker/db/snapdb_contracts.nim index 4de9e5b21..1980a63a2 100644 --- a/nimbus/sync/snap/worker/db/snapdb_contracts.nim +++ b/nimbus/sync/snap/worker/db/snapdb_contracts.nim @@ -16,9 +16,6 @@ import ../../range_desc, "."/[hexary_desc, hexary_error, snapdb_desc, snapdb_persistent] -import - ../../../../db/select_backend - logScope: topics = "snap-db" @@ -49,16 +46,15 @@ proc persistentContracts( ): Result[void,HexaryError] {.gcsafe, raises: [OSError,IOError,KeyError].} = ## Store contract codes onto permanent database - when dbBackend == rocksdb: - if ps.rockDb.isNil: - let rc = data.persistentContractPut ps.kvDb - if rc.isErr: - return rc - else: - let rc = data.persistentContractPut ps.rockDb - if rc.isErr: - return rc - ok() + if ps.rockDb.isNil: + let rc = data.persistentContractPut ps.kvDb + if rc.isErr: + return rc + else: + let rc = data.persistentContractPut ps.rockDb + if rc.isErr: + return rc + ok() # ------------------------------------------------------------------------------ # Public constructor diff --git a/nimbus/sync/snap/worker/db/snapdb_desc.nim b/nimbus/sync/snap/worker/db/snapdb_desc.nim index b60b80871..53cd57be9 100644 --- a/nimbus/sync/snap/worker/db/snapdb_desc.nim +++ b/nimbus/sync/snap/worker/db/snapdb_desc.nim @@ -15,7 +15,7 @@ import chronicles, eth/[common, p2p, trie/nibbles], ../../../../db/core_db/persistent, - ../../../../db/[core_db, select_backend, storage_types], + ../../../../db/[core_db, storage_types, kvstore_rocksdb], ../../../protocol, ../../range_desc, "."/[hexary_desc, hexary_error, hexary_import, hexary_nearby, hexary_paths, @@ -35,9 +35,7 @@ type ## Global, re-usable descriptor keyMap: Table[RepairKey,uint] ## For debugging only (will go away) db: CoreDbRef ## General database - # Allow hive sim to compile with dbBackend == none - when dbBackend == rocksdb: - rocky: RocksStoreRef ## Set if rocksdb is available + rocky: RocksStoreRef ## Set if rocksdb is available SnapDbBaseRef* = ref object of RootRef ## Session descriptor @@ -71,15 +69,14 @@ proc keyPp(a: RepairKey; pv: SnapDbRef): string = # Private helper # ------------------------------------------------------------------------------ -when dbBackend == rocksdb: - proc clearRockyCacheFile(rocky: RocksStoreRef): bool = - if not rocky.isNil: - # A cache file might hang about from a previous crash - try: - discard rocky.clearCacheFile(RockyBulkCache) - return true - except OSError as e: - error "Cannot clear rocksdb cache", exception=($e.name), msg=e.msg +proc clearRockyCacheFile(rocky: RocksStoreRef): bool = + if not rocky.isNil: + # A cache file might hang about from a previous crash + try: + discard rocky.clearCacheFile(RockyBulkCache) + return true + except OSError as e: + error "Cannot clear rocksdb cache", exception=($e.name), msg=e.msg # ------------------------------------------------------------------------------ # Public constructor @@ -90,8 +87,7 @@ proc init*( db: CoreDbRef ): T = ## Main object constructor - when dbBackend == rocksdb: - T(db: db, rocky: db.backend.toRocksStoreRef) + T(db: db, rocky: db.backend.toRocksStoreRef) proc init*( T: type HexaryTreeDbRef; @@ -138,14 +134,13 @@ proc hexaDb*(ps: SnapDbBaseRef): HexaryTreeDbRef = ## Getter, low level access to underlying session DB ps.xDb -when dbBackend == rocksdb: - proc rockDb*(ps: SnapDbBaseRef): RocksStoreRef = - ## Getter, low level access to underlying persistent rock DB interface - ps.base.rocky +proc rockDb*(ps: SnapDbBaseRef): RocksStoreRef = + ## Getter, low level access to underlying persistent rock DB interface + ps.base.rocky - proc rockDb*(pv: SnapDbRef): RocksStoreRef = - ## Getter variant - pv.rocky +proc rockDb*(pv: SnapDbRef): RocksStoreRef = + ## Getter variant + pv.rocky proc kvDb*(ps: SnapDbBaseRef): CoreDbRef = ## Getter, low level access to underlying persistent key-value DB @@ -198,13 +193,11 @@ template toOpenArray*(k: ByteArray33): openArray[byte] = proc dbBackendRocksDb*(pv: SnapDbRef): bool = ## Returns `true` if rocksdb features are available - when dbBackend == rocksdb: - not pv.rocky.isNil + not pv.rocky.isNil proc dbBackendRocksDb*(ps: SnapDbBaseRef): bool = ## Returns `true` if rocksdb features are available - when dbBackend == rocksdb: - not ps.base.rocky.isNil + not ps.base.rocky.isNil proc mergeProofs*( xDb: HexaryTreeDbRef; ## Session database diff --git a/nimbus/sync/snap/worker/db/snapdb_storage_slots.nim b/nimbus/sync/snap/worker/db/snapdb_storage_slots.nim index 9edda6a3d..40a2c1a12 100644 --- a/nimbus/sync/snap/worker/db/snapdb_storage_slots.nim +++ b/nimbus/sync/snap/worker/db/snapdb_storage_slots.nim @@ -21,9 +21,6 @@ import hexary_inspect, hexary_interpolate, hexary_paths, snapdb_desc, snapdb_persistent] -import - ../../../../db/select_backend - logScope: topics = "snap-db" @@ -63,14 +60,13 @@ proc persistentStorageSlots( ): Result[void,HexaryError] {.gcsafe, raises: [OSError,IOError,KeyError].} = ## Store accounts trie table on databse - when dbBackend == rocksdb: - if ps.rockDb.isNil: - let rc = db.persistentStorageSlotsPut(ps.kvDb) - if rc.isErr: return rc - else: - let rc = db.persistentStorageSlotsPut(ps.rockDb) - if rc.isErr: return rc - ok() + if ps.rockDb.isNil: + let rc = db.persistentStorageSlotsPut(ps.kvDb) + if rc.isErr: return rc + else: + let rc = db.persistentStorageSlotsPut(ps.rockDb) + if rc.isErr: return rc + ok() proc collectStorageSlots( diff --git a/nimbus/sync/snap/worker_desc.nim b/nimbus/sync/snap/worker_desc.nim index 50277d97d..9b10bdb0b 100644 --- a/nimbus/sync/snap/worker_desc.nim +++ b/nimbus/sync/snap/worker_desc.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2021 Status Research & Development GmbH +# Copyright (c) 2021-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -12,7 +12,6 @@ import eth/[common, p2p], - ../../db/select_backend, ../misc/ticker, ../sync_desc, ./worker/get/get_error, diff --git a/tests/test_sync_snap/test_accounts.nim b/tests/test_sync_snap/test_accounts.nim index f2338874e..95dddb7db 100644 --- a/tests/test_sync_snap/test_accounts.nim +++ b/tests/test_sync_snap/test_accounts.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2022-2023 Status Research & Development GmbH +# Copyright (c) 2022-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -60,7 +60,6 @@ import std/algorithm, eth/[common, p2p], unittest2, - ../../nimbus/db/select_backend, ../../nimbus/sync/protocol, ../../nimbus/sync/snap/range_desc, ../../nimbus/sync/snap/worker/db/[ diff --git a/tests/test_sync_snap/test_storage.nim b/tests/test_sync_snap/test_storage.nim index 23288dbfe..5ec7d58e9 100644 --- a/tests/test_sync_snap/test_storage.nim +++ b/tests/test_sync_snap/test_storage.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2022-2023 Status Research & Development GmbH +# Copyright (c) 2022-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -14,7 +14,6 @@ import std/[sequtils, tables], eth/[common, p2p], unittest2, - ../../nimbus/db/select_backend, ../../nimbus/sync/snap/range_desc, ../../nimbus/sync/snap/worker/db/[ hexary_desc, hexary_error, hexary_inspect,