Generalise prune mode option 4 different db models (#2139)
* Update README * Nimbus-main: replaced `PruneMode` options by `ChainDbMode` options details: For the legacy database, this changes the phrase - `conf.pruneMode == PruneMode.Full` to the expression + `conf.chainDbMode == ChainDbMode.Prune`. * Fix issues moaned about by NIM compiler * Fix copyright year
This commit is contained in:
parent
c610053a09
commit
e8eb3268f5
|
@ -230,9 +230,6 @@ available.)
|
||||||
Disable legacy chunked RLPx messages which are enabled by default for
|
Disable legacy chunked RLPx messages which are enabled by default for
|
||||||
synchronising against `Nethermind` nodes
|
synchronising against `Nethermind` nodes
|
||||||
|
|
||||||
* ENABLE_ETH_VERSION=66<br>
|
|
||||||
Enable legacy protocol `eth66` (or whatever other protocol version.)
|
|
||||||
|
|
||||||
* ENABLE_EVMC=1<br>
|
* ENABLE_EVMC=1<br>
|
||||||
Enable mostly EVMC compliant wrapper around the native Nim VM
|
Enable mostly EVMC compliant wrapper around the native Nim VM
|
||||||
|
|
||||||
|
@ -244,6 +241,12 @@ available.)
|
||||||
For these variables, using <variable>=0 is ignored and <variable>=2
|
For these variables, using <variable>=0 is ignored and <variable>=2
|
||||||
has the same effect as <variable>=1 (ditto for other numbers.)
|
has the same effect as <variable>=1 (ditto for other numbers.)
|
||||||
|
|
||||||
|
Other settings where the non-zero value matters:
|
||||||
|
|
||||||
|
* ENABLE_ETH_VERSION=66<br>
|
||||||
|
Enable legacy protocol `eth66` (or another available protocol version.)
|
||||||
|
|
||||||
|
|
||||||
### <a name="devel-tips"></a>Development tips
|
### <a name="devel-tips"></a>Development tips
|
||||||
|
|
||||||
Interesting Make variables and targets are documented in the [nimbus-build-system](https://github.com/status-im/nimbus-build-system) repo.
|
Interesting Make variables and targets are documented in the [nimbus-build-system](https://github.com/status-im/nimbus-build-system) repo.
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
# Nimbus Fluffy book
|
# Nimbus Fluffy book
|
||||||
# Copyright (c) 2023 Status Research & Development GmbH
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||||
# Licensed and distributed under either of
|
# Licensed and distributed under either of
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||||
|
|
|
@ -59,7 +59,7 @@ const
|
||||||
proc makeCom*(conf: NimbusConf): CommonRef =
|
proc makeCom*(conf: NimbusConf): CommonRef =
|
||||||
CommonRef.new(
|
CommonRef.new(
|
||||||
newCoreDbRef LegacyDbMemory,
|
newCoreDbRef LegacyDbMemory,
|
||||||
conf.pruneMode == PruneMode.Full,
|
conf.chainDbMode == ChainDbMode.Prune,
|
||||||
conf.networkId,
|
conf.networkId,
|
||||||
conf.networkParams
|
conf.networkParams
|
||||||
)
|
)
|
||||||
|
|
|
@ -53,11 +53,11 @@ proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) =
|
||||||
t.com = CommonRef.new(
|
t.com = CommonRef.new(
|
||||||
memDB,
|
memDB,
|
||||||
conf,
|
conf,
|
||||||
t.conf.pruneMode == PruneMode.Full
|
t.conf.chainDbMode == ChainDbMode.Prune
|
||||||
)
|
)
|
||||||
t.chainRef = newChain(t.com, extraValidation = true)
|
t.chainRef = newChain(t.com, extraValidation = true)
|
||||||
let
|
let
|
||||||
stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.pruneMode == PruneMode.Full)
|
stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.chainDbMode == ChainDbMode.Prune)
|
||||||
genesisHeader = node.genesisHeader
|
genesisHeader = node.genesisHeader
|
||||||
|
|
||||||
setupStateDB(node["pre"], stateDB)
|
setupStateDB(node["pre"], stateDB)
|
||||||
|
|
|
@ -61,7 +61,7 @@ proc stopRpcHttpServer(srv: RpcServer) =
|
||||||
|
|
||||||
proc setupEnv*(): TestEnv =
|
proc setupEnv*(): TestEnv =
|
||||||
let conf = makeConfig(@[
|
let conf = makeConfig(@[
|
||||||
"--prune-mode:archive",
|
"--chaindb:archive",
|
||||||
# "--nat:extip:0.0.0.0",
|
# "--nat:extip:0.0.0.0",
|
||||||
"--network:7",
|
"--network:7",
|
||||||
"--import-key:" & initPath / "private-key",
|
"--import-key:" & initPath / "private-key",
|
||||||
|
@ -77,7 +77,7 @@ proc setupEnv*(): TestEnv =
|
||||||
ethCtx = newEthContext()
|
ethCtx = newEthContext()
|
||||||
ethNode = setupEthNode(conf, ethCtx, eth)
|
ethNode = setupEthNode(conf, ethCtx, eth)
|
||||||
com = CommonRef.new(newCoreDbRef LegacyDbMemory,
|
com = CommonRef.new(newCoreDbRef LegacyDbMemory,
|
||||||
conf.pruneMode == PruneMode.Full,
|
conf.chainDbMode == ChainDbMode.Prune,
|
||||||
conf.networkId,
|
conf.networkId,
|
||||||
conf.networkParams
|
conf.networkParams
|
||||||
)
|
)
|
||||||
|
|
|
@ -105,9 +105,10 @@ const sharedLibText = if defined(linux): " (*.so, *.so.N)"
|
||||||
else: ""
|
else: ""
|
||||||
|
|
||||||
type
|
type
|
||||||
PruneMode* {.pure.} = enum
|
ChainDbMode* {.pure.} = enum
|
||||||
Full
|
Prune
|
||||||
Archive
|
Archive
|
||||||
|
Aristo
|
||||||
|
|
||||||
NimbusCmd* {.pure.} = enum
|
NimbusCmd* {.pure.} = enum
|
||||||
noCommand
|
noCommand
|
||||||
|
@ -154,12 +155,17 @@ type
|
||||||
abbr: "k"
|
abbr: "k"
|
||||||
name: "key-store" }: OutDir
|
name: "key-store" }: OutDir
|
||||||
|
|
||||||
pruneMode* {.
|
chainDbMode* {.
|
||||||
desc: "Blockchain prune mode (Full or Archive)"
|
desc: "Blockchain database"
|
||||||
defaultValue: PruneMode.Full
|
longDesc:
|
||||||
defaultValueDesc: $PruneMode.Full
|
"- Prune -- Legacy/reference database, full pruning\n" &
|
||||||
|
"- Archive -- Legacy/reference database without pruning\n" &
|
||||||
|
"- Aristo -- Experimental single state DB\n" &
|
||||||
|
""
|
||||||
|
defaultValue: ChainDbMode.Prune
|
||||||
|
defaultValueDesc: $ChainDbMode.Prune
|
||||||
abbr : "p"
|
abbr : "p"
|
||||||
name: "prune-mode" }: PruneMode
|
name: "chaindb" }: ChainDbMode
|
||||||
|
|
||||||
syncMode* {.
|
syncMode* {.
|
||||||
desc: "Specify particular blockchain sync mode."
|
desc: "Specify particular blockchain sync mode."
|
||||||
|
|
|
@ -60,10 +60,6 @@ when extraTraceMessages:
|
||||||
# Private helpers
|
# Private helpers
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
template logTxt(info: static[string]): static[string] =
|
|
||||||
"RocksDB " & info
|
|
||||||
|
|
||||||
|
|
||||||
proc newSession(db: RdbBackendRef): RdbPutHdlRef =
|
proc newSession(db: RdbBackendRef): RdbPutHdlRef =
|
||||||
new result
|
new result
|
||||||
result.TypedPutHdlRef.beginSession db
|
result.TypedPutHdlRef.beginSession db
|
||||||
|
|
|
@ -15,19 +15,12 @@
|
||||||
|
|
||||||
import
|
import
|
||||||
eth/common,
|
eth/common,
|
||||||
rocksdb/lib/librocksdb,
|
|
||||||
rocksdb,
|
rocksdb,
|
||||||
results,
|
results,
|
||||||
../../aristo_desc,
|
../../aristo_desc,
|
||||||
../init_common,
|
../init_common,
|
||||||
./rdb_desc
|
./rdb_desc
|
||||||
|
|
||||||
type
|
|
||||||
RdbPutSession = object
|
|
||||||
writer: ptr rocksdb_sstfilewriter_t
|
|
||||||
sstPath: string
|
|
||||||
nRecords: int
|
|
||||||
|
|
||||||
const
|
const
|
||||||
extraTraceMessages = false
|
extraTraceMessages = false
|
||||||
## Enable additional logging noise
|
## Enable additional logging noise
|
||||||
|
|
|
@ -141,15 +141,12 @@ when EnableDebugLog:
|
||||||
|
|
||||||
# -------------------------
|
# -------------------------
|
||||||
|
|
||||||
func getOrVoid(tab: TableRef[Blob,Blob]; w: openArray[byte]): Blob =
|
|
||||||
tab.getOrDefault(@w, EmptyBlob)
|
|
||||||
|
|
||||||
func leafTie(
|
func leafTie(
|
||||||
root: VertexID;
|
root: VertexID;
|
||||||
path: openArray[byte];
|
path: openArray[byte];
|
||||||
): Result[LeafTie,(VertexID,AristoError)] =
|
): Result[LeafTie,(VertexID,AristoError)] =
|
||||||
let tag = path.pathToTag.valueOr:
|
let tag = path.pathToTag.valueOr:
|
||||||
return err((VertexID(root), error))
|
return err((root, error))
|
||||||
ok LeafTie(root: root, path: tag)
|
ok LeafTie(root: root, path: tag)
|
||||||
|
|
||||||
proc blobify(
|
proc blobify(
|
||||||
|
|
|
@ -54,8 +54,10 @@ proc contains*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] =
|
||||||
|
|
||||||
proc del*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] =
|
proc del*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] =
|
||||||
|
|
||||||
let exists = ? store.db.keyExists(key)
|
let rc = store.db.keyExists(key)
|
||||||
if not exists:
|
if rc.isErr:
|
||||||
|
return rc
|
||||||
|
if not rc.value:
|
||||||
return ok(false)
|
return ok(false)
|
||||||
|
|
||||||
let res = store.db.delete(key)
|
let res = store.db.delete(key)
|
||||||
|
|
|
@ -39,7 +39,7 @@ func dataDir*(rdb: RdbInst): string =
|
||||||
rdb.baseDir / DataFolder
|
rdb.baseDir / DataFolder
|
||||||
|
|
||||||
|
|
||||||
template logTxt(info: static[string]): static[string] =
|
template logTxt*(info: static[string]): static[string] =
|
||||||
"RocksDB/" & info
|
"RocksDB/" & info
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
|
@ -15,7 +15,6 @@
|
||||||
|
|
||||||
import
|
import
|
||||||
eth/common,
|
eth/common,
|
||||||
stew/byteutils,
|
|
||||||
rocksdb,
|
rocksdb,
|
||||||
results,
|
results,
|
||||||
../../kvt_desc,
|
../../kvt_desc,
|
||||||
|
@ -26,7 +25,9 @@ const
|
||||||
## Enable additional logging noise
|
## Enable additional logging noise
|
||||||
|
|
||||||
when extraTraceMessages:
|
when extraTraceMessages:
|
||||||
import chronicles
|
import
|
||||||
|
chronicles,
|
||||||
|
stew/byteutils
|
||||||
|
|
||||||
logScope:
|
logScope:
|
||||||
topics = "kvt-rocksdb"
|
topics = "kvt-rocksdb"
|
||||||
|
@ -39,6 +40,7 @@ proc disposeSession(rdb: var RdbInst) =
|
||||||
rdb.session.close()
|
rdb.session.close()
|
||||||
rdb.session = WriteBatchRef(nil)
|
rdb.session = WriteBatchRef(nil)
|
||||||
|
|
||||||
|
when extraTraceMessages:
|
||||||
proc `$`(a: Blob): string =
|
proc `$`(a: Blob): string =
|
||||||
a.toHex
|
a.toHex
|
||||||
|
|
||||||
|
|
|
@ -206,7 +206,6 @@ proc init*(
|
||||||
## https://github.com/status-im/nimbus-eth1/issues/932.)
|
## https://github.com/status-im/nimbus-eth1/issues/932.)
|
||||||
const
|
const
|
||||||
info = "StorageLedger/init(): "
|
info = "StorageLedger/init(): "
|
||||||
noisy = true
|
|
||||||
let
|
let
|
||||||
db = al.distinctBase.parent
|
db = al.distinctBase.parent
|
||||||
stt = account.stoTrie
|
stt = account.stoTrie
|
||||||
|
|
|
@ -280,12 +280,16 @@ proc start(nimbus: NimbusNode, conf: NimbusConf) =
|
||||||
evmcSetLibraryPath(conf.evm)
|
evmcSetLibraryPath(conf.evm)
|
||||||
|
|
||||||
createDir(string conf.dataDir)
|
createDir(string conf.dataDir)
|
||||||
|
let coreDB =
|
||||||
|
# Resolve statically for database type
|
||||||
|
case conf.chainDbMode:
|
||||||
|
of Prune,Archive: LegacyDbPersistent.newCoreDbRef(string conf.dataDir)
|
||||||
|
of Aristo: AristoDbRocks.newCoreDbRef(string conf.dataDir)
|
||||||
let com = CommonRef.new(
|
let com = CommonRef.new(
|
||||||
newCoreDbRef(LegacyDbPersistent, string conf.dataDir),
|
db = coreDB,
|
||||||
conf.pruneMode == PruneMode.Full,
|
pruneTrie = (conf.chainDbMode == ChainDbMode.Prune),
|
||||||
conf.networkId,
|
networkId = conf.networkId,
|
||||||
conf.networkParams
|
params = conf.networkParams)
|
||||||
)
|
|
||||||
|
|
||||||
com.initializeEmptyDb()
|
com.initializeEmptyDb()
|
||||||
com.db.compensateLegacySetup()
|
com.db.compensateLegacySetup()
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
# Nimbus
|
# Nimbus
|
||||||
# Copyright (c) 2023 Status Research & Development GmbH
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||||
# Licensed and distributed under either of
|
# Licensed and distributed under either of
|
||||||
# * MIT license (license terms in the root directory or at
|
# * MIT license (license terms in the root directory or at
|
||||||
# https://opensource.org/licenses/MIT).
|
# https://opensource.org/licenses/MIT).
|
||||||
|
@ -64,7 +64,7 @@ proc setupEnv*(extraValidation: bool = false, ccm: CCModify = nil): TestEnv =
|
||||||
let
|
let
|
||||||
com = CommonRef.new(
|
com = CommonRef.new(
|
||||||
newCoreDbRef LegacyDbMemory,
|
newCoreDbRef LegacyDbMemory,
|
||||||
conf.pruneMode == PruneMode.Full,
|
conf.chainDbMode == ChainDbMode.Prune,
|
||||||
conf.networkId,
|
conf.networkId,
|
||||||
conf.networkParams
|
conf.networkParams
|
||||||
)
|
)
|
||||||
|
|
|
@ -39,18 +39,21 @@ proc configurationMain*() =
|
||||||
check dd.dataDir.string == "apple\\bin"
|
check dd.dataDir.string == "apple\\bin"
|
||||||
check dd.keyStore.string == "banana/bin"
|
check dd.keyStore.string == "banana/bin"
|
||||||
|
|
||||||
test "prune-mode":
|
test "chaindb-mode":
|
||||||
let aa = makeTestConfig()
|
let aa = makeTestConfig()
|
||||||
check aa.pruneMode == PruneMode.Full
|
check aa.chainDbMode == ChainDbMode.Prune
|
||||||
|
|
||||||
let bb = makeConfig(@["--prune-mode:full"])
|
let bb = makeConfig(@["--chaindb:prune"])
|
||||||
check bb.pruneMode == PruneMode.Full
|
check bb.chainDbMode == ChainDbMode.Prune
|
||||||
|
|
||||||
let cc = makeConfig(@["--prune-mode:archive"])
|
let cc = makeConfig(@["--chaindb:archive"])
|
||||||
check cc.pruneMode == PruneMode.Archive
|
check cc.chainDbMode == ChainDbMode.Archive
|
||||||
|
|
||||||
let dd = makeConfig(@["-p:archive"])
|
let dd = makeConfig(@["-p:archive"])
|
||||||
check dd.pruneMode == PruneMode.Archive
|
check dd.chainDbMode == ChainDbMode.Archive
|
||||||
|
|
||||||
|
let ee = makeConfig(@["--chaindb:aristo"])
|
||||||
|
check ee.chainDbMode == ChainDbMode.Aristo
|
||||||
|
|
||||||
test "import":
|
test "import":
|
||||||
let aa = makeTestConfig()
|
let aa = makeTestConfig()
|
||||||
|
|
|
@ -66,7 +66,7 @@ proc runTest(steps: Steps) =
|
||||||
ethNode = setupEthNode(conf, ctx, eth)
|
ethNode = setupEthNode(conf, ctx, eth)
|
||||||
com = CommonRef.new(
|
com = CommonRef.new(
|
||||||
newCoreDbRef LegacyDbMemory,
|
newCoreDbRef LegacyDbMemory,
|
||||||
conf.pruneMode == PruneMode.Full,
|
conf.chainDbMode == ChainDbMode.Prune,
|
||||||
conf.networkId,
|
conf.networkId,
|
||||||
conf.networkParams
|
conf.networkParams
|
||||||
)
|
)
|
||||||
|
|
|
@ -209,7 +209,7 @@ proc rpcMain*() =
|
||||||
ethNode = setupEthNode(conf, ctx, eth)
|
ethNode = setupEthNode(conf, ctx, eth)
|
||||||
com = CommonRef.new(
|
com = CommonRef.new(
|
||||||
newCoreDbRef LegacyDbMemory,
|
newCoreDbRef LegacyDbMemory,
|
||||||
conf.pruneMode == PruneMode.Full,
|
conf.chainDbMode == ChainDbMode.Prune,
|
||||||
conf.networkId,
|
conf.networkId,
|
||||||
conf.networkParams
|
conf.networkParams
|
||||||
)
|
)
|
||||||
|
|
|
@ -15,7 +15,7 @@ import
|
||||||
../replay/[pp, undump_blocks],
|
../replay/[pp, undump_blocks],
|
||||||
chronicles,
|
chronicles,
|
||||||
eth/[common, keys],
|
eth/[common, keys],
|
||||||
stew/[keyed_queue, sorted_set],
|
stew/[byteutils,keyed_queue, sorted_set],
|
||||||
stint
|
stint
|
||||||
|
|
||||||
# Make sure that the runner can stay on public view without the need
|
# Make sure that the runner can stay on public view without the need
|
||||||
|
@ -76,6 +76,10 @@ const
|
||||||
# Helpers
|
# Helpers
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc toHex2(a: byte): string =
|
||||||
|
# Avoids compiler warning with `toHex(2)`
|
||||||
|
[a].toHex
|
||||||
|
|
||||||
proc joinXX(s: string): string =
|
proc joinXX(s: string): string =
|
||||||
if s.len <= 30:
|
if s.len <= 30:
|
||||||
return s
|
return s
|
||||||
|
@ -92,13 +96,13 @@ proc toXX[T](s: T): string =
|
||||||
s.toHex.strip(leading=true,chars={'0'}).toLowerAscii
|
s.toHex.strip(leading=true,chars={'0'}).toLowerAscii
|
||||||
|
|
||||||
proc toXX(q: Blob): string =
|
proc toXX(q: Blob): string =
|
||||||
q.mapIt(it.toHex(2)).join(":")
|
q.mapIt(it.toHex2).join(":")
|
||||||
|
|
||||||
proc toXX(a: EthAddress): string =
|
proc toXX(a: EthAddress): string =
|
||||||
a.mapIt(it.toHex(2)).joinXX
|
a.mapIt(it.toHex2).joinXX
|
||||||
|
|
||||||
proc toXX(h: Hash256): string =
|
proc toXX(h: Hash256): string =
|
||||||
h.data.mapIt(it.toHex(2)).joinXX
|
h.data.mapIt(it.toHex2).joinXX
|
||||||
|
|
||||||
proc toXX(v: int64; r,s: UInt256): string =
|
proc toXX(v: int64; r,s: UInt256): string =
|
||||||
v.toXX & ":" & ($r).joinXX & ":" & ($s).joinXX
|
v.toXX & ":" & ($r).joinXX & ":" & ($s).joinXX
|
||||||
|
@ -184,7 +188,7 @@ proc isOk*(rc: ValidationResult): bool =
|
||||||
rc == ValidationResult.OK
|
rc == ValidationResult.OK
|
||||||
|
|
||||||
proc toHex*(acc: EthAddress): string =
|
proc toHex*(acc: EthAddress): string =
|
||||||
acc.toSeq.mapIt(it.toHex(2)).join
|
acc.toHex
|
||||||
|
|
||||||
proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) =
|
proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) =
|
||||||
if noisy:
|
if noisy:
|
||||||
|
|
|
@ -106,7 +106,7 @@ proc initEnv(envFork: HardFork): TestEnv =
|
||||||
let
|
let
|
||||||
com = CommonRef.new(
|
com = CommonRef.new(
|
||||||
newCoreDbRef LegacyDbMemory,
|
newCoreDbRef LegacyDbMemory,
|
||||||
conf.pruneMode == PruneMode.Full,
|
conf.chainDbMode == ChainDbMode.Prune,
|
||||||
conf.networkId,
|
conf.networkId,
|
||||||
conf.networkParams
|
conf.networkParams
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in New Issue