Simple stupid key-value table companion for Aristo DB (#1746)
why: Additional tables needed for the `CoreDB` object with separate key-value table and MPT. details: + Stripped down copy of Aristo DB to have a similar look'n feel. Otherwise it is just a posh way for accessing `Table` objects or `RocksDB` data. + No unit tests yet, will be tested on the go.
This commit is contained in:
Jordan Hrycaj
GitHub
parent
8e46953390
commit
dda049cd43
|
|
@ -0,0 +1,28 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Kvt DB -- Standard interface
|
||||||
|
## ============================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import kvt/[
|
||||||
|
kvt_constants, kvt_init, kvt_tx, kvt_utils]
|
||||||
|
export
|
||||||
|
kvt_constants, kvt_init, kvt_tx, kvt_utils
|
||||||
|
|
||||||
|
import
|
||||||
|
kvt/kvt_desc
|
||||||
|
export
|
||||||
|
KvtDbRef,
|
||||||
|
KvtError,
|
||||||
|
isValid
|
||||||
|
|
||||||
|
# End
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
import
|
||||||
|
../aristo/aristo_constants
|
||||||
|
|
||||||
|
export
|
||||||
|
EmptyBlob
|
||||||
|
|
||||||
|
# End
|
||||||
|
|
@ -0,0 +1,62 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Kvt DB -- key-value table
|
||||||
|
## =========================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/tables,
|
||||||
|
eth/common,
|
||||||
|
./kvt_constants,
|
||||||
|
./kvt_desc/[desc_error, desc_structural]
|
||||||
|
|
||||||
|
from ./kvt_desc/desc_backend
|
||||||
|
import BackendRef
|
||||||
|
|
||||||
|
# Not auto-exporting backend
|
||||||
|
export
|
||||||
|
kvt_constants, desc_error, desc_structural
|
||||||
|
|
||||||
|
type
|
||||||
|
KvtTxRef* = ref object
|
||||||
|
## Transaction descriptor
|
||||||
|
db*: KvtDbRef ## Database descriptor
|
||||||
|
parent*: KvtTxRef ## Previous transaction
|
||||||
|
txUid*: uint ## Unique ID among transactions
|
||||||
|
level*: int ## Stack index for this transaction
|
||||||
|
|
||||||
|
KvtDbRef* = ref KvtDbObj
|
||||||
|
KvtDbObj* = object
|
||||||
|
## Three tier database object supporting distributed instances.
|
||||||
|
top*: LayerRef ## Database working layer, mutable
|
||||||
|
stack*: seq[LayerRef] ## Stashed immutable parent layers
|
||||||
|
backend*: BackendRef ## Backend database (may well be `nil`)
|
||||||
|
|
||||||
|
txRef*: KvtTxRef ## Latest active transaction
|
||||||
|
txUidGen*: uint ## Tx-relative unique number generator
|
||||||
|
|
||||||
|
KvtDbAction* = proc(db: KvtDbRef) {.gcsafe, raises: [CatchableError].}
|
||||||
|
## Generic call back function/closure.
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func getOrVoid*(tab: Table[Blob,Blob]; w: Blob): Blob =
|
||||||
|
tab.getOrDefault(w, EmptyBlob)
|
||||||
|
|
||||||
|
func isValid*(key: Blob): bool =
|
||||||
|
key != EmptyBlob
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,71 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Kvt DB -- backend data types
|
||||||
|
## ============================
|
||||||
|
##
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
eth/common,
|
||||||
|
results,
|
||||||
|
./desc_error
|
||||||
|
|
||||||
|
type
|
||||||
|
GetKvpFn* =
|
||||||
|
proc(key: Blob): Result[Blob,KvtError] {.gcsafe, raises: [].}
|
||||||
|
## Generic backend database retrieval function
|
||||||
|
|
||||||
|
# -------------
|
||||||
|
|
||||||
|
PutHdlRef* = ref object of RootRef
|
||||||
|
## Persistent database transaction frame handle. This handle is used to
|
||||||
|
## wrap any of `PutVtxFn`, `PutKeyFn`, and `PutIdgFn` into and atomic
|
||||||
|
## transaction frame. These transaction frames must not be interleaved
|
||||||
|
## by any library function using the backend.
|
||||||
|
|
||||||
|
PutBegFn* =
|
||||||
|
proc(): PutHdlRef {.gcsafe, raises: [].}
|
||||||
|
## Generic transaction initialisation function
|
||||||
|
|
||||||
|
PutKvpFn* =
|
||||||
|
proc(hdl: PutHdlRef; kvps: openArray[(Blob,Blob)]) {.gcsafe, raises: [].}
|
||||||
|
## Generic backend database bulk storage function.
|
||||||
|
|
||||||
|
PutEndFn* =
|
||||||
|
proc(hdl: PutHdlRef): Result[void,KvtError] {.gcsafe, raises: [].}
|
||||||
|
## Generic transaction termination function
|
||||||
|
|
||||||
|
# -------------
|
||||||
|
|
||||||
|
CloseFn* =
|
||||||
|
proc(flush: bool) {.gcsafe, raises: [].}
|
||||||
|
## Generic destructor for the `Kvt DB` backend. The argument `flush`
|
||||||
|
## indicates that a full database deletion is requested. If passed
|
||||||
|
## `false` the outcome might differ depending on the type of backend
|
||||||
|
## (e.g. in-memory backends would flush on close.)
|
||||||
|
|
||||||
|
# -------------
|
||||||
|
|
||||||
|
BackendRef* = ref object of RootRef
|
||||||
|
## Backend interface.
|
||||||
|
|
||||||
|
getKvpFn*: GetKvpFn ## Read key-value pair
|
||||||
|
|
||||||
|
putBegFn*: PutBegFn ## Start bulk store session
|
||||||
|
putKvpFn*: PutKvpFn ## Bulk store key-value pairs
|
||||||
|
putEndFn*: PutEndFn ## Commit bulk store session
|
||||||
|
|
||||||
|
closeFn*: CloseFn ## Generic destructor
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
type
|
||||||
|
KvtError* = enum
|
||||||
|
NothingSerious = 0
|
||||||
|
GenericError
|
||||||
|
|
||||||
|
GetNotFound
|
||||||
|
KeyInvalid
|
||||||
|
DataInvalid
|
||||||
|
|
||||||
|
# RocksDB backend
|
||||||
|
RdbBeCantCreateDataDir
|
||||||
|
RdbBeCantCreateBackupDir
|
||||||
|
RdbBeCantCreateTmpDir
|
||||||
|
RdbBeDriverInitError
|
||||||
|
RdbBeDriverGetError
|
||||||
|
RdbBeDriverDelError
|
||||||
|
RdbBeCreateSstWriter
|
||||||
|
RdbBeOpenSstWriter
|
||||||
|
RdbBeAddSstWriter
|
||||||
|
RdbBeFinishSstWriter
|
||||||
|
RdbBeIngestSstWriter
|
||||||
|
|
||||||
|
# Transaction wrappers
|
||||||
|
TxArgStaleTx
|
||||||
|
TxBackendNotWritable
|
||||||
|
TxNoPendingTx
|
||||||
|
TxPendingTx
|
||||||
|
TxNotTopTx
|
||||||
|
TxStackGarbled
|
||||||
|
TxStackUnderflow
|
||||||
|
|
||||||
|
# End
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Kvt DB -- structural data types
|
||||||
|
## ===============================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/tables,
|
||||||
|
eth/common
|
||||||
|
|
||||||
|
type
|
||||||
|
LayerRef* = ref object
|
||||||
|
## Kvt database layer structures. Any layer holds the full
|
||||||
|
## change relative to the backend.
|
||||||
|
tab*: Table[Blob,Blob] ## Structural table
|
||||||
|
txUid*: uint ## Transaction identifier if positive
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public helpers, miscellaneous functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc dup*(layer: LayerRef): LayerRef =
|
||||||
|
## Duplicate layer.
|
||||||
|
result = LayerRef(
|
||||||
|
txUid: layer.txUid)
|
||||||
|
for (k,v) in layer.tab.pairs:
|
||||||
|
result.tab[k] = v
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Constructors for Key-Value Table DB
|
||||||
|
## ====================================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
./kvt_init/memory_only
|
||||||
|
export
|
||||||
|
memory_only
|
||||||
|
|
||||||
|
# End
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
../../aristo/aristo_init/init_common,
|
||||||
|
../kvt_desc,
|
||||||
|
../kvt_desc/desc_backend
|
||||||
|
|
||||||
|
export
|
||||||
|
BackendType # borrowed from Aristo
|
||||||
|
|
||||||
|
const
|
||||||
|
verifyIxId = true # and false
|
||||||
|
## Enforce session tracking
|
||||||
|
|
||||||
|
type
|
||||||
|
TypedBackendRef* = ref object of BackendRef
|
||||||
|
beKind*: BackendType ## Backend type identifier
|
||||||
|
when verifyIxId:
|
||||||
|
txGen: uint ## Transaction ID generator (for debugging)
|
||||||
|
txId: uint ## Active transaction ID (for debugging)
|
||||||
|
|
||||||
|
TypedPutHdlRef* = ref object of PutHdlRef
|
||||||
|
error*: KvtError ## Track error while collecting transaction
|
||||||
|
when verifyIxId:
|
||||||
|
txId: uint ## Transaction ID (for debugging)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc beginSession*(hdl: TypedPutHdlRef; db: TypedBackendRef) =
|
||||||
|
when verifyIxId:
|
||||||
|
doAssert db.txId == 0
|
||||||
|
if db.txGen == 0:
|
||||||
|
db.txGen = 1
|
||||||
|
db.txId = db.txGen
|
||||||
|
hdl.txId = db.txGen
|
||||||
|
db.txGen.inc
|
||||||
|
|
||||||
|
proc verifySession*(hdl: TypedPutHdlRef; db: TypedBackendRef) =
|
||||||
|
when verifyIxId:
|
||||||
|
doAssert db.txId == hdl.txId
|
||||||
|
|
||||||
|
proc finishSession*(hdl: TypedPutHdlRef; db: TypedBackendRef) =
|
||||||
|
when verifyIxId:
|
||||||
|
doAssert db.txId == hdl.txId
|
||||||
|
db.txId = 0
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,149 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## In-memory backend for Kvt DB
|
||||||
|
## ============================
|
||||||
|
##
|
||||||
|
## The iterators provided here are currently available only by direct
|
||||||
|
## backend access
|
||||||
|
## ::
|
||||||
|
## import
|
||||||
|
## kvt/kvt_init,
|
||||||
|
## kvt/kvt_init/kvt_memory
|
||||||
|
##
|
||||||
|
## let rc = newKvtDbRef(BackendMemory)
|
||||||
|
## if rc.isOk:
|
||||||
|
## let be = rc.value.to(MemBackendRef)
|
||||||
|
## for (n, key, vtx) in be.walkVtx:
|
||||||
|
## ...
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/tables,
|
||||||
|
chronicles,
|
||||||
|
eth/common,
|
||||||
|
results,
|
||||||
|
../kvt_desc,
|
||||||
|
../kvt_desc/desc_backend,
|
||||||
|
./init_common
|
||||||
|
|
||||||
|
type
|
||||||
|
MemBackendRef* = ref object of TypedBackendRef
|
||||||
|
## Inheriting table so access can be extended for debugging purposes
|
||||||
|
tab: Table[Blob,Blob] ## Structural key-value table
|
||||||
|
|
||||||
|
MemPutHdlRef = ref object of TypedPutHdlRef
|
||||||
|
tab: Table[Blob,Blob]
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
template logTxt(info: static[string]): static[string] =
|
||||||
|
"MemoryDB " & info
|
||||||
|
|
||||||
|
|
||||||
|
proc newSession(db: MemBackendRef): MemPutHdlRef =
|
||||||
|
new result
|
||||||
|
result.TypedPutHdlRef.beginSession db
|
||||||
|
|
||||||
|
proc getSession(hdl: PutHdlRef; db: MemBackendRef): MemPutHdlRef =
|
||||||
|
hdl.TypedPutHdlRef.verifySession db
|
||||||
|
hdl.MemPutHdlRef
|
||||||
|
|
||||||
|
proc endSession(hdl: PutHdlRef; db: MemBackendRef): MemPutHdlRef =
|
||||||
|
hdl.TypedPutHdlRef.finishSession db
|
||||||
|
hdl.MemPutHdlRef
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private functions: interface
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc getKvpFn(db: MemBackendRef): GetKvpFn =
|
||||||
|
result =
|
||||||
|
proc(key: Blob): Result[Blob,KvtError] =
|
||||||
|
if key.len == 0:
|
||||||
|
return err(KeyInvalid)
|
||||||
|
let data = db.tab.getOrVoid key
|
||||||
|
if data.isValid:
|
||||||
|
return ok(data)
|
||||||
|
err(GetNotFound)
|
||||||
|
|
||||||
|
# -------------
|
||||||
|
|
||||||
|
proc putBegFn(db: MemBackendRef): PutBegFn =
|
||||||
|
result =
|
||||||
|
proc(): PutHdlRef =
|
||||||
|
db.newSession()
|
||||||
|
|
||||||
|
proc putKvpFn(db: MemBackendRef): PutKvpFn =
|
||||||
|
result =
|
||||||
|
proc(hdl: PutHdlRef; kvps: openArray[(Blob,Blob)]) =
|
||||||
|
let hdl = hdl.getSession db
|
||||||
|
if hdl.error == KvtError(0):
|
||||||
|
for (k,v) in kvps:
|
||||||
|
if k.isValid:
|
||||||
|
hdl.tab[k] = v
|
||||||
|
else:
|
||||||
|
hdl.error = KeyInvalid
|
||||||
|
|
||||||
|
proc putEndFn(db: MemBackendRef): PutEndFn =
|
||||||
|
result =
|
||||||
|
proc(hdl: PutHdlRef): Result[void,KvtError] =
|
||||||
|
let hdl = hdl.endSession db
|
||||||
|
if hdl.error != KvtError(0):
|
||||||
|
debug logTxt "putEndFn: key/value failed", error=hdl.error
|
||||||
|
return err(hdl.error)
|
||||||
|
|
||||||
|
for (k,v) in hdl.tab.pairs:
|
||||||
|
db.tab[k] = v
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# -------------
|
||||||
|
|
||||||
|
proc closeFn(db: MemBackendRef): CloseFn =
|
||||||
|
result =
|
||||||
|
proc(ignore: bool) =
|
||||||
|
discard
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc memoryBackend*: BackendRef =
|
||||||
|
let db = MemBackendRef(
|
||||||
|
beKind: BackendMemory)
|
||||||
|
|
||||||
|
db.getKvpFn = getKvpFn db
|
||||||
|
|
||||||
|
db.putBegFn = putBegFn db
|
||||||
|
db.putKvpFn = putKvpFn db
|
||||||
|
db.putEndFn = putEndFn db
|
||||||
|
|
||||||
|
db.closeFn = closeFn db
|
||||||
|
|
||||||
|
db
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public iterators (needs direct backend access)
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
iterator walk*(
|
||||||
|
be: MemBackendRef;
|
||||||
|
): tuple[key: Blob, data: Blob] =
|
||||||
|
## Walk over all key-value pairs of the database.
|
||||||
|
for (k,v) in be.tab.pairs:
|
||||||
|
yield (k,v)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,88 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Non persistent constructors for Kvt DB
|
||||||
|
## ======================================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/sets,
|
||||||
|
results,
|
||||||
|
../kvt_desc,
|
||||||
|
../kvt_desc/desc_backend,
|
||||||
|
"."/[init_common, memory_db]
|
||||||
|
|
||||||
|
type
|
||||||
|
VoidBackendRef* = ref object of TypedBackendRef
|
||||||
|
## Dummy descriptor type, will typically used as `nil` reference
|
||||||
|
|
||||||
|
export
|
||||||
|
BackendType,
|
||||||
|
MemBackendRef
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public database constuctors, destructor
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc newKvtDbRef*(
|
||||||
|
backend: static[BackendType];
|
||||||
|
): KvtDbRef =
|
||||||
|
## Simplified prototype for `BackendNone` and `BackendMemory` type backend.
|
||||||
|
##
|
||||||
|
when backend == BackendVoid:
|
||||||
|
KvtDbRef(top: LayerRef())
|
||||||
|
|
||||||
|
elif backend == BackendMemory:
|
||||||
|
KvtDbRef(top: LayerRef(), backend: memoryBackend(qidLayout))
|
||||||
|
|
||||||
|
elif backend == BackendRocksDB:
|
||||||
|
{.error: "Kvt DB backend \"BackendRocksDB\" needs basePath argument".}
|
||||||
|
|
||||||
|
else:
|
||||||
|
{.error: "Unknown/unsupported Kvt DB backend".}
|
||||||
|
|
||||||
|
# -----------------
|
||||||
|
|
||||||
|
proc finish*(db: KvtDbRef; flush = false) =
|
||||||
|
## Backend destructor. The argument `flush` indicates that a full database
|
||||||
|
## deletion is requested. If set `false` the outcome might differ depending
|
||||||
|
## on the type of backend (e.g. the `BackendMemory` backend will always
|
||||||
|
## flush on close.)
|
||||||
|
##
|
||||||
|
## This distructor may be used on already *destructed* descriptors.
|
||||||
|
##
|
||||||
|
if not db.isNil:
|
||||||
|
if not db.backend.isNil:
|
||||||
|
db.backend.closeFn flush
|
||||||
|
db[] = KvtDbObj(top: LayerRef())
|
||||||
|
|
||||||
|
# -----------------
|
||||||
|
|
||||||
|
proc to*[W: TypedBackendRef|MemBackendRef|VoidBackendRef](
|
||||||
|
db: KvtDbRef;
|
||||||
|
T: type W;
|
||||||
|
): T =
|
||||||
|
## Handy helper for lew-level access to some backend functionality
|
||||||
|
db.backend.T
|
||||||
|
|
||||||
|
proc kind*(
|
||||||
|
be: BackendRef;
|
||||||
|
): BackendType =
|
||||||
|
## Retrieves the backend type symbol for a `TypedBackendRef` argument where
|
||||||
|
## `BackendVoid` is returned for the`nil` backend.
|
||||||
|
if be.isNil:
|
||||||
|
BackendVoid
|
||||||
|
else:
|
||||||
|
be.TypedBackendRef.beKind
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Persistent constructor for Kvt DB
|
||||||
|
## ====================================
|
||||||
|
##
|
||||||
|
## This module automatically pulls in the persistent backend library at the
|
||||||
|
## linking stage (e.g. `rocksdb`) which can be avoided for pure memory DB
|
||||||
|
## applications by importing `./kvt_init/memory_only` (rather than
|
||||||
|
## `./kvt_init/persistent`.)
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
results,
|
||||||
|
../kvt_desc,
|
||||||
|
"."/[init_common, rocks_db, memory_only]
|
||||||
|
export
|
||||||
|
RdbBackendRef,
|
||||||
|
memory_only
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public database constuctors, destructor
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc newKvtDbRef*(
|
||||||
|
backend: static[BackendType];
|
||||||
|
basePath: string;
|
||||||
|
): Result[KvtDbRef,KvtError] =
|
||||||
|
## Generic constructor, `basePath` argument is ignored for `BackendNone` and
|
||||||
|
## `BackendMemory` type backend database. Also, both of these backends
|
||||||
|
## aways succeed initialising.
|
||||||
|
##
|
||||||
|
when backend == BackendRocksDB:
|
||||||
|
ok KvtDbRef(top: LayerRef(vGen: vGen), backend: ? rocksDbBackend basePath)
|
||||||
|
|
||||||
|
elif backend == BackendVoid:
|
||||||
|
{.error: "Use BackendNone.init() without path argument".}
|
||||||
|
|
||||||
|
elif backend == BackendMemory:
|
||||||
|
{.error: "Use BackendMemory.init() without path argument".}
|
||||||
|
|
||||||
|
else:
|
||||||
|
{.error: "Unknown/unsupported Kvt DB backend".}
|
||||||
|
|
||||||
|
# -----------------
|
||||||
|
|
||||||
|
proc to*[W: RdbBackendRef](
|
||||||
|
db: KvtDbRef;
|
||||||
|
T: type W;
|
||||||
|
): T =
|
||||||
|
## Handy helper for lew-level access to some backend functionality
|
||||||
|
db.backend.T
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,180 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Rocksdb backend for Kvt DB
|
||||||
|
## ==========================
|
||||||
|
##
|
||||||
|
## The iterators provided here are currently available only by direct
|
||||||
|
## backend access
|
||||||
|
## ::
|
||||||
|
## import
|
||||||
|
## kvt/kvt_init,
|
||||||
|
## kvt/kvt_init/kvt_rocksdb
|
||||||
|
##
|
||||||
|
## let rc = KvtDb.init(BackendRocksDB, "/var/tmp")
|
||||||
|
## if rc.isOk:
|
||||||
|
## let be = rc.value.to(RdbBackendRef)
|
||||||
|
## for (n, key, vtx) in be.walkVtx:
|
||||||
|
## ...
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
chronicles,
|
||||||
|
eth/common,
|
||||||
|
rocksdb,
|
||||||
|
results,
|
||||||
|
../kvt_desc,
|
||||||
|
../kvt_desc/desc_backend,
|
||||||
|
./init_common,
|
||||||
|
./rocks_db/[rdb_desc, rdb_get, rdb_init, rdb_put, rdb_walk]
|
||||||
|
|
||||||
|
logScope:
|
||||||
|
topics = "kvt-backend"
|
||||||
|
|
||||||
|
type
|
||||||
|
RdbBackendRef* = ref object of TypedBackendRef
|
||||||
|
rdb: RdbInst ## Allows low level access to database
|
||||||
|
|
||||||
|
RdbPutHdlRef = ref object of TypedPutHdlRef
|
||||||
|
tab: Table[Blob,Blob] ## Transaction cache
|
||||||
|
|
||||||
|
const
|
||||||
|
extraTraceMessages = false or true
|
||||||
|
## Enabled additional logging noise
|
||||||
|
|
||||||
|
# ----------
|
||||||
|
|
||||||
|
maxOpenFiles = 512 ## Rocks DB setup, open files limit
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
template logTxt(info: static[string]): static[string] =
|
||||||
|
"RocksDB " & info
|
||||||
|
|
||||||
|
|
||||||
|
proc newSession(db: RdbBackendRef): RdbPutHdlRef =
|
||||||
|
new result
|
||||||
|
result.TypedPutHdlRef.beginSession db
|
||||||
|
|
||||||
|
proc getSession(hdl: PutHdlRef; db: RdbBackendRef): RdbPutHdlRef =
|
||||||
|
hdl.TypedPutHdlRef.verifySession db
|
||||||
|
hdl.RdbPutHdlRef
|
||||||
|
|
||||||
|
proc endSession(hdl: PutHdlRef; db: RdbBackendRef): RdbPutHdlRef =
|
||||||
|
hdl.TypedPutHdlRef.finishSession db
|
||||||
|
hdl.RdbPutHdlRef
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private functions: interface
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc getKvpFn(db: RdbBackendRef): GetKvpFn =
|
||||||
|
result =
|
||||||
|
proc(key: Blob): Result[Blob,KvtError] =
|
||||||
|
if key.len == 0:
|
||||||
|
return err(KeyInvalid)
|
||||||
|
let rc = db.rdb.get key.toOpenArray(0,key.len-1)
|
||||||
|
if rc.isErr:
|
||||||
|
debug logTxt "getKvpFn() failed", key,
|
||||||
|
error=rc.error[0], info=rc.error[1]
|
||||||
|
return err(rc.error[0])
|
||||||
|
|
||||||
|
# Decode data record
|
||||||
|
if 0 < rc.value.len:
|
||||||
|
return ok(rc.value)
|
||||||
|
|
||||||
|
err(GetNotFound)
|
||||||
|
|
||||||
|
# -------------
|
||||||
|
|
||||||
|
proc putBegFn(db: RdbBackendRef): PutBegFn =
|
||||||
|
result =
|
||||||
|
proc(): PutHdlRef =
|
||||||
|
db.newSession()
|
||||||
|
|
||||||
|
proc putKvpFn(db: RdbBackendRef): PutKvpFn =
|
||||||
|
result =
|
||||||
|
proc(hdl: PutHdlRef; kvps: openArray[(Blob,Blob)]) =
|
||||||
|
let hdl = hdl.getSession db
|
||||||
|
if hdl.error == KvtError(0):
|
||||||
|
for (k,v) in kvps:
|
||||||
|
if k.isValid:
|
||||||
|
hdl.tab[k] = v
|
||||||
|
else:
|
||||||
|
hdl.error = KeyInvalid
|
||||||
|
|
||||||
|
proc putEndFn(db: RdbBackendRef): PutEndFn =
|
||||||
|
result =
|
||||||
|
proc(hdl: PutHdlRef): Result[void,KvtError] =
|
||||||
|
let hdl = hdl.endSession db
|
||||||
|
if hdl.error != KvtError(0):
|
||||||
|
debug logTxt "putEndFn: key/value failed", error=hdl.error
|
||||||
|
return err(hdl.error)
|
||||||
|
let rc = db.rdb.put hdl.tab
|
||||||
|
if rc.isErr:
|
||||||
|
when extraTraceMessages:
|
||||||
|
debug logTxt "putEndFn: failed",
|
||||||
|
error=rc.error[0], info=rc.error[1]
|
||||||
|
return err(rc.error[0])
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc closeFn(db: RdbBackendRef): CloseFn =
|
||||||
|
result =
|
||||||
|
proc(flush: bool) =
|
||||||
|
db.rdb.destroy(flush)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc rocksDbBackend*(
|
||||||
|
path: string;
|
||||||
|
): Result[BackendRef,KvtError] =
|
||||||
|
let db = RdbBackendRef(
|
||||||
|
beKind: BackendRocksDB)
|
||||||
|
|
||||||
|
# Initialise RocksDB
|
||||||
|
block:
|
||||||
|
let rc = db.rdb.init(path, maxOpenFiles)
|
||||||
|
if rc.isErr:
|
||||||
|
when extraTraceMessages:
|
||||||
|
trace logTxt "constructor failed",
|
||||||
|
error=rc.error[0], info=rc.error[1]
|
||||||
|
return err(rc.error[0])
|
||||||
|
|
||||||
|
db.getKvpFn = getKvpFn db
|
||||||
|
|
||||||
|
db.putBegFn = putBegFn db
|
||||||
|
db.putKvpFn = putKvpFn db
|
||||||
|
db.putEndFn = putEndFn db
|
||||||
|
|
||||||
|
db.closeFn = closeFn db
|
||||||
|
|
||||||
|
ok db
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public iterators (needs direct backend access)
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
iterator walk*(
|
||||||
|
be: RdbBackendRef;
|
||||||
|
): tuple[key: Blob, data: Blob] =
|
||||||
|
## Walk over all key-value pairs of the database.
|
||||||
|
##
|
||||||
|
for (k,v) in be.rdb.walk:
|
||||||
|
yield (k,v)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Rocks DB internal driver descriptor
|
||||||
|
## ===================================
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
rocksdb
|
||||||
|
|
||||||
|
type
|
||||||
|
RdbInst* = object
|
||||||
|
store*: RocksDBInstance ## Rocks DB database handler
|
||||||
|
basePath*: string ## Database directory
|
||||||
|
|
||||||
|
# Low level Rocks DB access for bulk store
|
||||||
|
envOpt*: rocksdb_envoptions_t
|
||||||
|
impOpt*: rocksdb_ingestexternalfileoptions_t
|
||||||
|
|
||||||
|
const
|
||||||
|
BaseFolder* = "nimbus" # Same as for Legacy DB
|
||||||
|
DataFolder* = "kvt" # Legacy DB has "data"
|
||||||
|
BackupFolder* = "khistory" # Legacy DB has "backups"
|
||||||
|
SstCache* = "kbulkput" # Rocks DB bulk load file name in temp folder
|
||||||
|
TempFolder* = "tmp" # Not used with legacy DB (same for Aristo)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Rocks DB fetch data record
|
||||||
|
## ==========================
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
eth/common,
|
||||||
|
rocksdb,
|
||||||
|
results,
|
||||||
|
"../.."/[kvt_constants, kvt_desc],
|
||||||
|
./rdb_desc
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc get*(
|
||||||
|
rdb: RdbInst;
|
||||||
|
key: openArray[byte],
|
||||||
|
): Result[Blob,(KvtError,string)] =
|
||||||
|
var res: Blob
|
||||||
|
let onData: DataProc = proc(data: openArray[byte]) =
|
||||||
|
res = @data
|
||||||
|
let rc = rdb.store.get(key, onData)
|
||||||
|
if rc.isErr:
|
||||||
|
return err((RdbBeDriverGetError,rc.error))
|
||||||
|
if not rc.value:
|
||||||
|
res = EmptyBlob
|
||||||
|
ok res
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,107 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Rocksdb constructor/destructor for Kvt DB
|
||||||
|
## =========================================
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/os,
|
||||||
|
chronicles,
|
||||||
|
rocksdb,
|
||||||
|
results,
|
||||||
|
../../kvt_desc,
|
||||||
|
./rdb_desc
|
||||||
|
|
||||||
|
logScope:
|
||||||
|
topics = "kvt-backend"
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
template logTxt(info: static[string]): static[string] =
|
||||||
|
"RocksDB/init " & info
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public constructor
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc init*(
|
||||||
|
rdb: var RdbInst;
|
||||||
|
basePath: string;
|
||||||
|
openMax: int;
|
||||||
|
): Result[void,(KvtError,string)] =
|
||||||
|
## Constructor c ode inspired by `RocksStoreRef.init()` from
|
||||||
|
## kvstore_rocksdb.nim
|
||||||
|
let
|
||||||
|
dataDir = basePath / BaseFolder / DataFolder
|
||||||
|
backupsDir = basePath / BaseFolder / BackupFolder
|
||||||
|
tmpDir = basePath / BaseFolder / TempFolder
|
||||||
|
|
||||||
|
try:
|
||||||
|
dataDir.createDir
|
||||||
|
except OSError, IOError:
|
||||||
|
return err((RdbBeCantCreateDataDir, ""))
|
||||||
|
try:
|
||||||
|
backupsDir.createDir
|
||||||
|
except OSError, IOError:
|
||||||
|
return err((RdbBeCantCreateBackupDir, ""))
|
||||||
|
try:
|
||||||
|
tmpDir.createDir
|
||||||
|
except OSError, IOError:
|
||||||
|
return err((RdbBeCantCreateTmpDir, ""))
|
||||||
|
|
||||||
|
let rc = rdb.store.init(
|
||||||
|
dbPath=dataDir, dbBackuppath=backupsDir, readOnly=false,
|
||||||
|
maxOpenFiles=openMax)
|
||||||
|
if rc.isErr:
|
||||||
|
let error = RdbBeDriverInitError
|
||||||
|
debug logTxt "driver failed", dataDir, backupsDir, openMax,
|
||||||
|
error, info=rc.error
|
||||||
|
return err((RdbBeDriverInitError, rc.error))
|
||||||
|
|
||||||
|
# The following is a default setup (subject to change)
|
||||||
|
rdb.impOpt = rocksdb_ingestexternalfileoptions_create()
|
||||||
|
rdb.envOpt = rocksdb_envoptions_create()
|
||||||
|
|
||||||
|
rdb.basePath = basePath
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc destroy*(rdb: var RdbInst; flush: bool) =
|
||||||
|
## Destructor
|
||||||
|
rdb.envOpt.rocksdb_envoptions_destroy()
|
||||||
|
rdb.impOpt.rocksdb_ingestexternalfileoptions_destroy()
|
||||||
|
rdb.store.close()
|
||||||
|
|
||||||
|
let
|
||||||
|
base = rdb.basePath / BaseFolder
|
||||||
|
try:
|
||||||
|
(base / TempFolder).removeDir
|
||||||
|
|
||||||
|
if flush:
|
||||||
|
(base / DataFolder).removeDir
|
||||||
|
|
||||||
|
# Remove the base folder if it is empty
|
||||||
|
block done:
|
||||||
|
for w in base.walkDirRec:
|
||||||
|
# Ignore backup files
|
||||||
|
if 0 < w.len and w[^1] != '~':
|
||||||
|
break done
|
||||||
|
base.removeDir
|
||||||
|
|
||||||
|
except CatchableError:
|
||||||
|
discard
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,198 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Rocks DB store data record
|
||||||
|
## ==========================
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/[algorithm, os, sequtils, sets, tables],
|
||||||
|
chronicles,
|
||||||
|
eth/common,
|
||||||
|
rocksdb,
|
||||||
|
results,
|
||||||
|
../../kvt_desc,
|
||||||
|
./rdb_desc
|
||||||
|
|
||||||
|
logScope:
|
||||||
|
topics = "kvt-backend"
|
||||||
|
|
||||||
|
type
|
||||||
|
RdbPutSession = object
|
||||||
|
writer: rocksdb_sstfilewriter_t
|
||||||
|
sstPath: string
|
||||||
|
nRecords: int
|
||||||
|
|
||||||
|
const
|
||||||
|
extraTraceMessages = false or true
|
||||||
|
## Enable additional logging noise
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
template logTxt(info: static[string]): static[string] =
|
||||||
|
"RocksDB/put " & info
|
||||||
|
|
||||||
|
proc getFileSize(fileName: string): int64 =
|
||||||
|
var f: File
|
||||||
|
if f.open fileName:
|
||||||
|
defer: f.close
|
||||||
|
try:
|
||||||
|
result = f.getFileSize
|
||||||
|
except:
|
||||||
|
discard
|
||||||
|
|
||||||
|
proc rmFileIgnExpt(fileName: string) =
|
||||||
|
try:
|
||||||
|
fileName.removeFile
|
||||||
|
except:
|
||||||
|
discard
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc destroy(rps: RdbPutSession) =
|
||||||
|
rps.writer.rocksdb_sstfilewriter_destroy()
|
||||||
|
rps.sstPath.rmFileIgnExpt
|
||||||
|
|
||||||
|
proc begin(
|
||||||
|
rdb: var RdbInst;
|
||||||
|
): Result[RdbPutSession,(KvtError,string)] =
|
||||||
|
## Begin a new bulk load session storing data into a temporary cache file
|
||||||
|
## `fileName`. When finished, this file will bi direcly imported into the
|
||||||
|
## database.
|
||||||
|
var csError: cstring
|
||||||
|
|
||||||
|
var session = RdbPutSession(
|
||||||
|
writer: rocksdb_sstfilewriter_create(rdb.envOpt, rdb.store.options),
|
||||||
|
sstPath: rdb.basePath / BaseFolder / TempFolder / SstCache)
|
||||||
|
|
||||||
|
if session.writer.isNil:
|
||||||
|
return err((RdbBeCreateSstWriter, "Cannot create sst writer session"))
|
||||||
|
|
||||||
|
session.sstPath.rmFileIgnExpt
|
||||||
|
|
||||||
|
session.writer.rocksdb_sstfilewriter_open(
|
||||||
|
session.sstPath.cstring, addr csError)
|
||||||
|
if not csError.isNil:
|
||||||
|
session.destroy()
|
||||||
|
return err((RdbBeOpenSstWriter, $csError))
|
||||||
|
|
||||||
|
ok session
|
||||||
|
|
||||||
|
|
||||||
|
proc add(
|
||||||
|
session: var RdbPutSession;
|
||||||
|
key: openArray[byte];
|
||||||
|
val: openArray[byte];
|
||||||
|
): Result[void,(KvtError,string)] =
|
||||||
|
## Append a record to the SST file. Note that consecutive records must be
|
||||||
|
## strictly increasing.
|
||||||
|
##
|
||||||
|
## This function is a wrapper around `rocksdb_sstfilewriter_add()` or
|
||||||
|
## `rocksdb_sstfilewriter_put()` (stragely enough, there are two functions
|
||||||
|
## with exactly the same impementation code.)
|
||||||
|
var csError: cstring
|
||||||
|
|
||||||
|
session.writer.rocksdb_sstfilewriter_add(
|
||||||
|
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
|
||||||
|
cast[cstring](unsafeAddr val[0]), csize_t(val.len), addr csError)
|
||||||
|
if not csError.isNil:
|
||||||
|
return err((RdbBeAddSstWriter, $csError))
|
||||||
|
|
||||||
|
session.nRecords.inc
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc commit(
|
||||||
|
rdb: var RdbInst;
|
||||||
|
session: RdbPutSession;
|
||||||
|
): Result[void,(KvtError,string)] =
|
||||||
|
## Commit collected and cached data to the database. This function implies
|
||||||
|
## `destroy()` if successful. Otherwise `destroy()` must be called
|
||||||
|
## explicitely, e.g. after error analysis.
|
||||||
|
var csError: cstring
|
||||||
|
|
||||||
|
if 0 < session.nRecords:
|
||||||
|
session.writer.rocksdb_sstfilewriter_finish(addr csError)
|
||||||
|
if not csError.isNil:
|
||||||
|
return err((RdbBeFinishSstWriter, $csError))
|
||||||
|
|
||||||
|
rdb.store.db.rocksdb_ingest_external_file(
|
||||||
|
[session.sstPath].allocCStringArray, 1, rdb.impOpt, addr csError)
|
||||||
|
if not csError.isNil:
|
||||||
|
return err((RdbBeIngestSstWriter, $csError))
|
||||||
|
|
||||||
|
when extraTraceMessages:
|
||||||
|
let fileSize = session.sstPath.getFileSize
|
||||||
|
trace logTxt "finished sst", fileSize
|
||||||
|
|
||||||
|
session.destroy()
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc put*(
|
||||||
|
rdb: var RdbInst;
|
||||||
|
tab: Table[Blob,Blob];
|
||||||
|
): Result[void,(KvtError,string)] =
|
||||||
|
|
||||||
|
var session = block:
|
||||||
|
let rc = rdb.begin()
|
||||||
|
if rc.isErr:
|
||||||
|
return err(rc.error)
|
||||||
|
rc.value
|
||||||
|
|
||||||
|
# Vertices with empty table values will be deleted
|
||||||
|
var delKey: HashSet[Blob]
|
||||||
|
|
||||||
|
# Compare `Blob`s as left aligned big endian numbers, right padded with zeros
|
||||||
|
proc cmpBlobs(a, b: Blob): int =
|
||||||
|
let minLen = min(a.len, b.len)
|
||||||
|
for n in 0 ..< minLen:
|
||||||
|
if a[n] != b[n]:
|
||||||
|
return a[n].cmp b[n]
|
||||||
|
if a.len < b.len:
|
||||||
|
return -1
|
||||||
|
if b.len < a.len:
|
||||||
|
return 1
|
||||||
|
|
||||||
|
for key in tab.keys.toSeq.sorted cmpBlobs:
|
||||||
|
let val = tab.getOrVoid key
|
||||||
|
if val.isValid:
|
||||||
|
let rc = session.add(key, val)
|
||||||
|
if rc.isErr:
|
||||||
|
session.destroy()
|
||||||
|
return err(rc.error)
|
||||||
|
else:
|
||||||
|
delKey.incl key
|
||||||
|
|
||||||
|
block:
|
||||||
|
let rc = rdb.commit session
|
||||||
|
if rc.isErr:
|
||||||
|
trace logTxt "commit error", error=rc.error[0], info=rc.error[1]
|
||||||
|
return err(rc.error)
|
||||||
|
|
||||||
|
# Delete vertices after successfully updating vertices with non-zero values.
|
||||||
|
for key in delKey:
|
||||||
|
let rc = rdb.store.del key
|
||||||
|
if rc.isErr:
|
||||||
|
return err((RdbBeDriverDelError,rc.error))
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Rocks DB store data iterator
|
||||||
|
## ============================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/sequtils,
|
||||||
|
eth/common,
|
||||||
|
rocksdb,
|
||||||
|
./rdb_desc
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public iterators
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
iterator walk*(rdb: RdbInst): tuple[key: Blob, data: Blob] =
|
||||||
|
## Walk over all key-value pairs of the database.
|
||||||
|
##
|
||||||
|
let rit = rdb.store.db.rocksdb_create_iterator(rdb.store.readOptions)
|
||||||
|
defer: rit.rocksdb_iter_destroy()
|
||||||
|
|
||||||
|
rit.rocksdb_iter_seek_to_first()
|
||||||
|
while rit.rocksdb_iter_valid() != 0:
|
||||||
|
var kLen: csize_t
|
||||||
|
let kData = rit.rocksdb_iter_key(addr kLen)
|
||||||
|
|
||||||
|
if not kData.isNil and 0 < kLen:
|
||||||
|
var vLen: csize_t
|
||||||
|
let vData = rit.rocksdb_iter_value(addr vLen)
|
||||||
|
|
||||||
|
if not vData.isNil and 0 < vLen:
|
||||||
|
let
|
||||||
|
key = kData.toOpenArrayByte(0,int(kLen)-1).toSeq
|
||||||
|
data = vData.toOpenArrayByte(0,int(vLen)-1).toSeq
|
||||||
|
yield (key,data)
|
||||||
|
|
||||||
|
# Update Iterator (might overwrite kData/vdata)
|
||||||
|
rit.rocksdb_iter_next()
|
||||||
|
|
||||||
|
# End while
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
import
|
||||||
|
kvt_init/persistent as init_persistent,
|
||||||
|
kvt_walk/persistent as walk_persistent
|
||||||
|
export
|
||||||
|
init_persistent,
|
||||||
|
walk_persistent
|
||||||
|
|
||||||
|
# End
|
||||||
|
|
@ -0,0 +1,196 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Kvt DB -- Transaction interface
|
||||||
|
## ===============================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/[sequtils, tables],
|
||||||
|
results,
|
||||||
|
./kvt_desc/desc_backend,
|
||||||
|
./kvt_desc
|
||||||
|
|
||||||
|
func isTop*(tx: KvtTxRef): bool
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func getDbDescFromTopTx(tx: KvtTxRef): Result[KvtDbRef,KvtError] =
|
||||||
|
if not tx.isTop():
|
||||||
|
return err(TxNotTopTx)
|
||||||
|
let db = tx.db
|
||||||
|
if tx.level != db.stack.len:
|
||||||
|
return err(TxStackUnderflow)
|
||||||
|
ok db
|
||||||
|
|
||||||
|
proc getTxUid(db: KvtDbRef): uint =
|
||||||
|
if db.txUidGen == high(uint):
|
||||||
|
db.txUidGen = 0
|
||||||
|
db.txUidGen.inc
|
||||||
|
db.txUidGen
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions, getters
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func txTop*(db: KvtDbRef): Result[KvtTxRef,KvtError] =
|
||||||
|
## Getter, returns top level transaction if there is any.
|
||||||
|
if db.txRef.isNil:
|
||||||
|
err(TxNoPendingTx)
|
||||||
|
else:
|
||||||
|
ok(db.txRef)
|
||||||
|
|
||||||
|
func isTop*(tx: KvtTxRef): bool =
|
||||||
|
## Getter, returns `true` if the argument `tx` referes to the current top
|
||||||
|
## level transaction.
|
||||||
|
tx.db.txRef == tx and tx.db.top.txUid == tx.txUid
|
||||||
|
|
||||||
|
func level*(tx: KvtTxRef): int =
|
||||||
|
## Getter, positive nesting level of transaction argument `tx`
|
||||||
|
tx.level
|
||||||
|
|
||||||
|
func level*(db: KvtDbRef): int =
|
||||||
|
## Getter, non-negative nesting level (i.e. number of pending transactions)
|
||||||
|
if not db.txRef.isNil:
|
||||||
|
result = db.txRef.level
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func to*(tx: KvtTxRef; T: type[KvtDbRef]): T =
|
||||||
|
## Getter, retrieves the parent database descriptor from argument `tx`
|
||||||
|
tx.db
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions: Transaction frame
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc txBegin*(db: KvtDbRef): Result[KvtTxRef,KvtError] =
|
||||||
|
## Starts a new transaction.
|
||||||
|
##
|
||||||
|
## Example:
|
||||||
|
## ::
|
||||||
|
## proc doSomething(db: KvtDbRef) =
|
||||||
|
## let tx = db.begin
|
||||||
|
## defer: tx.rollback()
|
||||||
|
## ... continue using db ...
|
||||||
|
## tx.commit()
|
||||||
|
##
|
||||||
|
if db.level != db.stack.len:
|
||||||
|
return err(TxStackGarbled)
|
||||||
|
|
||||||
|
db.stack.add db.top.dup # push (save and use top later)
|
||||||
|
db.top.txUid = db.getTxUid()
|
||||||
|
|
||||||
|
db.txRef = KvtTxRef(
|
||||||
|
db: db,
|
||||||
|
txUid: db.top.txUid,
|
||||||
|
parent: db.txRef,
|
||||||
|
level: db.stack.len)
|
||||||
|
|
||||||
|
ok db.txRef
|
||||||
|
|
||||||
|
proc rollback*(
|
||||||
|
tx: KvtTxRef; # Top transaction on database
|
||||||
|
): Result[void,KvtError] =
|
||||||
|
## Given a *top level* handle, this function discards all database operations
|
||||||
|
## performed for this transactio. The previous transaction is returned if
|
||||||
|
## there was any.
|
||||||
|
##
|
||||||
|
let db = ? tx.getDbDescFromTopTx()
|
||||||
|
|
||||||
|
# Roll back to previous layer.
|
||||||
|
db.top = db.stack[^1]
|
||||||
|
db.stack.setLen(db.stack.len-1)
|
||||||
|
|
||||||
|
db.txRef = tx.parent
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc commit*(
|
||||||
|
tx: KvtTxRef; # Top transaction on database
|
||||||
|
): Result[void,KvtError] =
|
||||||
|
## Given a *top level* handle, this function accepts all database operations
|
||||||
|
## performed through this handle and merges it to the previous layer. The
|
||||||
|
## previous transaction is returned if there was any.
|
||||||
|
##
|
||||||
|
let db = ? tx.getDbDescFromTopTx()
|
||||||
|
|
||||||
|
# Keep top and discard layer below
|
||||||
|
db.top.txUid = db.stack[^1].txUid
|
||||||
|
db.stack.setLen(db.stack.len-1)
|
||||||
|
|
||||||
|
db.txRef = tx.parent
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc collapse*(
|
||||||
|
tx: KvtTxRef; # Top transaction on database
|
||||||
|
commit: bool; # Commit if `true`, otherwise roll back
|
||||||
|
): Result[void,KvtError] =
|
||||||
|
## Iterated application of `commit()` or `rollback()` performing the
|
||||||
|
## something similar to
|
||||||
|
## ::
|
||||||
|
## while true:
|
||||||
|
## discard tx.commit() # ditto for rollback()
|
||||||
|
## if db.topTx.isErr: break
|
||||||
|
## tx = db.topTx.value
|
||||||
|
##
|
||||||
|
let db = ? tx.getDbDescFromTopTx()
|
||||||
|
|
||||||
|
# If commit, then leave the current layer and clear the stack, oterwise
|
||||||
|
# install the stack bottom.
|
||||||
|
if not commit:
|
||||||
|
db.stack[0].swap db.top
|
||||||
|
|
||||||
|
db.top.txUid = 0
|
||||||
|
db.stack.setLen(0)
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions: save database
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc stow*(
|
||||||
|
db: KvtDbRef; # Database
|
||||||
|
): Result[void,KvtError] =
|
||||||
|
## The function saves the data from the top layer cache into the
|
||||||
|
## backend database.
|
||||||
|
##
|
||||||
|
## If there is no backend the function returns immediately with an error.
|
||||||
|
## The same happens if there is a pending transaction.
|
||||||
|
##
|
||||||
|
if not db.txRef.isNil:
|
||||||
|
return err(TxPendingTx)
|
||||||
|
if 0 < db.stack.len:
|
||||||
|
return err(TxStackGarbled)
|
||||||
|
|
||||||
|
let be = db.backend
|
||||||
|
if be.isNil:
|
||||||
|
return err(TxBackendNotWritable)
|
||||||
|
|
||||||
|
# Save structural and other table entries
|
||||||
|
let txFrame = be.putBegFn()
|
||||||
|
be.putKvpFn(txFrame, db.top.tab.pairs.toSeq)
|
||||||
|
? be.putEndFn txFrame
|
||||||
|
|
||||||
|
# Delete or clear stack and clear top
|
||||||
|
db.stack.setLen(0)
|
||||||
|
db.top = LayerRef(txUid: db.top.txUid)
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,95 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
## Kvt DB -- Common functions
|
||||||
|
## ==========================
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
eth/common,
|
||||||
|
results,
|
||||||
|
./kvt_desc/desc_backend,
|
||||||
|
./kvt_desc
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private helpers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc getBE*(
|
||||||
|
db: KvtDbRef; # Database
|
||||||
|
key: Blob; # Key of database record
|
||||||
|
): Result[Blob,KvtError] =
|
||||||
|
## For the argument `key` return the associated value from the backend
|
||||||
|
## database if available.
|
||||||
|
##
|
||||||
|
let be = db.backend
|
||||||
|
if not be.isNil:
|
||||||
|
return be.getKvpFn key
|
||||||
|
err(GetNotFound)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions, converters
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc put*(
|
||||||
|
db: KvtDbRef; # Database
|
||||||
|
key: Blob; # Key of database record to store
|
||||||
|
data: Blob; # Value of database record to store
|
||||||
|
): Result[void,KvtError] =
|
||||||
|
## For the argument `key` associated the argument `data` as value (which
|
||||||
|
## will be marked in the top layer cache.)
|
||||||
|
if key.len == 0:
|
||||||
|
return err(KeyInvalid)
|
||||||
|
if data.len == 0:
|
||||||
|
return err(DataInvalid)
|
||||||
|
|
||||||
|
db.top.tab[key] = data
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc del*(
|
||||||
|
db: KvtDbRef; # Database
|
||||||
|
key: Blob; # Key of database record to delete
|
||||||
|
): Result[void,KvtError] =
|
||||||
|
## For the argument `key` delete the associated value (which will be marked
|
||||||
|
## in the top layer cache.)
|
||||||
|
if key.len == 0:
|
||||||
|
return err(KeyInvalid)
|
||||||
|
|
||||||
|
let rc = db.getBE(key)
|
||||||
|
if rc.isOk:
|
||||||
|
db.top.tab[key] = EmptyBlob
|
||||||
|
elif rc.error == GetNotFound:
|
||||||
|
db.top.tab.del key
|
||||||
|
else:
|
||||||
|
return err(rc.error)
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# ------------
|
||||||
|
|
||||||
|
proc get*(
|
||||||
|
db: KvtDbRef; # Database
|
||||||
|
key: Blob; # Key of database record
|
||||||
|
): Result[Blob,KvtError] =
|
||||||
|
## For the argument `key` return the associated value preferably from the
|
||||||
|
## top layer, or the database otherwise.
|
||||||
|
##
|
||||||
|
if key.len == 0:
|
||||||
|
return err(KeyInvalid)
|
||||||
|
let data = db.top.tab.getOrVoid key
|
||||||
|
if data.isValid:
|
||||||
|
return ok(data)
|
||||||
|
db.getBE key
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
# Nimbus - Types, data structures and shared utilities used in network sync
|
||||||
|
#
|
||||||
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or
|
||||||
|
# distributed except according to those terms.
|
||||||
|
|
||||||
|
## Backend DB traversal for Kvt DB
|
||||||
|
## ===============================
|
||||||
|
##
|
||||||
|
## This module provides iterators for the memory based backend or the
|
||||||
|
## backend-less database. Do import `kvt_walk/persistent` for the
|
||||||
|
## persistent backend though avoiding to unnecessarily link to the persistent
|
||||||
|
## backend library (e.g. `rocksdb`) when a memory only database is used.
|
||||||
|
##
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
./kvt_walk/memory_only
|
||||||
|
export
|
||||||
|
memory_only
|
||||||
|
|
||||||
|
# End
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
# Nimbus - Types, data structures and shared utilities used in network sync
|
||||||
|
#
|
||||||
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or
|
||||||
|
# distributed except according to those terms.
|
||||||
|
|
||||||
|
## Iterators for non-persistent backend of the Kvt DB
|
||||||
|
## ==================================================
|
||||||
|
##
|
||||||
|
import
|
||||||
|
eth/common,
|
||||||
|
../kvt_init/[memory_db, memory_only],
|
||||||
|
../kvt_init
|
||||||
|
export
|
||||||
|
memory_db,
|
||||||
|
memory_only
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public iterators (all in one)
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
iterator walkAny*(
|
||||||
|
be: MemBackendRef|VoidBackendRef;
|
||||||
|
): tuple[key: Blob, data: Blob] =
|
||||||
|
## Iterate over backend filters.
|
||||||
|
when be isnot VoidBackendRef:
|
||||||
|
mixin walk
|
||||||
|
|
||||||
|
for (k,v) in be.walk:
|
||||||
|
yield (k,v)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
# Nimbus - Types, data structures and shared utilities used in network sync
|
||||||
|
#
|
||||||
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or
|
||||||
|
# distributed except according to those terms.
|
||||||
|
|
||||||
|
## Iterators for persistent backend of the Kvt DB
|
||||||
|
## ==============================================
|
||||||
|
##
|
||||||
|
## This module automatically pulls in the persistent backend library at the
|
||||||
|
## linking stage (e.g. `rocksdb`) which can be avoided for pure memory DB
|
||||||
|
## applications by importing `./kvt_walk/memory_only` (rather than
|
||||||
|
## `./kvt_walk/persistent`.)
|
||||||
|
##
|
||||||
|
import
|
||||||
|
eth/common,
|
||||||
|
../kvt_init/[rocks_db, persistent],
|
||||||
|
./memory_only
|
||||||
|
export
|
||||||
|
rocks_db,
|
||||||
|
memory_only,
|
||||||
|
persistent
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public iterators (all in one)
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
iterator walkAny*(
|
||||||
|
be: RdbBackendRef;
|
||||||
|
): tuple[key: Blob, data: Blob] =
|
||||||
|
## Walk filter slots in fifo order.
|
||||||
|
for (k,v) in be.walk:
|
||||||
|
yield (k,v)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
Loading…
Reference in New Issue