kvstore: fix raising, be harsher on database errors (#923)
* kvstore: fix raising, be harsher on database errors * bump stew/serialization
This commit is contained in:
parent
ae9ca33aca
commit
6729d3c032
|
@ -1,5 +1,7 @@
|
|||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
options, typetraits,
|
||||
options, typetraits, stew/endians2,
|
||||
serialization, chronicles,
|
||||
spec/[datatypes, digest, crypto],
|
||||
kvstore, ssz
|
||||
|
@ -8,7 +10,17 @@ type
|
|||
BeaconChainDB* = ref object
|
||||
## Database storing resolved blocks and states - resolved blocks are such
|
||||
## blocks that form a chain back to the tail block.
|
||||
backend: KVStoreRef
|
||||
##
|
||||
## We assume that the database backend is working / not corrupt - as such,
|
||||
## we will raise a Defect any time there is an issue. This should be
|
||||
## revisited in the future, when/if the calling code safely can handle
|
||||
## corruption of this kind.
|
||||
##
|
||||
## We do however make an effort not to crash on invalid data inside the
|
||||
## database - this may have a number of "natural" causes such as switching
|
||||
## between different versions of the client and accidentally using an old
|
||||
## database.
|
||||
backend: KvStoreRef
|
||||
|
||||
DbKeyKind = enum
|
||||
kHashToState
|
||||
|
@ -22,6 +34,9 @@ type
|
|||
## past the weak subjectivity period.
|
||||
kBlockSlotStateRoot ## BlockSlot -> state_root mapping
|
||||
|
||||
# Subkeys essentially create "tables" within the key-value store by prefixing
|
||||
# each entry with a table id
|
||||
|
||||
func subkey(kind: DbKeyKind): array[1, byte] =
|
||||
result[0] = byte ord(kind)
|
||||
|
||||
|
@ -30,88 +45,79 @@ func subkey[N: static int](kind: DbKeyKind, key: array[N, byte]):
|
|||
result[0] = byte ord(kind)
|
||||
result[1 .. ^1] = key
|
||||
|
||||
func subkey(kind: DbKeyKind, key: uint64): array[sizeof(key) + 1, byte] =
|
||||
result[0] = byte ord(kind)
|
||||
copyMem(addr result[1], unsafeAddr key, sizeof(key))
|
||||
|
||||
func subkey(kind: type BeaconState, key: Eth2Digest): auto =
|
||||
subkey(kHashToState, key.data)
|
||||
|
||||
func subkey(kind: type SignedBeaconBlock, key: Eth2Digest): auto =
|
||||
subkey(kHashToBlock, key.data)
|
||||
|
||||
func subkey(root: Eth2Digest, slot: Slot): auto =
|
||||
# TODO: Copy the SSZ data to `ret` properly.
|
||||
# We don't need multiple calls to SSZ.encode
|
||||
# Use memoryStream(ret) and SszWriter explicitly
|
||||
|
||||
var
|
||||
# takes care of endians..
|
||||
rootSSZ = SSZ.encode(root)
|
||||
slotSSZ = SSZ.encode(slot)
|
||||
|
||||
var ret: array[1 + 32 + 8, byte]
|
||||
doAssert sizeof(ret) == 1 + rootSSZ.len + slotSSZ.len,
|
||||
"Can't sizeof this in VM"
|
||||
|
||||
func subkey(root: Eth2Digest, slot: Slot): array[40, byte] =
|
||||
var ret: array[40, byte]
|
||||
# big endian to get a naturally ascending order on slots in sorted indices
|
||||
ret[0..<8] = toBytesBE(slot.uint64)
|
||||
# .. but 7 bytes should be enough for slots - in return, we get a nicely
|
||||
# rounded key length
|
||||
ret[0] = byte ord(kBlockSlotStateRoot)
|
||||
|
||||
copyMem(addr ret[1], unsafeaddr root, sizeof(root))
|
||||
copyMem(addr ret[1 + sizeof(root)], unsafeaddr slot, sizeof(slot))
|
||||
ret[8..<40] = root.data
|
||||
|
||||
ret
|
||||
|
||||
proc init*(T: type BeaconChainDB, backend: KVStoreRef): BeaconChainDB =
|
||||
T(backend: backend)
|
||||
|
||||
proc put(db: BeaconChainDB, key: openArray[byte], v: auto) =
|
||||
db.backend.put(key, SSZ.encode(v)).expect("working database")
|
||||
|
||||
proc get(db: BeaconChainDB, key: openArray[byte], T: typedesc): Option[T] =
|
||||
var res: Option[T]
|
||||
proc decode(data: openArray[byte]) =
|
||||
try:
|
||||
res = some(SSZ.decode(data, T))
|
||||
except SerializationError as e:
|
||||
# If the data can't be deserialized, it could be because it's from a
|
||||
# version of the software that uses a different SSZ encoding
|
||||
warn "Unable to deserialize data, old database?", err = e.msg
|
||||
discard
|
||||
|
||||
discard db.backend.get(key, decode).expect("working database")
|
||||
|
||||
res
|
||||
|
||||
proc putBlock*(db: BeaconChainDB, key: Eth2Digest, value: SignedBeaconBlock) =
|
||||
db.backend.put(subkey(type value, key), SSZ.encode(value))
|
||||
db.put(subkey(type value, key), value)
|
||||
|
||||
proc putState*(db: BeaconChainDB, key: Eth2Digest, value: BeaconState) =
|
||||
# TODO prune old states - this is less easy than it seems as we never know
|
||||
# when or if a particular state will become finalized.
|
||||
|
||||
db.backend.put(subkey(type value, key), SSZ.encode(value))
|
||||
db.put(subkey(type value, key), value)
|
||||
|
||||
proc putState*(db: BeaconChainDB, value: BeaconState) =
|
||||
db.putState(hash_tree_root(value), value)
|
||||
|
||||
proc putStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot,
|
||||
value: Eth2Digest) =
|
||||
db.backend.put(subkey(root, slot), value.data)
|
||||
db.backend.put(subkey(root, slot), value.data).expect(
|
||||
"working database")
|
||||
|
||||
proc putBlock*(db: BeaconChainDB, value: SignedBeaconBlock) =
|
||||
db.putBlock(hash_tree_root(value.message), value)
|
||||
|
||||
proc delBlock*(db: BeaconChainDB, key: Eth2Digest) =
|
||||
db.backend.del(subkey(SignedBeaconBlock, key))
|
||||
db.backend.del(subkey(SignedBeaconBlock, key)).expect(
|
||||
"working database")
|
||||
|
||||
proc delState*(db: BeaconChainDB, key: Eth2Digest) =
|
||||
db.backend.del(subkey(BeaconState, key))
|
||||
db.backend.del(subkey(BeaconState, key)).expect("working database")
|
||||
|
||||
proc delStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot) =
|
||||
db.backend.del(subkey(root, slot))
|
||||
db.backend.del(subkey(root, slot)).expect("working database")
|
||||
|
||||
proc putHeadBlock*(db: BeaconChainDB, key: Eth2Digest) =
|
||||
db.backend.put(subkey(kHeadBlock), key.data)
|
||||
db.backend.put(subkey(kHeadBlock), key.data).expect("working database")
|
||||
|
||||
proc putTailBlock*(db: BeaconChainDB, key: Eth2Digest) =
|
||||
db.backend.put(subkey(kTailBlock), key.data)
|
||||
|
||||
proc get(db: BeaconChainDB, key: auto, T: typedesc): Option[T] =
|
||||
var res: Option[T]
|
||||
discard db.backend.get(key) do (data: openArray[byte]):
|
||||
try:
|
||||
res = some(SSZ.decode(data, T))
|
||||
except SerializationError:
|
||||
# Please note that this is intentionally a normal assert.
|
||||
# We consider this a hard failure in debug mode, because
|
||||
# it suggests a corrupted database. Release builds "recover"
|
||||
# from the situation by failing to deliver a result from the
|
||||
# database.
|
||||
assert false
|
||||
error "Corrupt database entry", key, `type` = name(T)
|
||||
res
|
||||
db.backend.put(subkey(kTailBlock), key.data).expect("working database")
|
||||
|
||||
proc getBlock*(db: BeaconChainDB, key: Eth2Digest): Option[SignedBeaconBlock] =
|
||||
db.get(subkey(SignedBeaconBlock, key), SignedBeaconBlock)
|
||||
|
@ -131,11 +137,11 @@ proc getTailBlock*(db: BeaconChainDB): Option[Eth2Digest] =
|
|||
|
||||
proc containsBlock*(
|
||||
db: BeaconChainDB, key: Eth2Digest): bool =
|
||||
db.backend.contains(subkey(SignedBeaconBlock, key))
|
||||
db.backend.contains(subkey(SignedBeaconBlock, key)).expect("working database")
|
||||
|
||||
proc containsState*(
|
||||
db: BeaconChainDB, key: Eth2Digest): bool =
|
||||
db.backend.contains(subkey(BeaconState, key))
|
||||
db.backend.contains(subkey(BeaconState, key)).expect("working database")
|
||||
|
||||
iterator getAncestors*(db: BeaconChainDB, root: Eth2Digest):
|
||||
tuple[root: Eth2Digest, blck: SignedBeaconBlock] =
|
||||
|
|
|
@ -148,7 +148,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
|
|||
netKeys = getPersistentNetKeys(conf)
|
||||
nickname = if conf.nodeName == "auto": shortForm(netKeys)
|
||||
else: conf.nodeName
|
||||
db = BeaconChainDB.init(kvStore SqliteStoreRef.init(conf.databaseDir))
|
||||
db = BeaconChainDB.init(kvStore SqStoreRef.init(conf.databaseDir, "nbc").tryGet())
|
||||
|
||||
var mainchainMonitor: MainchainMonitor
|
||||
|
||||
|
|
|
@ -230,8 +230,8 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
|
|||
# initialized on an epoch boundary, but that is a reasonable readability,
|
||||
# simplicity, and non-special-casing tradeoff for the inefficiency.
|
||||
cachedStates: [
|
||||
init(BeaconChainDB, kvStore MemoryStoreRef.init()),
|
||||
init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
init(BeaconChainDB, kvStore MemStoreRef.init()),
|
||||
init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
],
|
||||
|
||||
blocks: blocks,
|
||||
|
@ -590,7 +590,7 @@ proc putState(pool: BlockPool, state: HashedBeaconState, blck: BlockRef) =
|
|||
# by contrast, has just finished filling from the previous epoch. The
|
||||
# resulting lookback window is thus >= SLOTS_PER_EPOCH in size, while
|
||||
# bounded from above by 2*SLOTS_PER_EPOCH.
|
||||
currentCache = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
currentCache = init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
else:
|
||||
# Need to be able to efficiently access states for both attestation
|
||||
# aggregation and to process block proposals going back to the last
|
||||
|
|
|
@ -1,19 +1,35 @@
|
|||
# Simple Key-Value store database interface
|
||||
# beacon_chain
|
||||
# Copyright (c) 2018-2020 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
## Simple Key-Value store database interface that allows creating multiple
|
||||
## tables within each store
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
stew/results,
|
||||
tables, hashes, sets
|
||||
|
||||
export results
|
||||
|
||||
type
|
||||
MemoryStoreRef* = ref object of RootObj
|
||||
MemStoreRef* = ref object of RootObj
|
||||
records: Table[seq[byte], seq[byte]]
|
||||
|
||||
DataProc* = proc(val: openArray[byte])
|
||||
PutProc = proc (db: RootRef, key, val: openArray[byte]) {.gcsafe.}
|
||||
GetProc = proc (db: RootRef, key: openArray[byte], onData: DataProc): bool {.gcsafe.}
|
||||
DelProc = proc (db: RootRef, key: openArray[byte]) {.gcsafe.}
|
||||
ContainsProc = proc (db: RootRef, key: openArray[byte]): bool {.gcsafe.}
|
||||
KvResult*[T] = Result[T, cstring]
|
||||
|
||||
KVStoreRef* = ref object
|
||||
DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [Defect].}
|
||||
|
||||
PutProc = proc (db: RootRef, key, val: openArray[byte]): KvResult[void] {.nimcall, gcsafe, raises: [Defect].}
|
||||
GetProc = proc (db: RootRef, key: openArray[byte], onData: DataProc): KvResult[bool] {.nimcall, gcsafe, raises: [Defect].}
|
||||
DelProc = proc (db: RootRef, key: openArray[byte]): KvResult[void] {.nimcall, gcsafe, raises: [Defect].}
|
||||
ContainsProc = proc (db: RootRef, key: openArray[byte]): KvResult[bool] {.nimcall, gcsafe, raises: [Defect].}
|
||||
|
||||
KvStoreRef* = ref object
|
||||
## Key-Value store virtual interface
|
||||
obj: RootRef
|
||||
putProc: PutProc
|
||||
|
@ -21,74 +37,79 @@ type
|
|||
delProc: DelProc
|
||||
containsProc: ContainsProc
|
||||
|
||||
template put*(db: KVStoreRef, key, val: openArray[byte]) =
|
||||
template put*(db: KvStoreRef, key, val: openArray[byte]): KvResult[void] =
|
||||
## Store ``value`` at ``key`` - overwrites existing value if already present
|
||||
db.putProc(db.obj, key, val)
|
||||
|
||||
template get*(db: KVStoreRef, key: openArray[byte], onData: untyped): bool =
|
||||
template get*(db: KvStoreRef, key: openArray[byte], onData: untyped): KvResult[bool] =
|
||||
## Retrive value at ``key`` and call ``onData`` with the value. The data is
|
||||
## valid for the duration of the callback.
|
||||
## ``onData``: ``proc(data: openArray[byte])``
|
||||
## returns true if found and false otherwise.
|
||||
db.getProc(db.obj, key, onData)
|
||||
|
||||
template del*(db: KVStoreRef, key: openArray[byte]) =
|
||||
template del*(db: KvStoreRef, key: openArray[byte]): KvResult[void] =
|
||||
## Remove value at ``key`` from store - do nothing if the value is not present
|
||||
db.delProc(db.obj, key)
|
||||
|
||||
template contains*(db: KVStoreRef, key: openArray[byte]): bool =
|
||||
template contains*(db: KvStoreRef, key: openArray[byte]): KvResult[bool] =
|
||||
## Return true iff ``key`` has a value in store
|
||||
db.containsProc(db.obj, key)
|
||||
|
||||
proc get*(db: MemoryStoreRef, key: openArray[byte], onData: DataProc): bool =
|
||||
let key = @key
|
||||
db.records.withValue(key, v):
|
||||
onData(v[])
|
||||
return true
|
||||
|
||||
proc del*(db: MemoryStoreRef, key: openArray[byte]) =
|
||||
# TODO: This is quite inefficient and it won't be necessary once
|
||||
# https://github.com/nim-lang/Nim/issues/7457 is developed.
|
||||
let key = @key
|
||||
db.records.del(key)
|
||||
|
||||
proc contains*(db: MemoryStoreRef, key: openArray[byte]): bool =
|
||||
db.records.contains(@key)
|
||||
|
||||
proc put*(db: MemoryStoreRef, key, val: openArray[byte]) =
|
||||
# TODO: This is quite inefficient and it won't be necessary once
|
||||
# https://github.com/nim-lang/Nim/issues/7457 is developed.
|
||||
let key = @key
|
||||
db.records[key] = @val
|
||||
|
||||
proc init*(T: type MemoryStoreRef): T =
|
||||
T(
|
||||
records: initTable[seq[byte], seq[byte]]()
|
||||
)
|
||||
|
||||
proc putImpl[T](db: RootRef, key, val: openArray[byte]) =
|
||||
proc putImpl[T](db: RootRef, key, val: openArray[byte]): KvResult[void] =
|
||||
mixin put
|
||||
put(T(db), key, val)
|
||||
|
||||
proc getImpl[T](db: RootRef, key: openArray[byte], onData: DataProc): bool =
|
||||
proc getImpl[T](db: RootRef, key: openArray[byte], onData: DataProc): KvResult[bool] =
|
||||
mixin get
|
||||
get(T(db), key, onData)
|
||||
|
||||
proc delImpl[T](db: RootRef, key: openArray[byte]) =
|
||||
proc delImpl[T](db: RootRef, key: openArray[byte]): KvResult[void] =
|
||||
mixin del
|
||||
del(T(db), key)
|
||||
|
||||
proc containsImpl[T](db: RootRef, key: openArray[byte]): bool =
|
||||
proc containsImpl[T](db: RootRef, key: openArray[byte]): KvResult[bool] =
|
||||
mixin contains
|
||||
contains(T(db), key)
|
||||
|
||||
func kvStore*[T: RootRef](x: T): KVStoreRef =
|
||||
func kvStore*[T: RootRef](x: T): KvStoreRef =
|
||||
mixin del, get, put, contains
|
||||
|
||||
KVStoreRef(
|
||||
KvStoreRef(
|
||||
obj: x,
|
||||
putProc: putImpl[T],
|
||||
getProc: getImpl[T],
|
||||
delProc: delImpl[T],
|
||||
containsProc: containsImpl[T]
|
||||
)
|
||||
|
||||
proc get*(db: MemStoreRef, key: openArray[byte], onData: DataProc): KvResult[bool] =
|
||||
let key = @key
|
||||
|
||||
db.records.withValue(key, v):
|
||||
onData(v[])
|
||||
return ok(true)
|
||||
|
||||
ok(false)
|
||||
|
||||
proc del*(db: MemStoreRef, key: openArray[byte]): KvResult[void] =
|
||||
# TODO: This is quite inefficient and it won't be necessary once
|
||||
# https://github.com/nim-lang/Nim/issues/7457 is developed.
|
||||
let key = @key
|
||||
db.records.del(key)
|
||||
ok()
|
||||
|
||||
proc contains*(db: MemStoreRef, key: openArray[byte]): KvResult[bool] =
|
||||
ok(db.records.contains(@key))
|
||||
|
||||
proc put*(db: MemStoreRef, key, val: openArray[byte]): KvResult[void] =
|
||||
# TODO: This is quite inefficient and it won't be necessary once
|
||||
# https://github.com/nim-lang/Nim/issues/7457 is developed.
|
||||
let key = @key
|
||||
db.records[key] = @val
|
||||
ok()
|
||||
|
||||
proc init*(T: type MemStoreRef): T =
|
||||
T(
|
||||
records: initTable[seq[byte], seq[byte]]()
|
||||
)
|
||||
|
|
|
@ -1,25 +1,23 @@
|
|||
## Implementation of KVStore based on Sqlite3
|
||||
## Implementation of KvStore based on sqlite3
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
os,
|
||||
sqlite3_abi,
|
||||
./kvstore
|
||||
|
||||
export kvstore
|
||||
|
||||
type
|
||||
SqliteStoreRef* = ref object of RootObj
|
||||
SqStoreRef* = ref object of RootObj
|
||||
env: ptr sqlite3
|
||||
selectStmt, insertStmt, deleteStmt: ptr sqlite3_stmt
|
||||
|
||||
SqliteError* = object of CatchableError
|
||||
|
||||
func raiseError(err: cint) {.noreturn.} =
|
||||
let tmp = sqlite3_errstr(err)
|
||||
raise (ref SqliteError)(msg: $tmp)
|
||||
|
||||
template checkErr(op, cleanup: untyped) =
|
||||
if (let v = (op); v != SQLITE_OK):
|
||||
cleanup
|
||||
raiseError(v)
|
||||
return err(sqlite3_errstr(v))
|
||||
|
||||
template checkErr(op) =
|
||||
checkErr(op): discard
|
||||
|
@ -27,7 +25,7 @@ template checkErr(op) =
|
|||
proc bindBlob(s: ptr sqlite3_stmt, n: int, blob: openarray[byte]): cint =
|
||||
sqlite3_bind_blob(s, n.cint, unsafeAddr blob[0], blob.len.cint, nil)
|
||||
|
||||
proc get*(db: SqliteStoreRef, key: openarray[byte], onData: DataProc): bool =
|
||||
proc get*(db: SqStoreRef, key: openarray[byte], onData: DataProc): KvResult[bool] =
|
||||
checkErr sqlite3_reset(db.selectStmt)
|
||||
checkErr sqlite3_clear_bindings(db.selectStmt)
|
||||
checkErr bindBlob(db.selectStmt, 1, key)
|
||||
|
@ -39,13 +37,13 @@ proc get*(db: SqliteStoreRef, key: openarray[byte], onData: DataProc): bool =
|
|||
p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(db.selectStmt, 0))
|
||||
l = sqlite3_column_bytes(db.selectStmt, 0)
|
||||
onData(toOpenArray(p, 0, l-1))
|
||||
true
|
||||
ok(true)
|
||||
of SQLITE_DONE:
|
||||
false
|
||||
ok(false)
|
||||
else:
|
||||
raiseError(v)
|
||||
err(sqlite3_errstr(v))
|
||||
|
||||
proc put*(db: SqliteStoreRef, key, value: openarray[byte]) =
|
||||
proc put*(db: SqStoreRef, key, value: openarray[byte]): KvResult[void] =
|
||||
checkErr sqlite3_reset(db.insertStmt)
|
||||
checkErr sqlite3_clear_bindings(db.insertStmt)
|
||||
|
||||
|
@ -53,9 +51,11 @@ proc put*(db: SqliteStoreRef, key, value: openarray[byte]) =
|
|||
checkErr bindBlob(db.insertStmt, 2, value)
|
||||
|
||||
if (let v = sqlite3_step(db.insertStmt); v != SQLITE_DONE):
|
||||
raiseError(v)
|
||||
err(sqlite3_errstr(v))
|
||||
else:
|
||||
ok()
|
||||
|
||||
proc contains*(db: SqliteStoreRef, key: openarray[byte]): bool =
|
||||
proc contains*(db: SqStoreRef, key: openarray[byte]): KvResult[bool] =
|
||||
checkErr sqlite3_reset(db.selectStmt)
|
||||
checkErr sqlite3_clear_bindings(db.selectStmt)
|
||||
|
||||
|
@ -63,45 +63,52 @@ proc contains*(db: SqliteStoreRef, key: openarray[byte]): bool =
|
|||
|
||||
let v = sqlite3_step(db.selectStmt)
|
||||
case v
|
||||
of SQLITE_ROW: result = true
|
||||
of SQLITE_DONE: result = false
|
||||
else: raiseError(v)
|
||||
of SQLITE_ROW: ok(true)
|
||||
of SQLITE_DONE: ok(false)
|
||||
else: err(sqlite3_errstr(v))
|
||||
|
||||
proc del*(db: SqliteStoreRef, key: openarray[byte]) =
|
||||
proc del*(db: SqStoreRef, key: openarray[byte]): KvResult[void] =
|
||||
checkErr sqlite3_reset(db.deleteStmt)
|
||||
checkErr sqlite3_clear_bindings(db.deleteStmt)
|
||||
|
||||
checkErr bindBlob(db.deleteStmt, 1, key)
|
||||
|
||||
if (let v = sqlite3_step(db.deleteStmt); v != SQLITE_DONE):
|
||||
raiseError(v)
|
||||
err(sqlite3_errstr(v))
|
||||
else:
|
||||
ok()
|
||||
|
||||
proc close*(db: SqliteStoreRef) =
|
||||
proc close*(db: SqStoreRef) =
|
||||
discard sqlite3_finalize(db.insertStmt)
|
||||
discard sqlite3_finalize(db.selectStmt)
|
||||
discard sqlite3_finalize(db.deleteStmt)
|
||||
|
||||
discard sqlite3_close(db.env)
|
||||
|
||||
db[] = SqliteStoreRef()[]
|
||||
db[] = SqStoreRef()[]
|
||||
|
||||
proc init*(
|
||||
T: type SqliteStoreRef,
|
||||
T: type SqStoreRef,
|
||||
basePath: string,
|
||||
name: string,
|
||||
readOnly = false,
|
||||
inMemory = false): T =
|
||||
inMemory = false): KvResult[T] =
|
||||
var
|
||||
env: ptr sqlite3
|
||||
|
||||
let
|
||||
name =
|
||||
if inMemory: ":memory:"
|
||||
else: basepath / "nimbus.sqlite3"
|
||||
else: basepath / name & ".sqlite3"
|
||||
flags =
|
||||
if readOnly: SQLITE_OPEN_READONLY
|
||||
else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE
|
||||
|
||||
if not inMemory:
|
||||
createDir(basePath)
|
||||
try:
|
||||
createDir(basePath)
|
||||
except OSError, IOError:
|
||||
return err("`sqlite: cannot create database directory")
|
||||
|
||||
checkErr sqlite3_open_v2(name, addr env, flags.cint, nil)
|
||||
|
||||
|
@ -118,11 +125,11 @@ proc init*(
|
|||
if (let x = sqlite3_step(s); x != SQLITE_DONE):
|
||||
discard sqlite3_finalize(s)
|
||||
discard sqlite3_close(env)
|
||||
raiseError(x)
|
||||
return err(sqlite3_errstr(x))
|
||||
|
||||
if (let x = sqlite3_finalize(s); x != SQLITE_OK):
|
||||
discard sqlite3_close(env)
|
||||
raiseError(x)
|
||||
return err(sqlite3_errstr(x))
|
||||
|
||||
# TODO: check current version and implement schema versioning
|
||||
checkExec "PRAGMA user_version = 1;"
|
||||
|
@ -143,9 +150,9 @@ proc init*(
|
|||
discard sqlite3_finalize(selectStmt)
|
||||
discard sqlite3_finalize(insertStmt)
|
||||
|
||||
T(
|
||||
ok(SqStoreRef(
|
||||
env: env,
|
||||
selectStmt: selectStmt,
|
||||
insertStmt: insertStmt,
|
||||
deleteStmt: deleteStmt
|
||||
)
|
||||
))
|
||||
|
|
|
@ -16,7 +16,7 @@ import options, unittest, sequtils,
|
|||
suiteReport "Beacon chain DB" & preset():
|
||||
timedTest "empty database" & preset():
|
||||
var
|
||||
db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
db = init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
|
||||
check:
|
||||
when const_preset=="minimal":
|
||||
|
@ -27,7 +27,7 @@ suiteReport "Beacon chain DB" & preset():
|
|||
|
||||
timedTest "sanity check blocks" & preset():
|
||||
var
|
||||
db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
db = init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
|
||||
let
|
||||
signedBlock = SignedBeaconBlock()
|
||||
|
@ -45,7 +45,7 @@ suiteReport "Beacon chain DB" & preset():
|
|||
|
||||
timedTest "sanity check states" & preset():
|
||||
var
|
||||
db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
db = init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
|
||||
let
|
||||
state = BeaconState()
|
||||
|
@ -59,7 +59,7 @@ suiteReport "Beacon chain DB" & preset():
|
|||
|
||||
timedTest "find ancestors" & preset():
|
||||
var
|
||||
db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
db = init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
|
||||
let
|
||||
a0 = SignedBeaconBlock(message: BeaconBlock(slot: GENESIS_SLOT + 0))
|
||||
|
@ -95,7 +95,7 @@ suiteReport "Beacon chain DB" & preset():
|
|||
# serialization where an all-zero default-initialized bls signature could
|
||||
# not be deserialized because the deserialization was too strict.
|
||||
var
|
||||
db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
db = init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
|
||||
let
|
||||
state = initialize_beacon_state_from_eth1(
|
||||
|
|
|
@ -4,42 +4,44 @@ import
|
|||
unittest,
|
||||
../beacon_chain/kvstore
|
||||
|
||||
proc testKVStore*(db: KVStoreRef) =
|
||||
let
|
||||
key = [0'u8, 1, 2, 3]
|
||||
value = [3'u8, 2, 1, 0]
|
||||
value2 = [5'u8, 2, 1, 0]
|
||||
const
|
||||
key = [0'u8, 1, 2, 3]
|
||||
value = [3'u8, 2, 1, 0]
|
||||
value2 = [5'u8, 2, 1, 0]
|
||||
|
||||
proc testKvStore*(db: KvStoreRef) =
|
||||
check:
|
||||
db != nil
|
||||
|
||||
not db.get(key, proc(data: openArray[byte]) = discard)
|
||||
not db.contains(key)
|
||||
not db.get(key, proc(data: openArray[byte]) = discard)[]
|
||||
not db.contains(key)[]
|
||||
|
||||
db.del(key) # does nothing
|
||||
db.del(key)[] # does nothing
|
||||
|
||||
db.put(key, value)
|
||||
db.put(key, value)[]
|
||||
|
||||
var v: seq[byte]
|
||||
proc grab(data: openArray[byte]) =
|
||||
v = @data
|
||||
|
||||
check:
|
||||
db.contains(key)
|
||||
db.get(key, proc(data: openArray[byte]) =
|
||||
check data == value
|
||||
)
|
||||
db.contains(key)[]
|
||||
db.get(key, grab)[]
|
||||
v == value
|
||||
|
||||
db.put(key, value2) # overwrite old value
|
||||
db.put(key, value2)[] # overwrite old value
|
||||
check:
|
||||
db.contains(key)
|
||||
db.get(key, proc(data: openArray[byte]) =
|
||||
check data == value2
|
||||
)
|
||||
db.contains(key)[]
|
||||
db.get(key, grab)[]
|
||||
v == value2
|
||||
|
||||
db.del(key)
|
||||
db.del(key)[]
|
||||
check:
|
||||
not db.get(key, proc(data: openArray[byte]) = discard)
|
||||
not db.contains(key)
|
||||
not db.get(key, proc(data: openArray[byte]) = discard)[]
|
||||
not db.contains(key)[]
|
||||
|
||||
db.del(key) # does nothing
|
||||
db.del(key)[] # does nothing
|
||||
|
||||
suite "MemoryStoreRef":
|
||||
test "KVStore interface":
|
||||
testKVStore(kvStore MemoryStoreRef.init())
|
||||
test "KvStore interface":
|
||||
testKvStore(kvStore MemStoreRef.init())
|
||||
|
|
|
@ -6,9 +6,9 @@ import
|
|||
../beacon_chain/[kvstore, kvstore_sqlite3],
|
||||
./test_kvstore
|
||||
|
||||
suite "Sqlite":
|
||||
test "KVStore interface":
|
||||
let db = SqliteStoreRef.init("", inMemory = true)
|
||||
suite "SqStoreRef":
|
||||
test "KvStore interface":
|
||||
let db = SqStoreRef.init("", "test", inMemory = true)[]
|
||||
defer: db.close()
|
||||
|
||||
testKVStore(kvStore db)
|
||||
testKvStore(kvStore db)
|
||||
|
|
|
@ -93,7 +93,7 @@ template timedTest*(name, body) =
|
|||
testTimes.add (f, name)
|
||||
|
||||
proc makeTestDB*(tailState: BeaconState, tailBlock: SignedBeaconBlock): BeaconChainDB =
|
||||
result = init(BeaconChainDB, kvStore MemoryStoreRef.init())
|
||||
result = init(BeaconChainDB, kvStore MemStoreRef.init())
|
||||
BlockPool.preInit(result, tailState, tailBlock)
|
||||
|
||||
proc makeTestDB*(validators: int): BeaconChainDB =
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit db037cbece06633021d4b8e2374704d9ffdb53f7
|
||||
Subproject commit 1989a551459caf9f0d20260e359f40249e31ef01
|
|
@ -1 +1 @@
|
|||
Subproject commit ff755bbf75d0d3f387b9f352b241d98eb3372323
|
||||
Subproject commit 8065e36c5af31f2f3f3b0d9ea242ae4eef193a30
|
Loading…
Reference in New Issue