kvstore: fix raising, be harsher on database errors (#923)

* kvstore: fix raising, be harsher on database errors

* bump stew/serialization

beacon_chain/beacon_chain_db.nim

@@ -1,5 +1,7 @@
+{.push raises: [Defect].}
+
 import
-  options, typetraits,
+  options, typetraits, stew/endians2,
   serialization, chronicles,
   spec/[datatypes, digest, crypto],
   kvstore, ssz
@@ -8,7 +10,17 @@ type
   BeaconChainDB* = ref object
     ## Database storing resolved blocks and states - resolved blocks are such
     ## blocks that form a chain back to the tail block.
-    backend: KVStoreRef
+    ##
+    ## We assume that the database backend is working / not corrupt - as such,
+    ## we will raise a Defect any time there is an issue. This should be
+    ## revisited in the future, when/if the calling code safely can handle
+    ## corruption of this kind.
+    ##
+    ## We do however make an effort not to crash on invalid data inside the
+    ## database - this may have a number of "natural" causes such as switching
+    ## between different versions of the client and accidentally using an old
+    ## database.
+    backend: KvStoreRef
 
   DbKeyKind = enum
     kHashToState
@@ -22,6 +34,9 @@ type
     ##       past the weak subjectivity period.
     kBlockSlotStateRoot ## BlockSlot -> state_root mapping
 
+# Subkeys essentially create "tables" within the key-value store by prefixing
+# each entry with a table id
+
 func subkey(kind: DbKeyKind): array[1, byte] =
   result[0] = byte ord(kind)
 
@@ -30,88 +45,79 @@ func subkey[N: static int](kind: DbKeyKind, key: array[N, byte]):
   result[0] = byte ord(kind)
   result[1 .. ^1] = key
 
-func subkey(kind: DbKeyKind, key: uint64): array[sizeof(key) + 1, byte] =
-  result[0] = byte ord(kind)
-  copyMem(addr result[1], unsafeAddr key, sizeof(key))
-
 func subkey(kind: type BeaconState, key: Eth2Digest): auto =
   subkey(kHashToState, key.data)
 
 func subkey(kind: type SignedBeaconBlock, key: Eth2Digest): auto =
   subkey(kHashToBlock, key.data)
 
-func subkey(root: Eth2Digest, slot: Slot): auto =
+func subkey(root: Eth2Digest, slot: Slot): array[40, byte] =
-  # TODO: Copy the SSZ data to `ret` properly.
+  var ret: array[40, byte]
-  # We don't need multiple calls to SSZ.encode
+  # big endian to get a naturally ascending order on slots in sorted indices
-  # Use memoryStream(ret) and SszWriter explicitly
+  ret[0..<8] = toBytesBE(slot.uint64)
-
+  # .. but 7 bytes should be enough for slots - in return, we get a nicely
-  var
+  # rounded key length
-    # takes care of endians..
-    rootSSZ = SSZ.encode(root)
-    slotSSZ = SSZ.encode(slot)
-
-  var ret: array[1 + 32 + 8, byte]
-  doAssert sizeof(ret) == 1 + rootSSZ.len + slotSSZ.len,
-    "Can't sizeof this in VM"
-
   ret[0] = byte ord(kBlockSlotStateRoot)
-
+  ret[8..<40] = root.data
-  copyMem(addr ret[1], unsafeaddr root, sizeof(root))
-  copyMem(addr ret[1 + sizeof(root)], unsafeaddr slot, sizeof(slot))
 
   ret
 
 proc init*(T: type BeaconChainDB, backend: KVStoreRef): BeaconChainDB =
   T(backend: backend)
 
+proc put(db: BeaconChainDB, key: openArray[byte], v: auto) =
+  db.backend.put(key, SSZ.encode(v)).expect("working database")
+
+proc get(db: BeaconChainDB, key: openArray[byte], T: typedesc): Option[T] =
+  var res: Option[T]
+  proc decode(data: openArray[byte]) =
+    try:
+      res = some(SSZ.decode(data, T))
+    except SerializationError as e:
+      # If the data can't be deserialized, it could be because it's from a
+      # version of the software that uses a different SSZ encoding
+      warn "Unable to deserialize data, old database?", err = e.msg
+      discard
+
+  discard db.backend.get(key, decode).expect("working database")
+
+  res
+
 proc putBlock*(db: BeaconChainDB, key: Eth2Digest, value: SignedBeaconBlock) =
-  db.backend.put(subkey(type value, key), SSZ.encode(value))
+  db.put(subkey(type value, key), value)
 
 proc putState*(db: BeaconChainDB, key: Eth2Digest, value: BeaconState) =
   # TODO prune old states - this is less easy than it seems as we never know
   #      when or if a particular state will become finalized.
 
-  db.backend.put(subkey(type value, key), SSZ.encode(value))
+  db.put(subkey(type value, key), value)
 
 proc putState*(db: BeaconChainDB, value: BeaconState) =
   db.putState(hash_tree_root(value), value)
 
 proc putStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot,
     value: Eth2Digest) =
-  db.backend.put(subkey(root, slot), value.data)
+  db.backend.put(subkey(root, slot), value.data).expect(
+    "working database")
 
 proc putBlock*(db: BeaconChainDB, value: SignedBeaconBlock) =
   db.putBlock(hash_tree_root(value.message), value)
 
 proc delBlock*(db: BeaconChainDB, key: Eth2Digest) =
-  db.backend.del(subkey(SignedBeaconBlock, key))
+  db.backend.del(subkey(SignedBeaconBlock, key)).expect(
+    "working database")
 
 proc delState*(db: BeaconChainDB, key: Eth2Digest) =
-  db.backend.del(subkey(BeaconState, key))
+  db.backend.del(subkey(BeaconState, key)).expect("working database")
 
 proc delStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot) =
-  db.backend.del(subkey(root, slot))
+  db.backend.del(subkey(root, slot)).expect("working database")
 
 proc putHeadBlock*(db: BeaconChainDB, key: Eth2Digest) =
-  db.backend.put(subkey(kHeadBlock), key.data)
+  db.backend.put(subkey(kHeadBlock), key.data).expect("working database")
 
 proc putTailBlock*(db: BeaconChainDB, key: Eth2Digest) =
-  db.backend.put(subkey(kTailBlock), key.data)
+  db.backend.put(subkey(kTailBlock), key.data).expect("working database")
-
-proc get(db: BeaconChainDB, key: auto, T: typedesc): Option[T] =
-  var res: Option[T]
-  discard db.backend.get(key) do (data: openArray[byte]):
-    try:
-      res = some(SSZ.decode(data, T))
-    except SerializationError:
-      # Please note that this is intentionally a normal assert.
-      # We consider this a hard failure in debug mode, because
-      # it suggests a corrupted database. Release builds "recover"
-      # from the situation by failing to deliver a result from the
-      # database.
-      assert false
-      error "Corrupt database entry", key, `type` = name(T)
-  res
 
 proc getBlock*(db: BeaconChainDB, key: Eth2Digest): Option[SignedBeaconBlock] =
   db.get(subkey(SignedBeaconBlock, key), SignedBeaconBlock)
@@ -131,11 +137,11 @@ proc getTailBlock*(db: BeaconChainDB): Option[Eth2Digest] =
 
 proc containsBlock*(
     db: BeaconChainDB, key: Eth2Digest): bool =
-  db.backend.contains(subkey(SignedBeaconBlock, key))
+  db.backend.contains(subkey(SignedBeaconBlock, key)).expect("working database")
 
 proc containsState*(
     db: BeaconChainDB, key: Eth2Digest): bool =
-  db.backend.contains(subkey(BeaconState, key))
+  db.backend.contains(subkey(BeaconState, key)).expect("working database")
 
 iterator getAncestors*(db: BeaconChainDB, root: Eth2Digest):
     tuple[root: Eth2Digest, blck: SignedBeaconBlock] =

beacon_chain/beacon_node.nim

@@ -148,7 +148,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
     netKeys = getPersistentNetKeys(conf)
     nickname = if conf.nodeName == "auto": shortForm(netKeys)
                else: conf.nodeName
-    db = BeaconChainDB.init(kvStore SqliteStoreRef.init(conf.databaseDir))
+    db = BeaconChainDB.init(kvStore SqStoreRef.init(conf.databaseDir, "nbc").tryGet())
 
   var mainchainMonitor: MainchainMonitor
 

beacon_chain/block_pool.nim

@@ -230,8 +230,8 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
     # initialized on an epoch boundary, but that is a reasonable readability,
     # simplicity, and non-special-casing tradeoff for the inefficiency.
     cachedStates: [
-      init(BeaconChainDB, kvStore MemoryStoreRef.init()),
+      init(BeaconChainDB, kvStore MemStoreRef.init()),
-      init(BeaconChainDB, kvStore MemoryStoreRef.init())
+      init(BeaconChainDB, kvStore MemStoreRef.init())
     ],
 
     blocks: blocks,
@@ -590,7 +590,7 @@ proc putState(pool: BlockPool, state: HashedBeaconState, blck: BlockRef) =
       # by contrast, has just finished filling from the previous epoch. The
       # resulting lookback window is thus >= SLOTS_PER_EPOCH in size, while
       # bounded from above by 2*SLOTS_PER_EPOCH.
-      currentCache = init(BeaconChainDB, kvStore MemoryStoreRef.init())
+      currentCache = init(BeaconChainDB, kvStore MemStoreRef.init())
   else:
     # Need to be able to efficiently access states for both attestation
     # aggregation and to process block proposals going back to the last

beacon_chain/kvstore.nim

@@ -1,19 +1,35 @@
-# Simple Key-Value store database interface
+# beacon_chain
+# Copyright (c) 2018-2020 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+## Simple Key-Value store database interface that allows creating multiple
+## tables within each store
+
+{.push raises: [Defect].}
 
 import
+  stew/results,
   tables, hashes, sets
 
+export results
+
 type
-  MemoryStoreRef* = ref object of RootObj
+  MemStoreRef* = ref object of RootObj
     records: Table[seq[byte], seq[byte]]
 
-  DataProc* = proc(val: openArray[byte])
+  KvResult*[T] = Result[T, cstring]
-  PutProc = proc (db: RootRef, key, val: openArray[byte]) {.gcsafe.}
-  GetProc = proc (db: RootRef, key: openArray[byte], onData: DataProc): bool {.gcsafe.}
-  DelProc = proc (db: RootRef, key: openArray[byte]) {.gcsafe.}
-  ContainsProc = proc (db: RootRef, key: openArray[byte]): bool {.gcsafe.}
 
-  KVStoreRef* = ref object
+  DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [Defect].}
+
+  PutProc = proc (db: RootRef, key, val: openArray[byte]): KvResult[void] {.nimcall, gcsafe, raises: [Defect].}
+  GetProc = proc (db: RootRef, key: openArray[byte], onData: DataProc): KvResult[bool] {.nimcall, gcsafe, raises: [Defect].}
+  DelProc = proc (db: RootRef, key: openArray[byte]): KvResult[void] {.nimcall, gcsafe, raises: [Defect].}
+  ContainsProc = proc (db: RootRef, key: openArray[byte]): KvResult[bool] {.nimcall, gcsafe, raises: [Defect].}
+
+  KvStoreRef* = ref object
     ## Key-Value store virtual interface
     obj: RootRef
     putProc: PutProc
@@ -21,74 +37,79 @@ type
     delProc: DelProc
     containsProc: ContainsProc
 
-template put*(db: KVStoreRef, key, val: openArray[byte]) =
+template put*(db: KvStoreRef, key, val: openArray[byte]): KvResult[void] =
   ## Store ``value`` at ``key`` - overwrites existing value if already present
   db.putProc(db.obj, key, val)
 
-template get*(db: KVStoreRef, key: openArray[byte], onData: untyped): bool =
+template get*(db: KvStoreRef, key: openArray[byte], onData: untyped): KvResult[bool] =
   ## Retrive value at ``key`` and call ``onData`` with the value. The data is
   ## valid for the duration of the callback.
   ## ``onData``: ``proc(data: openArray[byte])``
   ## returns true if found and false otherwise.
   db.getProc(db.obj, key, onData)
 
-template del*(db: KVStoreRef, key: openArray[byte]) =
+template del*(db: KvStoreRef, key: openArray[byte]): KvResult[void] =
   ## Remove value at ``key`` from store - do nothing if the value is not present
   db.delProc(db.obj, key)
 
-template contains*(db: KVStoreRef, key: openArray[byte]): bool =
+template contains*(db: KvStoreRef, key: openArray[byte]): KvResult[bool] =
   ## Return true iff ``key`` has a value in store
   db.containsProc(db.obj, key)
 
-proc get*(db: MemoryStoreRef, key: openArray[byte], onData: DataProc): bool =
+proc putImpl[T](db: RootRef, key, val: openArray[byte]): KvResult[void] =
-  let key = @key
-  db.records.withValue(key, v):
-    onData(v[])
-    return true
-
-proc del*(db: MemoryStoreRef, key: openArray[byte]) =
-  # TODO: This is quite inefficient and it won't be necessary once
-  # https://github.com/nim-lang/Nim/issues/7457 is developed.
-  let key = @key
-  db.records.del(key)
-
-proc contains*(db: MemoryStoreRef, key: openArray[byte]): bool =
-  db.records.contains(@key)
-
-proc put*(db: MemoryStoreRef, key, val: openArray[byte]) =
-  # TODO: This is quite inefficient and it won't be necessary once
-  # https://github.com/nim-lang/Nim/issues/7457 is developed.
-  let key = @key
-  db.records[key] = @val
-
-proc init*(T: type MemoryStoreRef): T =
-  T(
-    records: initTable[seq[byte], seq[byte]]()
-  )
-
-proc putImpl[T](db: RootRef, key, val: openArray[byte]) =
   mixin put
   put(T(db), key, val)
 
-proc getImpl[T](db: RootRef, key: openArray[byte], onData: DataProc): bool =
+proc getImpl[T](db: RootRef, key: openArray[byte], onData: DataProc): KvResult[bool] =
   mixin get
   get(T(db), key, onData)
 
-proc delImpl[T](db: RootRef, key: openArray[byte]) =
+proc delImpl[T](db: RootRef, key: openArray[byte]): KvResult[void] =
   mixin del
   del(T(db), key)
 
-proc containsImpl[T](db: RootRef, key: openArray[byte]): bool =
+proc containsImpl[T](db: RootRef, key: openArray[byte]): KvResult[bool] =
   mixin contains
   contains(T(db), key)
 
-func kvStore*[T: RootRef](x: T): KVStoreRef =
+func kvStore*[T: RootRef](x: T): KvStoreRef =
   mixin del, get, put, contains
 
-  KVStoreRef(
+  KvStoreRef(
     obj: x,
     putProc: putImpl[T],
     getProc: getImpl[T],
     delProc: delImpl[T],
     containsProc: containsImpl[T]
   )
+
+proc get*(db: MemStoreRef, key: openArray[byte], onData: DataProc): KvResult[bool] =
+  let key = @key
+
+  db.records.withValue(key, v):
+    onData(v[])
+    return ok(true)
+
+  ok(false)
+
+proc del*(db: MemStoreRef, key: openArray[byte]): KvResult[void] =
+  # TODO: This is quite inefficient and it won't be necessary once
+  # https://github.com/nim-lang/Nim/issues/7457 is developed.
+  let key = @key
+  db.records.del(key)
+  ok()
+
+proc contains*(db: MemStoreRef, key: openArray[byte]): KvResult[bool] =
+  ok(db.records.contains(@key))
+
+proc put*(db: MemStoreRef, key, val: openArray[byte]): KvResult[void] =
+  # TODO: This is quite inefficient and it won't be necessary once
+  # https://github.com/nim-lang/Nim/issues/7457 is developed.
+  let key = @key
+  db.records[key] = @val
+  ok()
+
+proc init*(T: type MemStoreRef): T =
+  T(
+    records: initTable[seq[byte], seq[byte]]()
+  )

beacon_chain/kvstore_sqlite3.nim

@@ -1,25 +1,23 @@
-## Implementation of KVStore based on Sqlite3
+## Implementation of KvStore based on sqlite3
+
+{.push raises: [Defect].}
 
 import
   os,
   sqlite3_abi,
   ./kvstore
 
+export kvstore
+
 type
-  SqliteStoreRef* = ref object of RootObj
+  SqStoreRef* = ref object of RootObj
     env: ptr sqlite3
     selectStmt, insertStmt, deleteStmt: ptr sqlite3_stmt
 
-  SqliteError* = object of CatchableError
-
-func raiseError(err: cint) {.noreturn.} =
-  let tmp = sqlite3_errstr(err)
-  raise (ref SqliteError)(msg: $tmp)
-
 template checkErr(op, cleanup: untyped) =
   if (let v = (op); v != SQLITE_OK):
     cleanup
-    raiseError(v)
+    return err(sqlite3_errstr(v))
 
 template checkErr(op) =
   checkErr(op): discard
@@ -27,7 +25,7 @@ template checkErr(op) =
 proc bindBlob(s: ptr sqlite3_stmt, n: int, blob: openarray[byte]): cint =
   sqlite3_bind_blob(s, n.cint, unsafeAddr blob[0], blob.len.cint, nil)
 
-proc get*(db: SqliteStoreRef, key: openarray[byte], onData: DataProc): bool =
+proc get*(db: SqStoreRef, key: openarray[byte], onData: DataProc): KvResult[bool] =
   checkErr sqlite3_reset(db.selectStmt)
   checkErr sqlite3_clear_bindings(db.selectStmt)
   checkErr bindBlob(db.selectStmt, 1, key)
@@ -39,13 +37,13 @@ proc get*(db: SqliteStoreRef, key: openarray[byte], onData: DataProc): bool =
       p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(db.selectStmt, 0))
       l = sqlite3_column_bytes(db.selectStmt, 0)
     onData(toOpenArray(p, 0, l-1))
-    true
+    ok(true)
   of SQLITE_DONE:
-    false
+    ok(false)
   else:
-    raiseError(v)
+    err(sqlite3_errstr(v))
 
-proc put*(db: SqliteStoreRef, key, value: openarray[byte]) =
+proc put*(db: SqStoreRef, key, value: openarray[byte]): KvResult[void] =
   checkErr sqlite3_reset(db.insertStmt)
   checkErr sqlite3_clear_bindings(db.insertStmt)
 
@@ -53,9 +51,11 @@ proc put*(db: SqliteStoreRef, key, value: openarray[byte]) =
   checkErr bindBlob(db.insertStmt, 2, value)
 
   if (let v = sqlite3_step(db.insertStmt); v != SQLITE_DONE):
-    raiseError(v)
+    err(sqlite3_errstr(v))
+  else:
+    ok()
 
-proc contains*(db: SqliteStoreRef, key: openarray[byte]): bool =
+proc contains*(db: SqStoreRef, key: openarray[byte]): KvResult[bool] =
   checkErr sqlite3_reset(db.selectStmt)
   checkErr sqlite3_clear_bindings(db.selectStmt)
 
@@ -63,45 +63,52 @@ proc contains*(db: SqliteStoreRef, key: openarray[byte]): bool =
 
   let v = sqlite3_step(db.selectStmt)
   case v
-  of SQLITE_ROW: result = true
+  of SQLITE_ROW: ok(true)
-  of SQLITE_DONE: result = false
+  of SQLITE_DONE: ok(false)
-  else: raiseError(v)
+  else: err(sqlite3_errstr(v))
 
-proc del*(db: SqliteStoreRef, key: openarray[byte]) =
+proc del*(db: SqStoreRef, key: openarray[byte]): KvResult[void] =
   checkErr sqlite3_reset(db.deleteStmt)
   checkErr sqlite3_clear_bindings(db.deleteStmt)
 
   checkErr bindBlob(db.deleteStmt, 1, key)
 
   if (let v = sqlite3_step(db.deleteStmt); v != SQLITE_DONE):
-    raiseError(v)
+    err(sqlite3_errstr(v))
+  else:
+    ok()
 
-proc close*(db: SqliteStoreRef) =
+proc close*(db: SqStoreRef) =
   discard sqlite3_finalize(db.insertStmt)
   discard sqlite3_finalize(db.selectStmt)
   discard sqlite3_finalize(db.deleteStmt)
+
   discard sqlite3_close(db.env)
 
-  db[] = SqliteStoreRef()[]
+  db[] = SqStoreRef()[]
 
 proc init*(
-    T: type SqliteStoreRef,
+    T: type SqStoreRef,
     basePath: string,
+    name: string,
     readOnly = false,
-    inMemory = false): T =
+    inMemory = false): KvResult[T] =
   var
     env: ptr sqlite3
 
   let
     name =
       if inMemory: ":memory:"
-      else: basepath / "nimbus.sqlite3"
+      else: basepath / name & ".sqlite3"
     flags =
       if readOnly: SQLITE_OPEN_READONLY
       else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE
 
   if not inMemory:
-    createDir(basePath)
+    try:
+      createDir(basePath)
+    except OSError, IOError:
+      return err("`sqlite: cannot create database directory")
 
   checkErr sqlite3_open_v2(name, addr env, flags.cint, nil)
 
@@ -118,11 +125,11 @@ proc init*(
     if (let x = sqlite3_step(s); x != SQLITE_DONE):
       discard sqlite3_finalize(s)
       discard sqlite3_close(env)
-      raiseError(x)
+      return err(sqlite3_errstr(x))
 
     if (let x = sqlite3_finalize(s); x != SQLITE_OK):
       discard sqlite3_close(env)
-      raiseError(x)
+      return err(sqlite3_errstr(x))
 
   # TODO: check current version and implement schema versioning
   checkExec "PRAGMA user_version = 1;"
@@ -143,9 +150,9 @@ proc init*(
       discard sqlite3_finalize(selectStmt)
       discard sqlite3_finalize(insertStmt)
 
-  T(
+  ok(SqStoreRef(
     env: env,
     selectStmt: selectStmt,
     insertStmt: insertStmt,
     deleteStmt: deleteStmt
-  )
+  ))

tests/test_beacon_chain_db.nim

@@ -16,7 +16,7 @@ import  options, unittest, sequtils,
 suiteReport "Beacon chain DB" & preset():
   timedTest "empty database" & preset():
     var
-      db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
+      db = init(BeaconChainDB, kvStore MemStoreRef.init())
 
     check:
       when const_preset=="minimal":
@@ -27,7 +27,7 @@ suiteReport "Beacon chain DB" & preset():
 
   timedTest "sanity check blocks" & preset():
     var
-      db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
+      db = init(BeaconChainDB, kvStore MemStoreRef.init())
 
     let
       signedBlock = SignedBeaconBlock()
@@ -45,7 +45,7 @@ suiteReport "Beacon chain DB" & preset():
 
   timedTest "sanity check states" & preset():
     var
-      db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
+      db = init(BeaconChainDB, kvStore MemStoreRef.init())
 
     let
       state = BeaconState()
@@ -59,7 +59,7 @@ suiteReport "Beacon chain DB" & preset():
 
   timedTest "find ancestors" & preset():
     var
-      db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
+      db = init(BeaconChainDB, kvStore MemStoreRef.init())
 
     let
       a0 = SignedBeaconBlock(message: BeaconBlock(slot: GENESIS_SLOT + 0))
@@ -95,7 +95,7 @@ suiteReport "Beacon chain DB" & preset():
     # serialization where an all-zero default-initialized bls signature could
     # not be deserialized because the deserialization was too strict.
     var
-      db = init(BeaconChainDB, kvStore MemoryStoreRef.init())
+      db = init(BeaconChainDB, kvStore MemStoreRef.init())
 
     let
       state = initialize_beacon_state_from_eth1(

tests/test_kvstore.nim

@@ -4,42 +4,44 @@ import
   unittest,
   ../beacon_chain/kvstore
 
-proc testKVStore*(db: KVStoreRef) =
+const
-  let
+  key = [0'u8, 1, 2, 3]
-    key = [0'u8, 1, 2, 3]
+  value = [3'u8, 2, 1, 0]
-    value = [3'u8, 2, 1, 0]
+  value2 = [5'u8, 2, 1, 0]
-    value2 = [5'u8, 2, 1, 0]
 
+proc testKvStore*(db: KvStoreRef) =
   check:
     db != nil
 
-    not db.get(key, proc(data: openArray[byte]) = discard)
+    not db.get(key, proc(data: openArray[byte]) = discard)[]
-    not db.contains(key)
+    not db.contains(key)[]
 
-  db.del(key) # does nothing
+  db.del(key)[] # does nothing
 
-  db.put(key, value)
+  db.put(key, value)[]
+
+  var v: seq[byte]
+  proc grab(data: openArray[byte]) =
+    v = @data
 
   check:
-    db.contains(key)
+    db.contains(key)[]
-    db.get(key, proc(data: openArray[byte]) =
+    db.get(key, grab)[]
-      check data == value
+    v == value
-    )
 
-  db.put(key, value2) # overwrite old value
+  db.put(key, value2)[] # overwrite old value
   check:
-    db.contains(key)
+    db.contains(key)[]
-    db.get(key, proc(data: openArray[byte]) =
+    db.get(key, grab)[]
-      check data == value2
+    v == value2
-    )
 
-  db.del(key)
+  db.del(key)[]
   check:
-    not db.get(key, proc(data: openArray[byte]) = discard)
+    not db.get(key, proc(data: openArray[byte]) = discard)[]
-    not db.contains(key)
+    not db.contains(key)[]
 
-  db.del(key) # does nothing
+  db.del(key)[] # does nothing
 
 suite "MemoryStoreRef":
-  test "KVStore interface":
+  test "KvStore interface":
-    testKVStore(kvStore MemoryStoreRef.init())
+    testKvStore(kvStore MemStoreRef.init())

tests/test_kvstore_sqlite3.nim

@@ -6,9 +6,9 @@ import
   ../beacon_chain/[kvstore, kvstore_sqlite3],
   ./test_kvstore
 
-suite "Sqlite":
+suite "SqStoreRef":
-  test "KVStore interface":
+  test "KvStore interface":
-    let db = SqliteStoreRef.init("", inMemory = true)
+    let db = SqStoreRef.init("", "test", inMemory = true)[]
     defer: db.close()
 
-    testKVStore(kvStore db)
+    testKvStore(kvStore db)

tests/testutil.nim

@@ -93,7 +93,7 @@ template timedTest*(name, body) =
   testTimes.add (f, name)
 
 proc makeTestDB*(tailState: BeaconState, tailBlock: SignedBeaconBlock): BeaconChainDB =
-  result = init(BeaconChainDB, kvStore MemoryStoreRef.init())
+  result = init(BeaconChainDB, kvStore MemStoreRef.init())
   BlockPool.preInit(result, tailState, tailBlock)
 
 proc makeTestDB*(validators: int): BeaconChainDB =

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit db037cbece06633021d4b8e2374704d9ffdb53f7
+Subproject commit 1989a551459caf9f0d20260e359f40249e31ef01

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit ff755bbf75d0d3f387b9f352b241d98eb3372323
+Subproject commit 8065e36c5af31f2f3f3b0d9ea242ae4eef193a30