Richer Sqlite API

* Adds a strongly typed API for creating custom SQL queries and
  executing them

* Uses destructors to simplify the error handling in the init logic

eth/db/kvstore_sqlite3.nim

@@ -10,14 +10,42 @@ import
 export kvstore
 
 type
-  Sqlite3Ptr* = ptr sqlite3
+  RawStmtPtr = ptr sqlite3_stmt
+
+  AutoDisposed[T: ptr|ref] = object
+    val: T
+
+  # TODO: These should become AutoDisposed
+  #       This is currently considered risky due to the destructor
+  #       problem found in FastStreams (triggered when objects in
+  #       the GC heap have destructors)
+  Sqlite* = ptr sqlite3
+  SqliteStmt*[Params; Result] = distinct RawStmtPtr
+  NoParams* = tuple # this is the empty tuple
+  ResultHandler*[T] = proc(val: T) {.gcsafe, raises: [Defect].}
 
   KeySpaceStatements = object
-    getStmt, putStmt, delStmt, containsStmt: ptr sqlite3_stmt
+    getStmt, putStmt, delStmt, containsStmt: RawStmtPtr
 
   SqStoreRef* = ref object of RootObj
-    env: Sqlite3Ptr
+    env: Sqlite
     keyspaces: seq[KeySpaceStatements]
+    extraStmts: seq[RawStmtPtr]
+
+template dispose(db: Sqlite) =
+  discard sqlite3_close(db)
+
+template dispose(db: RawStmtPtr) =
+  discard sqlite3_finalize(db)
+
+proc release[T](x: var AutoDisposed[T]): T =
+  result = x.val
+  x.val = nil
+
+proc `=destroy`*[T](x: var AutoDisposed[T]) =
+  mixin dispose
+  if x.val != nil:
+    dispose(x.release)
 
 template checkErr(op, cleanup: untyped) =
   if (let v = (op); v != SQLITE_OK):
@@ -27,15 +55,129 @@ template checkErr(op, cleanup: untyped) =
 template checkErr(op) =
   checkErr(op): discard
 
-proc bindBlob(s: ptr sqlite3_stmt, n: int, blob: openarray[byte]): cint =
+proc prepareStmt*(db: SqStoreRef,
-  sqlite3_bind_blob(s, n.cint, unsafeAddr blob[0], blob.len.cint, nil)
+                  stmt: string,
+                  Params: type,
+                  Res: type): KvResult[SqliteStmt[Params, Res]] =
+  var s: RawStmtPtr
+  checkErr sqlite3_prepare_v2(db.env, stmt, stmt.len.cint, addr s, nil)
+  ok SqliteStmt[Params, Res](s)
+
+proc bindParam(s: RawStmtPtr, n: int, val: auto): cint =
+  when val is openarray[byte]|seq[byte]:
+    if val.len > 0:
+      sqlite3_bind_blob(s, n.cint, unsafeAddr val[0], val.len.cint, nil)
+    else:
+      sqlite3_bind_blob(s, n.cint, nil, 0.cint, nil)
+  elif val is int32:
+    sqlite3_bind_int(s, n.cint, val)
+  elif val is int64:
+    sqlite3_bind_int64(s, n.cint, val)
+  else:
+    {.fatal: "Please add support for the '" & $(T) & "' type".}
+
+template bindParams(s: RawStmtPtr, params: auto) =
+  when params is tuple:
+    var i = 1
+    for param in fields(params):
+      checkErr bindParam(s, i, param)
+      inc i
+  else:
+    checkErr bindParam(s, 1, params)
+
+proc exec*[P](s: SqliteStmt[P, void], params: P): KvResult[void] =
+  let s = RawStmtPtr s
+  bindParams(s, params)
+
+  let res =
+    if (let v = sqlite3_step(s); v != SQLITE_DONE):
+      err($sqlite3_errstr(v))
+    else:
+      ok()
+
+  # release implict transaction
+  discard sqlite3_reset(s) # same return information as step
+  discard sqlite3_clear_bindings(s) # no errors possible
+
+  res
+
+template readResult(s: RawStmtPtr, column: cint, T: type): auto =
+  when T is int32:
+    sqlite3_column_int(s, column)
+  elif T is int64:
+    sqlite3_column_int64(s, column)
+  elif T is int:
+    {.fatal: "Please use specify either int32 or int64 precisely".}
+  elif T is openarray[byte]:
+    let
+      p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, column))
+      l = sqlite3_column_bytes(s, column)
+    toOpenArray(p, 0, l-1)
+  elif T is seq[byte]:
+    var res: seq[byte]
+    let len = sqlite3_column_bytes(s, column)
+    if len > 0:
+      res.setLen(len)
+      copyMem(addr res[0], sqlite3_column_blob(s, column), len)
+    res
+  else:
+    {.fatal: "Please add support for the '" & $(T) & "' type".}
+
+template readResult(s: RawStmtPtr, T: type): auto =
+  when T is tuple:
+    var res: T
+    var i = cint 0
+    for field in fields(res):
+      field = readResult(s, i, typeof(field))
+      inc i
+    res
+  else:
+    readResult(s, 0.cint, T)
+
+proc exec*[Params, Res](s: SqliteStmt[Params, Res],
+                        params: Params,
+                        onData: ResultHandler[Res]): KvResult[bool] =
+  let s = RawStmtPtr s
+  bindParams(s, params)
+
+  try:
+    var gotResults = false
+    while true:
+      let v = sqlite3_step(s)
+      case v
+      of SQLITE_ROW:
+        onData(readResult(s, Res))
+        gotResults = true
+      of SQLITE_DONE:
+        break
+      else:
+        return err($sqlite3_errstr(v))
+    return ok gotResults
+  finally:
+    # release implicit transaction
+    discard sqlite3_reset(s) # same return information as step
+    discard sqlite3_clear_bindings(s) # no errors possible
+
+template exec*(s: SqliteStmt[NoParams, void]): KvResult[void] =
+  exec(s, ())
+
+template exec*[Res](s: SqliteStmt[NoParams, Res],
+                    onData: ResultHandler[Res]): KvResult[bool] =
+  exec(s, (), onData)
+
+proc exec*(db: SqStoreRef, stmt: string): KvResult[void] =
+  let stmt = ? db.prepareStmt(stmt, NoParams, void)
+  result = exec(stmt)
+  let finalizeStatus = sqlite3_finalize(RawStmtPtr stmt)
+  if finalizeStatus != SQLITE_OK and result.isOk:
+    return err($sqlite3_errstr(finalizeStatus))
 
 proc getImpl(db: SqStoreRef,
              keyspace: int,
              key: openarray[byte],
              onData: DataProc): KvResult[bool] =
   let getStmt = db.keyspaces[keyspace].getStmt
-  checkErr bindBlob(getStmt, 1, key)
+  checkErr bindParam(getStmt, 1, key)
 
   let
     v = sqlite3_step(getStmt)
@@ -66,8 +208,8 @@ template get*(db: SqStoreRef, keyspace: int, key: openarray[byte], onData: DataP
 proc putImpl(db: SqStoreRef, keyspace: int, key, value: openarray[byte]): KvResult[void] =
   let putStmt = db.keyspaces[keyspace].putStmt
 
-  checkErr bindBlob(putStmt, 1, key)
+  checkErr bindParam(putStmt, 1, key)
-  checkErr bindBlob(putStmt, 2, value)
+  checkErr bindParam(putStmt, 2, value)
 
   let res =
     if (let v = sqlite3_step(putStmt); v != SQLITE_DONE):
@@ -89,7 +231,7 @@ template put*(db: SqStoreRef, keyspace: int, key, value: openarray[byte]): KvRes
 
 proc containsImpl(db: SqStoreRef, keyspace: int, key: openarray[byte]): KvResult[bool] =
   let containsStmt = db.keyspaces[keyspace].containsStmt
-  checkErr bindBlob(containsStmt, 1, key)
+  checkErr bindParam(containsStmt, 1, key)
 
   let
     v = sqlite3_step(containsStmt)
@@ -112,7 +254,7 @@ template contains*(db: SqStoreRef, keyspace: int, key: openarray[byte]): KvResul
 
 proc delImpl(db: SqStoreRef, keyspace: int, key: openarray[byte]): KvResult[void] =
   let delStmt = db.keyspaces[keyspace].delStmt
-  checkErr bindBlob(delStmt, 1, key)
+  checkErr bindParam(delStmt, 1, key)
 
   let res =
     if (let v = sqlite3_step(delStmt); v != SQLITE_DONE):
@@ -139,10 +281,16 @@ proc close*(db: SqStoreRef) =
     discard sqlite3_finalize(keyspace.delStmt)
     discard sqlite3_finalize(keyspace.containsStmt)
 
+  for stmt in db.extraStmts:
+    discard sqlite3_finalize(stmt)
+
   discard sqlite3_close(db.env)
 
   db[] = SqStoreRef()[]
 
+proc isClosed*(db: SqStoreRef): bool =
+  db.env != nil
+
 proc init*(
     T: type SqStoreRef,
     basePath: string,
@@ -150,8 +298,7 @@ proc init*(
     readOnly = false,
     inMemory = false,
     keyspaces: openarray[string] = ["kvstore"]): KvResult[T] =
-  var
+  var env: AutoDisposed[ptr sqlite3]
-    env: ptr sqlite3
 
   let
     name =
@@ -167,23 +314,20 @@ proc init*(
     except OSError, IOError:
       return err("`sqlite: cannot create database directory")
 
-  checkErr sqlite3_open_v2(name, addr env, flags.cint, nil)
+  checkErr sqlite3_open_v2(name, addr env.val, flags.cint, nil)
 
   template prepare(q: string, cleanup: untyped): ptr sqlite3_stmt =
     var s: ptr sqlite3_stmt
-    checkErr sqlite3_prepare_v2(env, q, q.len.cint, addr s, nil):
+    checkErr sqlite3_prepare_v2(env.val, q, q.len.cint, addr s, nil):
       cleanup
-      discard sqlite3_close(env)
     s
 
   template checkExec(s: ptr sqlite3_stmt) =
     if (let x = sqlite3_step(s); x != SQLITE_DONE):
       discard sqlite3_finalize(s)
-      discard sqlite3_close(env)
       return err($sqlite3_errstr(x))
 
     if (let x = sqlite3_finalize(s); x != SQLITE_OK):
-      discard sqlite3_close(env)
       return err($sqlite3_errstr(x))
 
   template checkExec(q: string) =
@@ -193,18 +337,15 @@ proc init*(
   template checkWalPragmaResult(journalModePragma: ptr sqlite3_stmt) =
     if (let x = sqlite3_step(journalModePragma); x != SQLITE_ROW):
       discard sqlite3_finalize(journalModePragma)
-      discard sqlite3_close(env)
       return err($sqlite3_errstr(x))
 
     if (let x = sqlite3_column_type(journalModePragma, 0); x != SQLITE3_TEXT):
       discard sqlite3_finalize(journalModePragma)
-      discard sqlite3_close(env)
       return err($sqlite3_errstr(x))
 
     if (let x = sqlite3_column_text(journalModePragma, 0);
         x != "memory" and x != "wal"):
       discard sqlite3_finalize(journalModePragma)
-      discard sqlite3_close(env)
       return err("Invalid pragma result: " & $x)
 
   # TODO: check current version and implement schema versioning
@@ -243,7 +384,7 @@ proc init*(
       containsStmt: containsStmt)
 
   ok(SqStoreRef(
-    env: env,
+    env: env.release,
     keyspaces: keyspaceStatements
   ))
 

tests/db/test_kvstore_sqlite3.nim

@@ -2,13 +2,96 @@
 
 import
   os,
-  unittest,
+  testutils/unittests,
   ../../eth/db/[kvstore, kvstore_sqlite3],
   ./test_kvstore
 
-suite "SqStoreRef":
+procSuite "SqStoreRef":
   test "KvStore interface":
     let db = SqStoreRef.init("", "test", inMemory = true)[]
     defer: db.close()
 
     testKvStore(kvStore db)
+
+  test "Prepare and execute statements":
+    let db = SqStoreRef.init("", "test", inMemory = true)[]
+    defer: db.close()
+
+    let createTableRes = db.exec """
+      CREATE TABLE IF NOT EXISTS records(
+         key INTEGER PRIMARY KEY,
+         value BLOB
+      );
+    """
+    check createTableRes.isOk
+
+    let insertStmt = db.prepareStmt(
+      "INSERT INTO records(value) VALUES (?);",
+      openarray[byte], void).get
+
+    let insert1Res = insertStmt.exec [byte 1, 2, 3, 4]
+    let insert2Res = insertStmt.exec @[]
+    let insert3Res = insertStmt.exec @[byte 5]
+
+    check:
+      insert1Res.isOk
+      insert2Res.isOk
+      insert3Res.isOk
+
+    let countStmt = db.prepareStmt(
+      "SELECT COUNT(*) FROM records;",
+      NoParams, int64).get
+
+    var totalRecords = 0
+    echo "About to call total records"
+    let countRes = countStmt.exec do (res: int64):
+      totalRecords = int res
+
+    check:
+      countRes.isOk and countRes.get == true
+      totalRecords == 3
+
+    let selectRangeStmt = db.prepareStmt(
+      "SELECT value FROM records WHERE key >= ? and key < ?;",
+      (int64, int64), openarray[byte]).get
+
+    block:
+      var allBytes = newSeq[byte]()
+      let selectRangeRes = selectRangeStmt.exec((0'i64, 5'i64)) do (bytes: openarray[byte]) {.gcsafe.}:
+        allBytes.add byte(bytes.len)
+        allBytes.add bytes
+
+      if selectRangeRes.isErr:
+        echo selectRangeRes.error
+
+      check:
+        selectRangeRes.isOk and selectRangeRes.get == true
+        allBytes == [byte 4, 1, 2, 3, 4,
+                     0,
+                     1, 5]
+    block:
+      let selectRangeRes = selectRangeStmt.exec((10'i64, 20'i64)) do (bytes: openarray[byte]):
+        echo "Got unexpected bytes: ", bytes
+
+      check:
+        selectRangeRes.isOk and selectRangeRes.get == false
+
+    let selectAllStmt = db.prepareStmt(
+      "SELECT * FROM records;",
+      NoParams, (int64, seq[byte])).get
+
+    var indices = newSeq[int64]()
+    var values = newSeq[seq[byte]]()
+
+    discard selectAllStmt.exec do (res: (int64, seq[byte])):
+      indices.add res[0]
+      values.add res[1]
+
+    check:
+      indices == [int64 1, 2, 3]
+      values == [
+        @[byte 1, 2, 3, 4],
+        @[],
+        @[byte 5]
+      ]
+