Add a basic higher-level API

The API is still very basic and suffering from multiple Nim issues
that will be resolved in the near future:

* destructors will handle the resource cleanup better
* converter concepts will enable a wider range of possible types
  to be used as keys and values.

The API currently uses an ErrorResult type to communicate errors,
but RocksDB doesn't seem to have many recoverable failure modes
and I anticipate that the API will be migrated to use exceptions
once we get a bit more experience with RocksDB.

The C interface file was moved to a separate directory to make
nimble happy (`nimble check`).

examples/.gitignore

@@ -0,0 +1,6 @@
+# ignore all executable files
+*
+!*.*
+!*/
+*.exe
+

nim.cfg

@@ -0,0 +1,2 @@
+-p:"src"
+

rocksdb.nimble

@@ -1,12 +1,13 @@
 packageName   = "rocksdb"
-version       = "0.1.0"
+version       = "0.2.0"
 author        = "Status Research & Development GmbH"
 description   = "A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage"
 license       = "Apache License 2.0 or GPLv2"
 srcDir        = "src"
 
 ### Dependencies
-requires "nim >= 0.17.2"
+requires "nim >= 0.18.1",
+         "ranges"
 
 proc test(name: string, lang: string = "c") =
   if not dirExists "build":
@@ -19,4 +20,8 @@ proc test(name: string, lang: string = "c") =
   setCommand lang, "tests/" & name & ".nim"
 
 task test_c, "Run tests for the C wrapper":
-  test "test_rocksdb_c"
+  test "test_rocksdb_c"
+
+task test, "Run tests for the Nim API":
+  test "test_rocksdb"
+

src/rocksdb.nim

@@ -7,5 +7,224 @@
 #
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-import ./rocksdb_c
-export rocksdb_c
+import cpuinfo, options, ranges
+
+const useCApi = true
+
+when useCApi:
+  import rocksdb/librocksdb
+  export librocksdb
+
+  template managedResource(name) =
+    template freeResource(r: ptr `rocksdb name t`) =
+      `rocksdb name destroy`(r)
+
+  managedResource(WriteOptions)
+  managedResource(ReadOptions)
+
+  type
+    RocksPtr[T] = object
+      res: ptr T
+
+  import typetraits
+
+  when false:
+    # XXX: generic types cannot have destructors at the moment:
+    # https://github.com/nim-lang/Nim/issues/5366
+    proc `=destroy`*[T](rocksPtr: var RocksPtr[T]) =
+      freeResource rocksPtr.res
+
+  proc toRocksPtr[T](res: ptr T): RocksPtr[T] =
+    result.res = res
+
+  template initResource(resourceName): auto =
+    var p = toRocksPtr(`rocksdb resourceName create`())
+    # XXX: work-around the destructor issue above:
+    defer: freeResource p.res
+    p.res
+else:
+  {.error: "The C++ API of RocksDB is not supported yet".}
+
+  # The intention of this template is that it will hide the
+  # difference between the C and C++ APIs for objects such
+  # as Read/WriteOptions, which are allocated either on the
+  # stack or the heap.
+  template initResource(resourceName) =
+    var res = resourceName()
+    res
+
+type
+  # TODO: Replace this with a converter concept that will
+  # handle openarray[char] and openarray[byte] in the same way.
+  KeyValueType = openarray[byte]
+
+  RocksDBInstance* = object
+    db: ptr rocksdb_t
+    backupEngine: ptr rocksdb_backup_engine_t
+    options: ptr rocksdb_options_t
+
+  RocksDBResult*[T] = object
+    case ok*: bool
+    of true:
+      when T isnot void: value*: T
+    else:
+      error*: string
+
+template `$`*(s: RocksDBResult): string =
+  if s.ok:
+    $s.value
+  else:
+    "(error) " & s.error
+
+template returnOk() =
+  result.ok = true
+  return
+
+template returnVal(v: auto) =
+  result.ok = true
+  result.value = v
+  return
+
+template bailOnErrors {.dirty.} =
+  if not errors.isNil:
+    result.ok = false
+    result.error = $(errors[0])
+    return
+
+proc init*(rocks: var RocksDBInstance,
+           dbPath, dbBackupPath: string,
+           cpus = countProcessors(),
+           createIfMissing = true): RocksDBResult[void] =
+  rocks.options = rocksdb_options_create()
+
+  # Optimize RocksDB. This is the easiest way to get RocksDB to perform well:
+  rocksdb_options_increase_parallelism(rocks.options, cpus.int32)
+  rocksdb_options_optimize_level_style_compaction(rocks.options, 0)
+  rocksdb_options_set_create_if_missing(rocks.options, uint8(createIfMissing))
+
+  var errors: cstringArray
+  rocks.db = rocksdb_open(rocks.options, dbPath, errors)
+  bailOnErrors()
+  rocks.backupEngine = rocksdb_backup_engine_open(rocks.options,
+                                                  dbBackupPath, errors)
+  bailOnErrors()
+  returnOk()
+
+template initRocksDB*(args: varargs[untyped]): Option[RocksDBInstance] =
+  var db: RocksDBInstance
+  if not init(db, args):
+    none(RocksDBInstance)
+  else:
+    some(db)
+
+when false:
+  # TODO: These should be in the standard lib somewhere.
+  proc to*(chars: openarray[char], S: typedesc[string]): string =
+    result = newString(chars.len)
+    copyMem(addr result[0], unsafeAddr chars[0], chars.len * sizeof(char))
+
+  proc to*(chars: openarray[char], S: typedesc[seq[byte]]): seq[byte] =
+    result = newSeq[byte](chars.len)
+    copyMem(addr result[0], unsafeAddr chars[0], chars.len * sizeof(char))
+
+  template toOpenArray*[T](p: ptr T, sz: int): openarray[T] =
+    # XXX: The `TT` type is a work-around the fact that the `T`
+    # generic param is not resolved properly within the body of
+    # the template: https://github.com/nim-lang/Nim/issues/7995
+    type TT = type(p[])
+    let arr = cast[ptr UncheckedArray[TT]](p)
+    toOpenArray(arr[], 0, sz)
+else:
+  proc copyFrom(v: var seq[byte], data: cstring, sz: int) =
+    v = newSeq[byte](sz)
+    if sz > 0:
+      copyMem(addr v[0], unsafeAddr data[0], sz)
+
+  proc copyFrom(v: var string, data: cstring, sz: int) =
+    v = newString(sz)
+    if sz > 0:
+      copyMem(addr v[0], unsafeAddr data[0], sz)
+
+template getImpl {.dirty.} =
+  assert key.len > 0
+
+  var
+    options = initResource ReadOptions
+    errors: cstringArray
+    len: csize
+    data = rocksdb_get(db.db, options,
+                       cast[cstring](unsafeAddr key[0]), key.len,
+                       addr len, errors)
+  bailOnErrors()
+  result.ok = true
+  result.value.copyFrom(data, len)
+  # returnVal toOpenArray(cast[ptr char](data), len).to(type(result.value))
+
+proc get*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[string] =
+  getImpl
+
+proc getBytes*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[seq[byte]] =
+  getImpl
+
+proc put*(db: RocksDBInstance, key, val: KeyValueType): RocksDBResult[void] =
+  assert key.len > 0
+
+  var
+    options = initResource WriteOptions
+    errors: cstringArray
+
+  rocksdb_put(db.db, options,
+              cast[cstring](unsafeAddr key[0]), key.len,
+              cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil), val.len,
+              errors)
+
+  bailOnErrors()
+  returnOk()
+
+proc del*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[void] =
+  when false:
+    # XXX: For yet unknown reasons, the code below fails with SIGSEGV.
+    # Investigate if this the correct usage of `rocksdb_delete`.
+    var options = initResource WriteOptions
+    var errors: cstringArray
+    echo key.len
+    rocksdb_delete(db.db, options,
+                   cast[cstring](unsafeAddr key[0]), key.len,
+                   errors)
+    bailOnErrors()
+    returnOk()
+  else:
+    put(db, key, @[])
+
+proc contains*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[bool] =
+  assert key.len > 0
+
+  let res = db.get(key)
+  if res.ok:
+    returnVal res.value.len > 0
+  else:
+    result.ok = false
+    result.error = res.error
+
+proc backup*(db: RocksDBInstance): RocksDBResult[void] =
+  var errors: cstringArray
+  rocksdb_backup_engine_create_new_backup(db.backupEngine, db.db, errors)
+  bailOnErrors()
+  returnOk()
+
+# XXX: destructors are just too buggy at the moment:
+# https://github.com/nim-lang/Nim/issues/8112
+# proc `=destroy`*(db: var RocksDBInstance) =
+proc close*(db: var RocksDBInstance) =
+  if db.backupEngine != nil:
+    rocksdb_backup_engine_close(db.backupEngine)
+    db.backupEngine = nil
+
+  if db.db != nil:
+    rocksdb_close(db.db)
+    db.db = nil
+
+  if db.options != nil:
+    rocksdb_options_destroy(db.options)
+    db.options = nil
+

src/rocksdb/librocksdb.nim

@@ -23,16 +23,21 @@
 
 ## This file exposes the low-level C API of RocksDB
 
+import strutils
+from ospaths import DirSep
+
 {.deadCodeElim: on.}
 when defined(windows):
-  const librocksdb = "librocksdb.dll"
+  const librocksdb = "librocksdb(|_lite).dll"
 elif defined(macosx):
-  const librocksdb = "librocksdb.dylib"
+  const librocksdb = "librocksdb(|_lite).dylib"
 else:
-  const librocksdb = "librocksdb.so"
+  const librocksdb = "librocksdb(|_lite).so"
 ##  Exported types
 
-const rocksdb_header = "rocksdb/c.h"
+const
+  package_base_dir = currentSourcePath.rsplit(DirSep, 3)[0]
+  rocksdb_header = package_base_dir & DirSep & "headers" & DirSep & "c.h"
 
 type
   rocksdb_t* {.importc: "rocksdb_t", header: rocksdb_header.} = object
@@ -210,7 +215,7 @@ proc rocksdb_get_cf*(db: ptr rocksdb_t; options: ptr rocksdb_readoptions_t;
 ##  if values_list[i] == NULL and errs[i] == NULL,
 ##  then we got status.IsNotFound(), which we will not return.
 ##  all errors except status status.ok() and status.IsNotFound() are returned.
-## 
+##
 ##  errs, values_list and values_list_sizes must be num_keys in length,
 ##  allocated by the caller.
 ##  errs is a list of strings as opposed to the conventional one error,
@@ -1565,4 +1570,4 @@ proc rocksdb_get_pinned_cf*(db: ptr rocksdb_t; options: ptr rocksdb_readoptions_
 proc rocksdb_pinnableslice_destroy*(v: ptr rocksdb_pinnableslice_t) {.cdecl,
     importc: "rocksdb_pinnableslice_destroy", dynlib: librocksdb.}
 proc rocksdb_pinnableslice_value*(t: ptr rocksdb_pinnableslice_t; vlen: ptr csize): cstring {.
-    cdecl, importc: "rocksdb_pinnableslice_value", dynlib: librocksdb.}
+    cdecl, importc: "rocksdb_pinnableslice_value", dynlib: librocksdb.}

tests/.gitignore

@@ -0,0 +1,6 @@
+# ignore all executable files
+*
+!*.*
+!*/
+*.exe
+

tests/test_rocksdb.nim

@@ -0,0 +1,48 @@
+import rocksdb, os
+
+type
+  MyDB = object
+    rocksdb: RocksDBInstance
+
+proc initMyDb(path: string): MyDB =
+  let
+    dataDir = path / "data"
+    backupsDir = path / "backups"
+
+  createDir(dataDir)
+  createDir(backupsDir)
+
+  var s = result.rocksdb.init(dataDir, backupsDir)
+  doAssert s.ok
+
+proc main =
+  var db = initMyDb("/tmp/mydb")
+  defer: close(db.rocksdb)
+
+  let key = @[byte(1), 2, 3, 4, 5]
+  let otherKey = @[byte(1), 2, 3, 4, 5, 6]
+  let val = @[byte(1), 2, 3, 4, 5]
+
+  var s = db.rocksdb.put(key, val)
+  doAssert s.ok
+
+  var r1 = db.rocksdb.getBytes(key)
+  doAssert r1.ok and r1.value == val
+
+  var r2 = db.rocksdb.getBytes(otherKey)
+  doAssert r2.ok and r2.value.len == 0
+
+  var e1 = db.rocksdb.contains(key)
+  doAssert e1.ok and e1.value == true
+
+  var e2 = db.rocksdb.contains(otherKey)
+  doAssert e2.ok and e2.value == false
+
+  s = db.rocksdb.del(key)
+  doAssert s.ok
+
+  e1 = db.rocksdb.contains(key)
+  doAssert e1.ok and e1.value == false
+
+main()
+