From 5575192d4c62c83df42eeaed8119895f8bf5c769 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 25 Jun 2018 02:13:10 +0300 Subject: [PATCH] Add a basic higher-level API The API is still very basic and suffering from multiple Nim issues that will be resolved in the near future: * destructors will handle the resource cleanup better * converter concepts will enable a wider range of possible types to be used as keys and values. The API currently uses an ErrorResult type to communicate errors, but RocksDB doesn't seem to have many recoverable failure modes and I anticipate that the API will be migrated to use exceptions once we get a bit more experience with RocksDB. The C interface file was moved to a separate directory to make nimble happy (`nimble check`). --- examples/.gitignore | 6 + nim.cfg | 2 + rocksdb.nimble | 11 +- src/rocksdb.nim | 223 +++++++++++++++++- src/{rocksdb_c.nim => rocksdb/librocksdb.nim} | 17 +- tests/.gitignore | 6 + tests/test_rocksdb.nim | 48 ++++ 7 files changed, 302 insertions(+), 11 deletions(-) create mode 100644 examples/.gitignore create mode 100644 nim.cfg rename src/{rocksdb_c.nim => rocksdb/librocksdb.nim} (99%) create mode 100644 tests/.gitignore create mode 100644 tests/test_rocksdb.nim diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000..8e17a54 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,6 @@ +# ignore all executable files +* +!*.* +!*/ +*.exe + diff --git a/nim.cfg b/nim.cfg new file mode 100644 index 0000000..4e7b49d --- /dev/null +++ b/nim.cfg @@ -0,0 +1,2 @@ +-p:"src" + diff --git a/rocksdb.nimble b/rocksdb.nimble index 07ba406..ac528c9 100644 --- a/rocksdb.nimble +++ b/rocksdb.nimble @@ -1,12 +1,13 @@ packageName = "rocksdb" -version = "0.1.0" +version = "0.2.0" author = "Status Research & Development GmbH" description = "A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage" license = "Apache License 2.0 or GPLv2" srcDir = "src" ### Dependencies -requires "nim >= 0.17.2" +requires "nim >= 0.18.1", + "ranges" proc test(name: string, lang: string = "c") = if not dirExists "build": @@ -19,4 +20,8 @@ proc test(name: string, lang: string = "c") = setCommand lang, "tests/" & name & ".nim" task test_c, "Run tests for the C wrapper": - test "test_rocksdb_c" \ No newline at end of file + test "test_rocksdb_c" + +task test, "Run tests for the Nim API": + test "test_rocksdb" + diff --git a/src/rocksdb.nim b/src/rocksdb.nim index d5d79c7..8447f6c 100644 --- a/src/rocksdb.nim +++ b/src/rocksdb.nim @@ -7,5 +7,224 @@ # # at your option. This file may not be copied, modified, or distributed except according to those terms. -import ./rocksdb_c -export rocksdb_c \ No newline at end of file +import cpuinfo, options, ranges + +const useCApi = true + +when useCApi: + import rocksdb/librocksdb + export librocksdb + + template managedResource(name) = + template freeResource(r: ptr `rocksdb name t`) = + `rocksdb name destroy`(r) + + managedResource(WriteOptions) + managedResource(ReadOptions) + + type + RocksPtr[T] = object + res: ptr T + + import typetraits + + when false: + # XXX: generic types cannot have destructors at the moment: + # https://github.com/nim-lang/Nim/issues/5366 + proc `=destroy`*[T](rocksPtr: var RocksPtr[T]) = + freeResource rocksPtr.res + + proc toRocksPtr[T](res: ptr T): RocksPtr[T] = + result.res = res + + template initResource(resourceName): auto = + var p = toRocksPtr(`rocksdb resourceName create`()) + # XXX: work-around the destructor issue above: + defer: freeResource p.res + p.res +else: + {.error: "The C++ API of RocksDB is not supported yet".} + + # The intention of this template is that it will hide the + # difference between the C and C++ APIs for objects such + # as Read/WriteOptions, which are allocated either on the + # stack or the heap. + template initResource(resourceName) = + var res = resourceName() + res + +type + # TODO: Replace this with a converter concept that will + # handle openarray[char] and openarray[byte] in the same way. + KeyValueType = openarray[byte] + + RocksDBInstance* = object + db: ptr rocksdb_t + backupEngine: ptr rocksdb_backup_engine_t + options: ptr rocksdb_options_t + + RocksDBResult*[T] = object + case ok*: bool + of true: + when T isnot void: value*: T + else: + error*: string + +template `$`*(s: RocksDBResult): string = + if s.ok: + $s.value + else: + "(error) " & s.error + +template returnOk() = + result.ok = true + return + +template returnVal(v: auto) = + result.ok = true + result.value = v + return + +template bailOnErrors {.dirty.} = + if not errors.isNil: + result.ok = false + result.error = $(errors[0]) + return + +proc init*(rocks: var RocksDBInstance, + dbPath, dbBackupPath: string, + cpus = countProcessors(), + createIfMissing = true): RocksDBResult[void] = + rocks.options = rocksdb_options_create() + + # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: + rocksdb_options_increase_parallelism(rocks.options, cpus.int32) + rocksdb_options_optimize_level_style_compaction(rocks.options, 0) + rocksdb_options_set_create_if_missing(rocks.options, uint8(createIfMissing)) + + var errors: cstringArray + rocks.db = rocksdb_open(rocks.options, dbPath, errors) + bailOnErrors() + rocks.backupEngine = rocksdb_backup_engine_open(rocks.options, + dbBackupPath, errors) + bailOnErrors() + returnOk() + +template initRocksDB*(args: varargs[untyped]): Option[RocksDBInstance] = + var db: RocksDBInstance + if not init(db, args): + none(RocksDBInstance) + else: + some(db) + +when false: + # TODO: These should be in the standard lib somewhere. + proc to*(chars: openarray[char], S: typedesc[string]): string = + result = newString(chars.len) + copyMem(addr result[0], unsafeAddr chars[0], chars.len * sizeof(char)) + + proc to*(chars: openarray[char], S: typedesc[seq[byte]]): seq[byte] = + result = newSeq[byte](chars.len) + copyMem(addr result[0], unsafeAddr chars[0], chars.len * sizeof(char)) + + template toOpenArray*[T](p: ptr T, sz: int): openarray[T] = + # XXX: The `TT` type is a work-around the fact that the `T` + # generic param is not resolved properly within the body of + # the template: https://github.com/nim-lang/Nim/issues/7995 + type TT = type(p[]) + let arr = cast[ptr UncheckedArray[TT]](p) + toOpenArray(arr[], 0, sz) +else: + proc copyFrom(v: var seq[byte], data: cstring, sz: int) = + v = newSeq[byte](sz) + if sz > 0: + copyMem(addr v[0], unsafeAddr data[0], sz) + + proc copyFrom(v: var string, data: cstring, sz: int) = + v = newString(sz) + if sz > 0: + copyMem(addr v[0], unsafeAddr data[0], sz) + +template getImpl {.dirty.} = + assert key.len > 0 + + var + options = initResource ReadOptions + errors: cstringArray + len: csize + data = rocksdb_get(db.db, options, + cast[cstring](unsafeAddr key[0]), key.len, + addr len, errors) + bailOnErrors() + result.ok = true + result.value.copyFrom(data, len) + # returnVal toOpenArray(cast[ptr char](data), len).to(type(result.value)) + +proc get*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[string] = + getImpl + +proc getBytes*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[seq[byte]] = + getImpl + +proc put*(db: RocksDBInstance, key, val: KeyValueType): RocksDBResult[void] = + assert key.len > 0 + + var + options = initResource WriteOptions + errors: cstringArray + + rocksdb_put(db.db, options, + cast[cstring](unsafeAddr key[0]), key.len, + cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil), val.len, + errors) + + bailOnErrors() + returnOk() + +proc del*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[void] = + when false: + # XXX: For yet unknown reasons, the code below fails with SIGSEGV. + # Investigate if this the correct usage of `rocksdb_delete`. + var options = initResource WriteOptions + var errors: cstringArray + echo key.len + rocksdb_delete(db.db, options, + cast[cstring](unsafeAddr key[0]), key.len, + errors) + bailOnErrors() + returnOk() + else: + put(db, key, @[]) + +proc contains*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[bool] = + assert key.len > 0 + + let res = db.get(key) + if res.ok: + returnVal res.value.len > 0 + else: + result.ok = false + result.error = res.error + +proc backup*(db: RocksDBInstance): RocksDBResult[void] = + var errors: cstringArray + rocksdb_backup_engine_create_new_backup(db.backupEngine, db.db, errors) + bailOnErrors() + returnOk() + +# XXX: destructors are just too buggy at the moment: +# https://github.com/nim-lang/Nim/issues/8112 +# proc `=destroy`*(db: var RocksDBInstance) = +proc close*(db: var RocksDBInstance) = + if db.backupEngine != nil: + rocksdb_backup_engine_close(db.backupEngine) + db.backupEngine = nil + + if db.db != nil: + rocksdb_close(db.db) + db.db = nil + + if db.options != nil: + rocksdb_options_destroy(db.options) + db.options = nil + diff --git a/src/rocksdb_c.nim b/src/rocksdb/librocksdb.nim similarity index 99% rename from src/rocksdb_c.nim rename to src/rocksdb/librocksdb.nim index 49dab96..7c37151 100644 --- a/src/rocksdb_c.nim +++ b/src/rocksdb/librocksdb.nim @@ -23,16 +23,21 @@ ## This file exposes the low-level C API of RocksDB +import strutils +from ospaths import DirSep + {.deadCodeElim: on.} when defined(windows): - const librocksdb = "librocksdb.dll" + const librocksdb = "librocksdb(|_lite).dll" elif defined(macosx): - const librocksdb = "librocksdb.dylib" + const librocksdb = "librocksdb(|_lite).dylib" else: - const librocksdb = "librocksdb.so" + const librocksdb = "librocksdb(|_lite).so" ## Exported types -const rocksdb_header = "rocksdb/c.h" +const + package_base_dir = currentSourcePath.rsplit(DirSep, 3)[0] + rocksdb_header = package_base_dir & DirSep & "headers" & DirSep & "c.h" type rocksdb_t* {.importc: "rocksdb_t", header: rocksdb_header.} = object @@ -210,7 +215,7 @@ proc rocksdb_get_cf*(db: ptr rocksdb_t; options: ptr rocksdb_readoptions_t; ## if values_list[i] == NULL and errs[i] == NULL, ## then we got status.IsNotFound(), which we will not return. ## all errors except status status.ok() and status.IsNotFound() are returned. -## +## ## errs, values_list and values_list_sizes must be num_keys in length, ## allocated by the caller. ## errs is a list of strings as opposed to the conventional one error, @@ -1565,4 +1570,4 @@ proc rocksdb_get_pinned_cf*(db: ptr rocksdb_t; options: ptr rocksdb_readoptions_ proc rocksdb_pinnableslice_destroy*(v: ptr rocksdb_pinnableslice_t) {.cdecl, importc: "rocksdb_pinnableslice_destroy", dynlib: librocksdb.} proc rocksdb_pinnableslice_value*(t: ptr rocksdb_pinnableslice_t; vlen: ptr csize): cstring {. - cdecl, importc: "rocksdb_pinnableslice_value", dynlib: librocksdb.} \ No newline at end of file + cdecl, importc: "rocksdb_pinnableslice_value", dynlib: librocksdb.} diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..8e17a54 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,6 @@ +# ignore all executable files +* +!*.* +!*/ +*.exe + diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim new file mode 100644 index 0000000..2ea5074 --- /dev/null +++ b/tests/test_rocksdb.nim @@ -0,0 +1,48 @@ +import rocksdb, os + +type + MyDB = object + rocksdb: RocksDBInstance + +proc initMyDb(path: string): MyDB = + let + dataDir = path / "data" + backupsDir = path / "backups" + + createDir(dataDir) + createDir(backupsDir) + + var s = result.rocksdb.init(dataDir, backupsDir) + doAssert s.ok + +proc main = + var db = initMyDb("/tmp/mydb") + defer: close(db.rocksdb) + + let key = @[byte(1), 2, 3, 4, 5] + let otherKey = @[byte(1), 2, 3, 4, 5, 6] + let val = @[byte(1), 2, 3, 4, 5] + + var s = db.rocksdb.put(key, val) + doAssert s.ok + + var r1 = db.rocksdb.getBytes(key) + doAssert r1.ok and r1.value == val + + var r2 = db.rocksdb.getBytes(otherKey) + doAssert r2.ok and r2.value.len == 0 + + var e1 = db.rocksdb.contains(key) + doAssert e1.ok and e1.value == true + + var e2 = db.rocksdb.contains(otherKey) + doAssert e2.ok and e2.value == false + + s = db.rocksdb.del(key) + doAssert s.ok + + e1 = db.rocksdb.contains(key) + doAssert e1.ok and e1.value == false + +main() +