diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000..8e17a54 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,6 @@ +# ignore all executable files +* +!*.* +!*/ +*.exe + diff --git a/nim.cfg b/nim.cfg new file mode 100644 index 0000000..4e7b49d --- /dev/null +++ b/nim.cfg @@ -0,0 +1,2 @@ +-p:"src" + diff --git a/rocksdb.nimble b/rocksdb.nimble index 07ba406..ac528c9 100644 --- a/rocksdb.nimble +++ b/rocksdb.nimble @@ -1,12 +1,13 @@ packageName = "rocksdb" -version = "0.1.0" +version = "0.2.0" author = "Status Research & Development GmbH" description = "A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage" license = "Apache License 2.0 or GPLv2" srcDir = "src" ### Dependencies -requires "nim >= 0.17.2" +requires "nim >= 0.18.1", + "ranges" proc test(name: string, lang: string = "c") = if not dirExists "build": @@ -19,4 +20,8 @@ proc test(name: string, lang: string = "c") = setCommand lang, "tests/" & name & ".nim" task test_c, "Run tests for the C wrapper": - test "test_rocksdb_c" \ No newline at end of file + test "test_rocksdb_c" + +task test, "Run tests for the Nim API": + test "test_rocksdb" + diff --git a/src/rocksdb.nim b/src/rocksdb.nim index d5d79c7..8447f6c 100644 --- a/src/rocksdb.nim +++ b/src/rocksdb.nim @@ -7,5 +7,224 @@ # # at your option. This file may not be copied, modified, or distributed except according to those terms. -import ./rocksdb_c -export rocksdb_c \ No newline at end of file +import cpuinfo, options, ranges + +const useCApi = true + +when useCApi: + import rocksdb/librocksdb + export librocksdb + + template managedResource(name) = + template freeResource(r: ptr `rocksdb name t`) = + `rocksdb name destroy`(r) + + managedResource(WriteOptions) + managedResource(ReadOptions) + + type + RocksPtr[T] = object + res: ptr T + + import typetraits + + when false: + # XXX: generic types cannot have destructors at the moment: + # https://github.com/nim-lang/Nim/issues/5366 + proc `=destroy`*[T](rocksPtr: var RocksPtr[T]) = + freeResource rocksPtr.res + + proc toRocksPtr[T](res: ptr T): RocksPtr[T] = + result.res = res + + template initResource(resourceName): auto = + var p = toRocksPtr(`rocksdb resourceName create`()) + # XXX: work-around the destructor issue above: + defer: freeResource p.res + p.res +else: + {.error: "The C++ API of RocksDB is not supported yet".} + + # The intention of this template is that it will hide the + # difference between the C and C++ APIs for objects such + # as Read/WriteOptions, which are allocated either on the + # stack or the heap. + template initResource(resourceName) = + var res = resourceName() + res + +type + # TODO: Replace this with a converter concept that will + # handle openarray[char] and openarray[byte] in the same way. + KeyValueType = openarray[byte] + + RocksDBInstance* = object + db: ptr rocksdb_t + backupEngine: ptr rocksdb_backup_engine_t + options: ptr rocksdb_options_t + + RocksDBResult*[T] = object + case ok*: bool + of true: + when T isnot void: value*: T + else: + error*: string + +template `$`*(s: RocksDBResult): string = + if s.ok: + $s.value + else: + "(error) " & s.error + +template returnOk() = + result.ok = true + return + +template returnVal(v: auto) = + result.ok = true + result.value = v + return + +template bailOnErrors {.dirty.} = + if not errors.isNil: + result.ok = false + result.error = $(errors[0]) + return + +proc init*(rocks: var RocksDBInstance, + dbPath, dbBackupPath: string, + cpus = countProcessors(), + createIfMissing = true): RocksDBResult[void] = + rocks.options = rocksdb_options_create() + + # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: + rocksdb_options_increase_parallelism(rocks.options, cpus.int32) + rocksdb_options_optimize_level_style_compaction(rocks.options, 0) + rocksdb_options_set_create_if_missing(rocks.options, uint8(createIfMissing)) + + var errors: cstringArray + rocks.db = rocksdb_open(rocks.options, dbPath, errors) + bailOnErrors() + rocks.backupEngine = rocksdb_backup_engine_open(rocks.options, + dbBackupPath, errors) + bailOnErrors() + returnOk() + +template initRocksDB*(args: varargs[untyped]): Option[RocksDBInstance] = + var db: RocksDBInstance + if not init(db, args): + none(RocksDBInstance) + else: + some(db) + +when false: + # TODO: These should be in the standard lib somewhere. + proc to*(chars: openarray[char], S: typedesc[string]): string = + result = newString(chars.len) + copyMem(addr result[0], unsafeAddr chars[0], chars.len * sizeof(char)) + + proc to*(chars: openarray[char], S: typedesc[seq[byte]]): seq[byte] = + result = newSeq[byte](chars.len) + copyMem(addr result[0], unsafeAddr chars[0], chars.len * sizeof(char)) + + template toOpenArray*[T](p: ptr T, sz: int): openarray[T] = + # XXX: The `TT` type is a work-around the fact that the `T` + # generic param is not resolved properly within the body of + # the template: https://github.com/nim-lang/Nim/issues/7995 + type TT = type(p[]) + let arr = cast[ptr UncheckedArray[TT]](p) + toOpenArray(arr[], 0, sz) +else: + proc copyFrom(v: var seq[byte], data: cstring, sz: int) = + v = newSeq[byte](sz) + if sz > 0: + copyMem(addr v[0], unsafeAddr data[0], sz) + + proc copyFrom(v: var string, data: cstring, sz: int) = + v = newString(sz) + if sz > 0: + copyMem(addr v[0], unsafeAddr data[0], sz) + +template getImpl {.dirty.} = + assert key.len > 0 + + var + options = initResource ReadOptions + errors: cstringArray + len: csize + data = rocksdb_get(db.db, options, + cast[cstring](unsafeAddr key[0]), key.len, + addr len, errors) + bailOnErrors() + result.ok = true + result.value.copyFrom(data, len) + # returnVal toOpenArray(cast[ptr char](data), len).to(type(result.value)) + +proc get*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[string] = + getImpl + +proc getBytes*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[seq[byte]] = + getImpl + +proc put*(db: RocksDBInstance, key, val: KeyValueType): RocksDBResult[void] = + assert key.len > 0 + + var + options = initResource WriteOptions + errors: cstringArray + + rocksdb_put(db.db, options, + cast[cstring](unsafeAddr key[0]), key.len, + cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil), val.len, + errors) + + bailOnErrors() + returnOk() + +proc del*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[void] = + when false: + # XXX: For yet unknown reasons, the code below fails with SIGSEGV. + # Investigate if this the correct usage of `rocksdb_delete`. + var options = initResource WriteOptions + var errors: cstringArray + echo key.len + rocksdb_delete(db.db, options, + cast[cstring](unsafeAddr key[0]), key.len, + errors) + bailOnErrors() + returnOk() + else: + put(db, key, @[]) + +proc contains*(db: RocksDBInstance, key: KeyValueType): RocksDBResult[bool] = + assert key.len > 0 + + let res = db.get(key) + if res.ok: + returnVal res.value.len > 0 + else: + result.ok = false + result.error = res.error + +proc backup*(db: RocksDBInstance): RocksDBResult[void] = + var errors: cstringArray + rocksdb_backup_engine_create_new_backup(db.backupEngine, db.db, errors) + bailOnErrors() + returnOk() + +# XXX: destructors are just too buggy at the moment: +# https://github.com/nim-lang/Nim/issues/8112 +# proc `=destroy`*(db: var RocksDBInstance) = +proc close*(db: var RocksDBInstance) = + if db.backupEngine != nil: + rocksdb_backup_engine_close(db.backupEngine) + db.backupEngine = nil + + if db.db != nil: + rocksdb_close(db.db) + db.db = nil + + if db.options != nil: + rocksdb_options_destroy(db.options) + db.options = nil + diff --git a/src/rocksdb_c.nim b/src/rocksdb/librocksdb.nim similarity index 99% rename from src/rocksdb_c.nim rename to src/rocksdb/librocksdb.nim index 49dab96..7c37151 100644 --- a/src/rocksdb_c.nim +++ b/src/rocksdb/librocksdb.nim @@ -23,16 +23,21 @@ ## This file exposes the low-level C API of RocksDB +import strutils +from ospaths import DirSep + {.deadCodeElim: on.} when defined(windows): - const librocksdb = "librocksdb.dll" + const librocksdb = "librocksdb(|_lite).dll" elif defined(macosx): - const librocksdb = "librocksdb.dylib" + const librocksdb = "librocksdb(|_lite).dylib" else: - const librocksdb = "librocksdb.so" + const librocksdb = "librocksdb(|_lite).so" ## Exported types -const rocksdb_header = "rocksdb/c.h" +const + package_base_dir = currentSourcePath.rsplit(DirSep, 3)[0] + rocksdb_header = package_base_dir & DirSep & "headers" & DirSep & "c.h" type rocksdb_t* {.importc: "rocksdb_t", header: rocksdb_header.} = object @@ -210,7 +215,7 @@ proc rocksdb_get_cf*(db: ptr rocksdb_t; options: ptr rocksdb_readoptions_t; ## if values_list[i] == NULL and errs[i] == NULL, ## then we got status.IsNotFound(), which we will not return. ## all errors except status status.ok() and status.IsNotFound() are returned. -## +## ## errs, values_list and values_list_sizes must be num_keys in length, ## allocated by the caller. ## errs is a list of strings as opposed to the conventional one error, @@ -1565,4 +1570,4 @@ proc rocksdb_get_pinned_cf*(db: ptr rocksdb_t; options: ptr rocksdb_readoptions_ proc rocksdb_pinnableslice_destroy*(v: ptr rocksdb_pinnableslice_t) {.cdecl, importc: "rocksdb_pinnableslice_destroy", dynlib: librocksdb.} proc rocksdb_pinnableslice_value*(t: ptr rocksdb_pinnableslice_t; vlen: ptr csize): cstring {. - cdecl, importc: "rocksdb_pinnableslice_value", dynlib: librocksdb.} \ No newline at end of file + cdecl, importc: "rocksdb_pinnableslice_value", dynlib: librocksdb.} diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..8e17a54 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,6 @@ +# ignore all executable files +* +!*.* +!*/ +*.exe + diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim new file mode 100644 index 0000000..2ea5074 --- /dev/null +++ b/tests/test_rocksdb.nim @@ -0,0 +1,48 @@ +import rocksdb, os + +type + MyDB = object + rocksdb: RocksDBInstance + +proc initMyDb(path: string): MyDB = + let + dataDir = path / "data" + backupsDir = path / "backups" + + createDir(dataDir) + createDir(backupsDir) + + var s = result.rocksdb.init(dataDir, backupsDir) + doAssert s.ok + +proc main = + var db = initMyDb("/tmp/mydb") + defer: close(db.rocksdb) + + let key = @[byte(1), 2, 3, 4, 5] + let otherKey = @[byte(1), 2, 3, 4, 5, 6] + let val = @[byte(1), 2, 3, 4, 5] + + var s = db.rocksdb.put(key, val) + doAssert s.ok + + var r1 = db.rocksdb.getBytes(key) + doAssert r1.ok and r1.value == val + + var r2 = db.rocksdb.getBytes(otherKey) + doAssert r2.ok and r2.value.len == 0 + + var e1 = db.rocksdb.contains(key) + doAssert e1.ok and e1.value == true + + var e2 = db.rocksdb.contains(otherKey) + doAssert e2.ok and e2.value == false + + s = db.rocksdb.del(key) + doAssert s.ok + + e1 = db.rocksdb.contains(key) + doAssert e1.ok and e1.value == false + +main() +