status-im
/
nim-rocksdb
mirror of https://github.com/status-im/nim-rocksdb.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rocksdb rewrite integration (#38) 

* Column families support (#34)

* Update library to support column families. If not specified, uses the 'default' column family.

* Added tests for column family changes.

* Update library version and readme.

* Updated the librocksdb c library to the latest stable version.

* Started rewrite of library.

* Commit library rewrite progress.

* Completed initial rewrite and refactored tests.

* Completed implementation of backup engine.

* Added tests for new types.

* Completed tests for existing features.

* Remove features not supported by older versions of RocksDB to fix CI (temporary fix).

* Remove flush before backup support from BackupEngine to fix CI.

* Transactions support (#36)

* Update library to support column families. If not specified, uses the 'default' column family.

* Added tests for column family changes.

* Update library version and readme.

* Updated the librocksdb c library to the latest stable version.

* Started rewrite of library.

* Commit library rewrite progress.

* Completed initial rewrite and refactored tests.

* Completed implementation of backup engine.

* Added tests for new types.

* Completed tests for existing features.

* Remove features not supported by older versions of RocksDB to fix CI (temporary fix).

* Remove flush before backup support from BackupEngine to fix CI.

* Implemented RocksDB iterator.

* Implemented pairs iterator.

* Completed implementation of WriteBatch with tests.

* Fix example code.

* Completed implementation of TransactionDb.

* Support setting default column family.

* Remove unneeded usage of var for ref type parameters.

* Completed transactiondb tests.

* Improve and refactor rocksdb test.

* Added support for ingesting sst files using the SstFileWriter. (#37)

* Create ColFamilyReadOnly and ColFamilyReadWrite types for using a specific column family.

* Use inline pragma for small procs and add lock to RocksDbRef type close to prevent double free.

* Added documentation for the public API.

* Initial implementation of sst filewriter.

* Added tests for sstfilewriter.

* Documentation minor improvements.
This commit is contained in:
web3-developer 2024-03-05 11:12:37 +08:00 committed by GitHub
parent 5e2b026f84
commit 5f6282e8d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
48 changed files with 9739 additions and 2996 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@ -10,7 +10,7 @@ A Nim wrapper for [Facebook's RocksDB](https://github.com/facebook/rocksdb), a p
## Current status
Nim-RocksDB currently provides a wrapper for the low-level functions of RocksDB
Nim-RocksDB provides a wrapper for the low-level functions in the librocksdb c library.
## Requirements
@ -30,14 +30,6 @@ nim c -d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++ --threads:on
(we need the C++ linker profile because it's a C++ library)
## Future directions
In the future, Nim-RocksDB might provide a high-level API that:
- is more in line with Nim conventions (types in CamelCase),
- automatically checks for errors,
- leverage Nim features like destructors for automatic resource cleanup.
### Contribution
Any contribution intentionally submitted for inclusion in the work by you shall be dual licensed as above, without any

22
examples/simple_example.nim
View File

@ -1,4 +1,4 @@
import ../rocksdb, cpuinfo
import ../rocksdb/lib/librocksdb, cpuinfo
const
dbPath: cstring = "/tmp/rocksdb_simple_example"
@ -6,8 +6,8 @@ const
proc main() =
var
db: rocksdb_t
be: rocksdb_backup_engine_t
db: ptr rocksdb_t
be: ptr rocksdb_backup_engine_t
options = rocksdb_options_create()
# Optimize RocksDB. This is the easiest way to
# get RocksDB to perform well
@ -21,24 +21,26 @@ proc main() =
# open DB
var err: cstring # memory leak: example code does not free error string!
db = rocksdb_open(options, dbPath, err.addr)
db = rocksdb_open(options, dbPath, cast[cstringArray](err.addr))
doAssert err.isNil, $err
# open Backup Engine that we will use for backing up our database
be = rocksdb_backup_engine_open(options, dbBackupPath, err.addr)
be = rocksdb_backup_engine_open(options, dbBackupPath, cast[cstringArray](err.addr))
doAssert err.isNil, $err
# Put key-value
var writeOptions = rocksdb_writeoptions_create()
let key = "key"
let put_value = "value"
rocksdb_put(db, writeOptions, key.cstring, key.len.csize_t, put_value.cstring, put_value.len.csize_t, err.addr)
rocksdb_put(db, writeOptions, key.cstring, key.len.csize_t, put_value.cstring,
put_value.len.csize_t, cast[cstringArray](err.addr))
doAssert err.isNil, $err
# Get value
var readOptions = rocksdb_readoptions_create()
var len: csize_t
let raw_value = rocksdb_get(db, readOptions, key, key.len.csize_t, addr len, err.addr) # Important: rocksdb_get is not null-terminated
let raw_value = rocksdb_get(db, readOptions, key.cstring, key.len.csize_t, addr len,
cast[cstringArray](err.addr)) # Important: rocksdb_get is not null-terminated
doAssert err.isNil, $err
# Copy it to a regular Nim string (copyMem workaround because raw value is NOT null-terminated)
@ -48,7 +50,7 @@ proc main() =
doAssert get_value == put_value
# create new backup in a directory specified by DBBackupPath
rocksdb_backup_engine_create_new_backup(be, db, err.addr)
rocksdb_backup_engine_create_new_backup(be, db, cast[cstringArray](err.addr))
doAssert err.isNil, $err
rocksdb_close(db)
@ -56,11 +58,11 @@ proc main() =
# If something is wrong, you might want to restore data from last backup
var restoreOptions = rocksdb_restore_options_create()
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbPath, dbPath,
restoreOptions, err.addr)
restoreOptions, cast[cstringArray](err.addr))
doAssert err.isNil, $err
rocksdb_restore_options_destroy(restore_options)
db = rocksdb_open(options, dbPath, err.addr)
db = rocksdb_open(options, dbPath, cast[cstringArray](err.addr))
doAssert err.isNil, $err
# cleanup

239
rocksdb.nim
View File

@ -1,5 +1,5 @@
# Nim-RocksDB
# Copyright 2018 Status Research & Development GmbH
# Copyright 2018-2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
@ -7,230 +7,15 @@
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [Defect].}
import
./rocksdb/[backup, columnfamily, rocksdb, rocksiterator],
./rocksdb/[sstfilewriter, transactiondb, writebatch]
import cpuinfo, options, stew/[byteutils, results]
from system/ansi_c import c_free
export results
const useCApi = true
when useCApi:
import rocksdb/librocksdb
export librocksdb
else:
{.error: "The C++ API of RocksDB is not supported yet".}
# The intention of this template is that it will hide the
# difference between the C and C++ APIs for objects such
# as Read/WriteOptions, which are allocated either on the
# stack or the heap.
template initResource(resourceName) =
var res = resourceName()
res
type
RocksDBInstance* = object
db*: rocksdb_t
backupEngine: rocksdb_backup_engine_t
options*: rocksdb_options_t
readOptions*: rocksdb_readoptions_t
writeOptions: rocksdb_writeoptions_t
dbPath: string # needed for clear()
DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [Defect].}
RocksDBResult*[T] = Result[T, string]
template bailOnErrors {.dirty.} =
if not errors.isNil:
result.err($errors)
rocksdb_free(errors)
return
proc init*(rocks: var RocksDBInstance,
dbPath, dbBackupPath: string,
readOnly = false,
cpus = countProcessors(),
createIfMissing = true,
maxOpenFiles = -1): RocksDBResult[void] =
rocks.options = rocksdb_options_create()
rocks.readOptions = rocksdb_readoptions_create()
rocks.writeOptions = rocksdb_writeoptions_create()
rocks.dbPath = dbPath
# Optimize RocksDB. This is the easiest way to get RocksDB to perform well:
rocksdb_options_increase_parallelism(rocks.options, cpus.int32)
# This requires snappy - disabled because rocksdb is not always compiled with
# snappy support (for example Fedora 28, certain Ubuntu versions)
# rocksdb_options_optimize_level_style_compaction(options, 0);
rocksdb_options_set_create_if_missing(rocks.options, uint8(createIfMissing))
# default set to keep all files open (-1), allow setting it to a specific
# value, e.g. in case the application limit would be reached.
rocksdb_options_set_max_open_files(rocks.options, maxOpenFiles.cint)
var errors: cstring
if readOnly:
rocks.db = rocksdb_open_for_read_only(rocks.options, dbPath, 0'u8, errors.addr)
else:
rocks.db = rocksdb_open(rocks.options, dbPath, errors.addr)
bailOnErrors()
rocks.backupEngine = rocksdb_backup_engine_open(rocks.options,
dbBackupPath, errors.addr)
bailOnErrors()
ok()
template initRocksDB*(args: varargs[untyped]): Option[RocksDBInstance] =
var db: RocksDBInstance
if not init(db, args):
none(RocksDBInstance)
else:
some(db)
template getImpl(T: type) {.dirty.} =
if key.len <= 0:
return err("rocksdb: key cannot be empty on get")
var
errors: cstring
len: csize_t
data = rocksdb_get(db.db, db.readOptions,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
addr len, addr errors)
bailOnErrors()
if not data.isNil:
result = ok(toOpenArray(data, 0, int(len) - 1).to(T))
rocksdb_free(data)
else:
result = err("")
proc get*(db: RocksDBInstance, key: openArray[byte], onData: DataProc): RocksDBResult[bool] =
if key.len <= 0:
return err("rocksdb: key cannot be empty on get")
var
errors: cstring
len: csize_t
data = rocksdb_get(db.db, db.readOptions,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
addr len, addr errors)
bailOnErrors()
if not data.isNil:
# TODO onData may raise a Defect - in theory we could catch it and free the
# memory but this has a small overhead - setjmp (C) or RTTI (C++) -
# reconsider this once the exception dust settles
onData(toOpenArrayByte(data, 0, int(len) - 1))
rocksdb_free(data)
ok(true)
else:
ok(false)
proc get*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[string] {.deprecated: "DataProc".} =
## Get value for `key`. If no value exists, set `result.ok` to `false`,
## and result.error to `""`.
var res: RocksDBResult[string]
proc onData(data: openArray[byte]) =
res.ok(string.fromBytes(data))
if ? db.get(key, onData):
res
else:
ok("")
proc getBytes*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[seq[byte]] {.deprecated: "DataProc".} =
## Get value for `key`. If no value exists, set `result.ok` to `false`,
## and result.error to `""`.
var res: RocksDBResult[seq[byte]]
proc onData(data: openArray[byte]) =
res.ok(@data)
if ? db.get(key, onData):
res
else:
err("")
proc put*(db: RocksDBInstance, key, val: openArray[byte]): RocksDBResult[void] =
if key.len <= 0:
return err("rocksdb: key cannot be empty on put")
var
errors: cstring
rocksdb_put(db.db, db.writeOptions,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
csize_t(val.len),
errors.addr)
bailOnErrors()
ok()
proc contains*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[bool] =
if key.len <= 0:
return err("rocksdb: key cannot be empty on contains")
var
errors: cstring
len: csize_t
data = rocksdb_get(db.db, db.readOptions,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
addr len, errors.addr)
bailOnErrors()
if not data.isNil:
rocksdb_free(data)
ok(true)
else:
ok(false)
proc del*(db: RocksDBInstance, key: openArray[byte]): RocksDBResult[bool] =
if key.len <= 0:
return err("rocksdb: key cannot be empty on del")
# This seems like a bad idea, but right now I don't want to
# get sidetracked by this. --Adam
if not db.contains(key).get:
return ok(false)
var errors: cstring
rocksdb_delete(db.db, db.writeOptions,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
errors.addr)
bailOnErrors()
ok(true)
proc clear*(db: var RocksDBInstance): RocksDBResult[bool] =
raiseAssert "unimplemented"
proc backup*(db: RocksDBInstance): RocksDBResult[void] =
var errors: cstring
rocksdb_backup_engine_create_new_backup(db.backupEngine, db.db, errors.addr)
bailOnErrors()
ok()
# XXX: destructors are just too buggy at the moment:
# https://github.com/nim-lang/Nim/issues/8112
# proc `=destroy`*(db: var RocksDBInstance) =
proc close*(db: var RocksDBInstance) =
template freeField(name) =
type FieldType = typeof db.`name`
if db.`name`.isNil:
`rocksdb name destroy`(db.`name`)
db.`name` = FieldType(nil)
template setFieldToNil(name) =
type FieldType = typeof db.`name`
db.`name` = FieldType(nil)
freeField(writeOptions)
freeField(readOptions)
freeField(options)
if not db.backupEngine.isNil:
rocksdb_backup_engine_close(db.backupEngine)
setFieldToNil(backupEngine)
if not db.db.isNil:
rocksdb_close(db.db)
setFieldToNil(db)
export
backup,
columnfamily,
rocksdb,
rocksiterator,
sstfilewriter,
transactiondb,
writebatch

11
rocksdb.nimble
View File

@ -1,5 +1,5 @@
packageName = "rocksdb"
version = "0.3.1"
version = "0.4.0"
author = "Status Research & Development GmbH"
description = "A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage"
license = "Apache License 2.0 or GPLv2"
@ -7,9 +7,10 @@ skipDirs = @["examples", "tests"]
mode = ScriptMode.Verbose
### Dependencies
requires "nim >= 1.2.0",
requires "nim >= 1.6",
"stew",
"tempfile"
"tempfile",
"unittest2"
proc test(args, path: string) =
if not dirExists "build":
@ -18,8 +19,8 @@ proc test(args, path: string) =
" --outdir:build -r --hints:off --threads:on --skipParentCfg " & path
task test, "Run tests":
test "", "tests/all.nim"
test "", "tests/test_all.nim"
# Too troublesome to install "librocksdb.a" in CI, but this is how we would
# test it (we need the C++ linker profile because it's a C++ library):
# test "-d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++", "tests/all.nim"
# test "-d:LibrocksbStaticArgs='-l:librocksdb.a' --gcc.linkerexe=g++", "tests/test_all.nim"

102
rocksdb/backup.nim Normal file
View File

@ -0,0 +1,102 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## A `BackupEngineRef` is used to create and manage backups against a RocksDB database.
{.push raises: [].}
import
./lib/librocksdb,
./internal/utils,
./options/backupopts,
./rocksdb,
./rocksresult
export
backupopts,
rocksdb,
rocksresult
type
BackupEnginePtr* = ptr rocksdb_backup_engine_t
BackupEngineRef* = ref object
cPtr: BackupEnginePtr
path: string
backupOpts: BackupEngineOptionsRef
proc openBackupEngine*(
path: string,
backupOpts = defaultBackupEngineOptions()): RocksDBResult[BackupEngineRef] =
## Create a new backup engine. The `path` parameter is the path of the backup
## directory. Note that the same directory should not be used for both backups
## and the database itself.
var errors: cstring
let backupEnginePtr = rocksdb_backup_engine_open(
backupOpts.cPtr,
path.cstring,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
let engine = BackupEngineRef(
cPtr: backupEnginePtr,
path: path,
backupOpts: backupOpts)
ok(engine)
proc isClosed*(backupEngine: BackupEngineRef): bool {.inline.} =
## Returns `true` if the `BackupEngineRef` has been closed.
backupEngine.cPtr.isNil()
proc createNewBackup*(
backupEngine: BackupEngineRef,
db: RocksDbRef): RocksDBResult[void] =
## Create a new backup of the database.
doAssert not backupEngine.isClosed()
doAssert not db.isClosed()
var errors: cstring
rocksdb_backup_engine_create_new_backup(
backupEngine.cPtr,
db.cPtr,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc restoreDbFromLatestBackup*(
backupEngine: BackupEngineRef,
dbDir: string,
walDir = dbDir,
keepLogFiles = false): RocksDBResult[void] =
## Restore the database from the latest backup.
doAssert not backupEngine.isClosed()
let restoreOptions = rocksdb_restore_options_create()
rocksdb_restore_options_set_keep_log_files(restoreOptions, keepLogFiles.cint)
var errors: cstring
rocksdb_backup_engine_restore_db_from_latest_backup(
backupEngine.cPtr,
dbDir.cstring,
walDir.cstring,
restoreOptions,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
rocksdb_restore_options_destroy(restoreOptions)
ok()
proc close*(backupEngine: BackupEngineRef) =
## Close the `BackupEngineRef`.
if not backupEngine.isClosed():
rocksdb_backup_engine_close(backupEngine.cPtr)
backupEngine.cPtr = nil

113
rocksdb/columnfamily.nim Normal file
View File

@ -0,0 +1,113 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## `ColFamilyReadOnly` and `ColFamilyReadWrite` types both hold a reference to a
## `RocksDbReadOnlyRef` or `RocksDbReadWriteRef` respectively. They are convenience
## types which enable writing to a specific column family without having to specify the
## column family in each call.
##
## These column family types do not own the underlying `RocksDbRef` and therefore
## to close the database, simply call `columnFamily.db.close()` which will close
## the underlying `RocksDbRef`. Note that doing so will also impact any other column
## families that hold a reference to the same `RocksDbRef`.
{.push raises: [].}
import
./rocksdb
export rocksdb
type
ColFamilyReadOnly* = object
db: RocksDbReadOnlyRef
name: string
ColFamilyReadWrite* = object
db: RocksDbReadWriteRef
name: string
proc withColFamily*(
db: RocksDbReadOnlyRef,
name: string): RocksDBResult[ColFamilyReadOnly] =
## Creates a new `ColFamilyReadOnly` from the given `RocksDbReadOnlyRef` and
## column family name.
# validate that the column family exists
discard db.keyExists(@[0.byte], name).valueOr:
return err(error)
ok(ColFamilyReadOnly(db: db, name: name))
proc withColFamily*(
db: RocksDbReadWriteRef,
name: string): RocksDBResult[ColFamilyReadWrite] =
## Create a new `ColFamilyReadWrite` from the given `RocksDbReadWriteRef` and
## column family name.
# validate that the column family exists
discard db.keyExists(@[0.byte], name).valueOr:
return err(error)
ok(ColFamilyReadWrite(db: db, name: name))
proc db*(cf: ColFamilyReadOnly | ColFamilyReadWrite): auto {.inline.} =
## Returns the underlying `RocksDbReadOnlyRef` or `RocksDbReadWriteRef`.
cf.db
proc name*(cf: ColFamilyReadOnly | ColFamilyReadWrite): string {.inline.} =
## Returns the name of the column family.
cf.name
proc get*(
cf: ColFamilyReadOnly | ColFamilyReadWrite,
key: openArray[byte],
onData: DataProc): RocksDBResult[bool] {.inline.} =
## Gets the value of the given key from the column family using the `onData`
## callback.
cf.db.get(key, onData, cf.name)
proc get*(
cf: ColFamilyReadOnly | ColFamilyReadWrite,
key: openArray[byte]): RocksDBResult[seq[byte]] {.inline.} =
## Gets the value of the given key from the column family.
cf.db.get(key, cf.name)
proc put*(
cf: ColFamilyReadWrite,
key, val: openArray[byte]): RocksDBResult[void] {.inline.} =
## Puts a value for the given key into the column family.
cf.db.put(key, val, cf.name)
proc keyExists*(
cf: ColFamilyReadOnly | ColFamilyReadWrite,
key: openArray[byte]): RocksDBResult[bool] {.inline.} =
## Checks if the given key exists in the column family.
cf.db.keyExists(key, cf.name)
proc delete*(
cf: ColFamilyReadWrite,
key: openArray[byte]): RocksDBResult[void] {.inline.} =
## Deletes the given key from the column family.
cf.db.delete(key, cf.name)
proc openIterator*(
cf: ColFamilyReadOnly | ColFamilyReadWrite): RocksDBResult[RocksIteratorRef] {.inline.} =
## Opens an `RocksIteratorRef` for the given column family.
cf.db.openIterator(cf.name)
proc openWriteBatch*(cf: ColFamilyReadWrite): WriteBatchRef {.inline.} =
## Opens a `WriteBatchRef` for the given column family.
cf.db.openWriteBatch(cf.name)
proc write*(
cf: ColFamilyReadWrite,
updates: WriteBatchRef): RocksDBResult[void] {.inline.} =
## Writes the updates in the `WriteBatchRef` to the column family.
cf.db.write(updates)

44
rocksdb/columnfamily/cfdescriptor.nim Normal file
View File

@ -0,0 +1,44 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../internal/utils,
./cfopts
export cfopts
type
ColFamilyDescriptor* = object
name: string
options: ColFamilyOptionsRef
proc initColFamilyDescriptor*(
name: string,
options = defaultColFamilyOptions()): ColFamilyDescriptor =
ColFamilyDescriptor(name: name, options: options)
proc name*(descriptor: ColFamilyDescriptor): string {.inline.} =
descriptor.name
proc options*(descriptor: ColFamilyDescriptor): ColFamilyOptionsRef {.inline.} =
descriptor.options
proc isDefault*(descriptor: ColFamilyDescriptor): bool {.inline.} =
descriptor.name == DEFAULT_COLUMN_FAMILY_NAME
proc defaultColFamilyDescriptor*(): ColFamilyDescriptor {.inline.} =
initColFamilyDescriptor(DEFAULT_COLUMN_FAMILY_NAME)
proc isClosed*(descriptor: ColFamilyDescriptor): bool {.inline.} =
descriptor.options.isClosed()
proc close*(descriptor: ColFamilyDescriptor) {.inline.} =
descriptor.options.close()

50
rocksdb/columnfamily/cfhandle.nim Normal file
View File

@ -0,0 +1,50 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../lib/librocksdb
type
ColFamilyHandlePtr* = ptr rocksdb_column_family_handle_t
ColFamilyHandleRef* = ref object
cPtr: ColFamilyHandlePtr
proc newColFamilyHandle*(cPtr: ColFamilyHandlePtr): ColFamilyHandleRef =
ColFamilyHandleRef(cPtr: cPtr)
proc isClosed*(handle: ColFamilyHandleRef): bool {.inline.} =
handle.cPtr.isNil()
proc cPtr*(handle: ColFamilyHandleRef): ColFamilyHandlePtr =
doAssert not handle.isClosed()
handle.cPtr
# TODO: These procs below will not work unless using the latest version of rocksdb
# Currently, when installing librocksdb-dev on linux the RocksDb version used is 6.11.4
# Need to complete this task: https://github.com/status-im/nim-rocksdb/issues/10
# proc getId*(handle: ColFamilyHandleRef): int =
# doAssert not handle.isClosed()
# rocksdb_column_family_handle_get_id(handle.cPtr).int
# proc getName*(handle: ColFamilyHandleRef): string =
# doAssert not handle.isClosed()
# var nameLen: csize_t
# $rocksdb_column_family_handle_get_name(handle.cPtr, nameLen.addr)
# proc isDefault*(handle: ColFamilyHandleRef): bool {.inline.} =
# handle.getName() == DEFAULT_COLUMN_FAMILY_NAME
proc close*(handle: ColFamilyHandleRef) =
if not handle.isClosed():
rocksdb_column_family_handle_destroy(handle.cPtr)
handle.cPtr = nil

52
rocksdb/columnfamily/cfopts.nim Normal file
View File

@ -0,0 +1,52 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../lib/librocksdb
type
ColFamilyOptionsPtr* = ptr rocksdb_options_t
ColFamilyOptionsRef* = ref object
cPtr: ColFamilyOptionsPtr
proc newColFamilyOptions*(): ColFamilyOptionsRef =
ColFamilyOptionsRef(cPtr: rocksdb_options_create())
proc isClosed*(cfOpts: ColFamilyOptionsRef): bool {.inline.} =
cfOpts.cPtr.isNil()
proc cPtr*(cfOpts: ColFamilyOptionsRef): ColFamilyOptionsPtr =
doAssert not cfOpts.isClosed()
cfOpts.cPtr
proc setCreateMissingColumnFamilies*(cfOpts: ColFamilyOptionsRef, flag: bool) =
doAssert not cfOpts.isClosed()
rocksdb_options_set_create_missing_column_families(cfOpts.cPtr, flag.uint8)
proc defaultColFamilyOptions*(): ColFamilyOptionsRef =
let opts = newColFamilyOptions()
# Enable creating column families if they do not exist
opts.setCreateMissingColumnFamilies(true)
return opts
# TODO: These procs below will not work unless using the latest version of rocksdb
# Currently, when installing librocksdb-dev on linux the RocksDb version used is 6.11.4
# Need to complete this task: https://github.com/status-im/nim-rocksdb/issues/10
# proc getCreateMissingColumnFamilies*(cfOpts: ColFamilyOptionsRef): bool =
# doAssert not cfOpts.isClosed()
# rocksdb_options_get_create_missing_column_families(cfOpts.cPtr).bool
proc close*(cfOpts: ColFamilyOptionsRef) =
if not cfOpts.isClosed():
rocksdb_options_destroy(cfOpts.cPtr)
cfOpts.cPtr = nil

1568
rocksdb/headers/c.h
View File

File diff suppressed because it is too large Load Diff

44
rocksdb/internal/cftable.nim Normal file
View File

@ -0,0 +1,44 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
std/tables,
../columnfamily/cfhandle
export
cfhandle
type
ColFamilyTableRef* = ref object
columnFamilies: TableRef[string, ColFamilyHandleRef]
proc newColFamilyTable*(
names: openArray[string],
handles: openArray[ColFamilyHandlePtr]): ColFamilyTableRef =
doAssert names.len() == handles.len()
let cfTable = newTable[string, ColFamilyHandleRef]()
for i, name in names:
cfTable[name] = newColFamilyHandle(handles[i])
ColFamilyTableRef(columnFamilies: cfTable)
proc isClosed*(table: ColFamilyTableRef): bool {.inline.} =
table.columnFamilies.isNil()
proc get*(table: ColFamilyTableRef, name: string): ColFamilyHandleRef =
table.columnFamilies.getOrDefault(name)
proc close*(table: ColFamilyTableRef) =
if not table.isClosed():
for _, v in table.columnFamilies.mpairs():
v.close()
table.columnFamilies = nil

27
rocksdb/internal/utils.nim Normal file
View File

@ -0,0 +1,27 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
std/locks,
../lib/librocksdb
const DEFAULT_COLUMN_FAMILY_NAME* = "default"
proc createLock*(): Lock =
var lock = Lock()
initLock(lock)
lock
template bailOnErrors*(errors: cstring): auto =
if not errors.isNil:
let res = err($(errors))
rocksdb_free(errors)
return res

3042
rocksdb/lib/headers/c.h Normal file
View File

File diff suppressed because it is too large Load Diff

3205
rocksdb/lib/librocksdb.nim Normal file
View File

File diff suppressed because it is too large Load Diff

1111
rocksdb/librocksdb.nim
View File

File diff suppressed because it is too large Load Diff

40
rocksdb/options/backupopts.nim Normal file
View File

@ -0,0 +1,40 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../lib/librocksdb
type
BackupEngineOptionsPtr* = ptr rocksdb_options_t
BackupEngineOptionsRef* = ref object
cPtr: BackupEngineOptionsPtr
proc newBackupEngineOptions*(): BackupEngineOptionsRef =
BackupEngineOptionsRef(cPtr: rocksdb_options_create())
proc isClosed*(engineOpts: BackupEngineOptionsRef): bool {.inline.} =
engineOpts.cPtr.isNil()
proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr =
doAssert not engineOpts.isClosed()
engineOpts.cPtr
# TODO: Add setters and getters for backup options properties.
proc defaultBackupEngineOptions*(): BackupEngineOptionsRef {.inline.} =
newBackupEngineOptions()
# TODO: set prefered defaults
proc close*(engineOpts: BackupEngineOptionsRef) =
if not engineOpts.isClosed():
rocksdb_options_destroy(engineOpts.cPtr)
engineOpts.cPtr = nil

84
rocksdb/options/dbopts.nim Normal file
View File

@ -0,0 +1,84 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
std/cpuinfo,
../lib/librocksdb
type
DbOptionsPtr* = ptr rocksdb_options_t
DbOptionsRef* = ref object
cPtr: DbOptionsPtr
proc newDbOptions*(): DbOptionsRef =
DbOptionsRef(cPtr: rocksdb_options_create())
proc isClosed*(dbOpts: DbOptionsRef): bool {.inline.} =
dbOpts.cPtr.isNil()
proc cPtr*(dbOpts: DbOptionsRef): DbOptionsPtr =
doAssert not dbOpts.isClosed()
dbOpts.cPtr
proc setIncreaseParallelism*(dbOpts: DbOptionsRef, totalThreads: int) =
doAssert totalThreads > 0
doAssert not dbOpts.isClosed()
rocksdb_options_increase_parallelism(dbOpts.cPtr, totalThreads.cint)
proc setCreateIfMissing*(dbOpts: DbOptionsRef, flag: bool) =
doAssert not dbOpts.isClosed()
rocksdb_options_set_create_if_missing(dbOpts.cPtr, flag.uint8)
proc setMaxOpenFiles*(dbOpts: DbOptionsRef, maxOpenFiles: int) =
doAssert maxOpenFiles >= -1
doAssert not dbOpts.isClosed()
rocksdb_options_set_max_open_files(dbOpts.cPtr, maxOpenFiles.cint)
proc setCreateMissingColumnFamilies*(dbOpts: DbOptionsRef, flag: bool) =
doAssert not dbOpts.isClosed()
rocksdb_options_set_create_missing_column_families(dbOpts.cPtr, flag.uint8)
proc defaultDbOptions*(): DbOptionsRef =
let opts = newDbOptions()
# Optimize RocksDB. This is the easiest way to get RocksDB to perform well:
opts.setIncreaseParallelism(countProcessors())
# This requires snappy - disabled because rocksdb is not always compiled with
# snappy support (for example Fedora 28, certain Ubuntu versions)
# rocksdb_options_optimize_level_style_compaction(options, 0);
opts.setCreateIfMissing(true)
# default set to keep all files open (-1), allow setting it to a specific
# value, e.g. in case the application limit would be reached.
opts.setMaxOpenFiles(-1)
# Enable creating column families if they do not exist
opts.setCreateMissingColumnFamilies(true)
return opts
# TODO: These procs below will not work unless using the latest version of rocksdb
# Currently, when installing librocksdb-dev on linux the RocksDb version used is 6.11.4
# Need to complete this task: https://github.com/status-im/nim-rocksdb/issues/10
# proc getCreateIfMissing*(dbOpts: DbOptionsRef): bool =
# doAssert not dbOpts.isClosed()
# rocksdb_options_get_create_if_missing(dbOpts.cPtr).bool
# proc getMaxOpenFiles*(dbOpts: DbOptionsRef): int =
# doAssert not dbOpts.isClosed()
# rocksdb_options_get_max_open_files(dbOpts.cPtr).int
# proc getCreateMissingColumnFamilies*(dbOpts: DbOptionsRef): bool =
# doAssert not dbOpts.isClosed()
# rocksdb_options_get_create_missing_column_families(dbOpts.cPtr).bool
proc close*(dbOpts: DbOptionsRef) =
if not dbOpts.isClosed():
rocksdb_options_destroy(dbOpts.cPtr)
dbOpts.cPtr = nil

40
rocksdb/options/readopts.nim Normal file
View File

@ -0,0 +1,40 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../lib/librocksdb
type
ReadOptionsPtr* = ptr rocksdb_readoptions_t
ReadOptionsRef* = ref object
cPtr: ReadOptionsPtr
proc newReadOptions*(): ReadOptionsRef =
ReadOptionsRef(cPtr: rocksdb_readoptions_create())
proc isClosed*(readOpts: ReadOptionsRef): bool {.inline.} =
readOpts.cPtr.isNil()
proc cPtr*(readOpts: ReadOptionsRef): ReadOptionsPtr =
doAssert not readOpts.isClosed()
readOpts.cPtr
# TODO: Add setters and getters for read options properties.
proc defaultReadOptions*(): ReadOptionsRef {.inline.} =
newReadOptions()
# TODO: set prefered defaults
proc close*(readOpts: ReadOptionsRef) =
if not readOpts.isClosed():
rocksdb_readoptions_destroy(readOpts.cPtr)
readOpts.cPtr = nil

40
rocksdb/options/writeopts.nim Normal file
View File

@ -0,0 +1,40 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../lib/librocksdb
type
WriteOptionsPtr* = ptr rocksdb_writeoptions_t
WriteOptionsRef* = ref object
cPtr: WriteOptionsPtr
proc newWriteOptions*(): WriteOptionsRef =
WriteOptionsRef(cPtr: rocksdb_writeoptions_create())
proc isClosed*(writeOpts: WriteOptionsRef): bool {.inline.} =
writeOpts.cPtr.isNil()
proc cPtr*(writeOpts: WriteOptionsRef): WriteOptionsPtr =
doAssert not writeOpts.isClosed()
writeOpts.cPtr
# TODO: Add setters and getters for write options properties.
proc defaultWriteOptions*(): WriteOptionsRef {.inline.} =
newWriteOptions()
# TODO: set prefered defaults
proc close*(writeOpts: WriteOptionsRef) =
if not writeOpts.isClosed():
rocksdb_writeoptions_destroy(writeOpts.cPtr)
writeOpts.cPtr = nil

371
rocksdb/rocksdb.nim Normal file
View File

@ -0,0 +1,371 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## A `RocksDBRef` represents a reference to a RocksDB instance. It can be opened
## in read-only or read-write mode in which case a `RocksDbReadOnlyRef` or
## `RocksDbReadWriteRef` will be returned respectively. The `RocksDbReadOnlyRef`
## type doesn't support any of the write operations such as `put`, `delete` or
## `write`.
##
## Many of the operations on these types can potentially fail for various reasons,
## in which case a `RocksDbResult` containing an error will be returned.
##
## The types wrap and hold a handle to a c pointer which needs to be freed
## so `close` should be called to prevent a memory leak after use.
##
## Most of the procs below support passing in the name of the column family
## which should be used for the operation. The default column family will be
## used if none is provided.
{.push raises: [].}
import
std/[sequtils, locks],
./lib/librocksdb,
./options/[dbopts, readopts, writeopts],
./columnfamily/[cfopts, cfdescriptor, cfhandle],
./internal/[cftable, utils],
./rocksiterator,
./rocksresult,
./writebatch
export
rocksresult,
dbopts,
readopts,
writeopts,
cfdescriptor,
rocksiterator,
writebatch
type
RocksDbPtr* = ptr rocksdb_t
IngestExternalFilesOptionsPtr = ptr rocksdb_ingestexternalfileoptions_t
RocksDbRef* = ref object of RootObj
lock: Lock
cPtr: RocksDbPtr
path: string
dbOpts: DbOptionsRef
readOpts: ReadOptionsRef
defaultCfName: string
cfTable: ColFamilyTableRef
RocksDbReadOnlyRef* = ref object of RocksDbRef
RocksDbReadWriteRef* = ref object of RocksDbRef
writeOpts: WriteOptionsRef
ingestOptsPtr: IngestExternalFilesOptionsPtr
proc openRocksDb*(
path: string,
dbOpts = defaultDbOptions(),
readOpts = defaultReadOptions(),
writeOpts = defaultWriteOptions(),
columnFamilies = @[defaultColFamilyDescriptor()]): RocksDBResult[RocksDbReadWriteRef] =
## Open a RocksDB instance in read-write mode. If `columnFamilies` is empty
## then it will open the default column family. If `dbOpts`, `readOpts`, or
## `writeOpts` are not supplied then the default options will be used.
## By default, column families will be created if they don't yet exist.
## All existing column families must be specified if the database has
## previously created any column families. This means that the list of column
## families must always at least contain the default column family.
if columnFamilies.len == 0:
return err("rocksdb: no column families")
var
cfNames = columnFamilies.mapIt(it.name().cstring)
cfOpts = columnFamilies.mapIt(it.options.cPtr)
columnFamilyHandles = newSeq[ColFamilyHandlePtr](columnFamilies.len)
errors: cstring
let rocksDbPtr = rocksdb_open_column_families(
dbOpts.cPtr,
path.cstring,
cfNames.len().cint,
cast[cstringArray](cfNames[0].addr),
cfOpts[0].addr,
columnFamilyHandles[0].addr,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
let db = RocksDbReadWriteRef(
lock: createLock(),
cPtr: rocksDbPtr,
path: path,
dbOpts: dbOpts,
readOpts: readOpts,
writeOpts: writeOpts,
ingestOptsPtr: rocksdb_ingestexternalfileoptions_create(),
defaultCfName: DEFAULT_COLUMN_FAMILY_NAME,
cfTable: newColFamilyTable(cfNames.mapIt($it), columnFamilyHandles))
ok(db)
proc openRocksDbReadOnly*(
path: string,
dbOpts = defaultDbOptions(),
readOpts = defaultReadOptions(),
columnFamilies = @[defaultColFamilyDescriptor()],
errorIfWalFileExists = false): RocksDBResult[RocksDbReadOnlyRef] =
## Open a RocksDB instance in read-only mode. If `columnFamilies` is empty
## then it will open the default column family. If `dbOpts` or `readOpts` are
## not supplied then the default options will be used. By default, column
## families will be created if they don't yet exist. If the database already
## contains any column families, then all or a subset of the existing column
## families can be opened for reading.
if columnFamilies.len == 0:
return err("rocksdb: no column families")
var
cfNames = columnFamilies.mapIt(it.name().cstring)
cfOpts = columnFamilies.mapIt(it.options.cPtr)
columnFamilyHandles = newSeq[ColFamilyHandlePtr](columnFamilies.len)
errors: cstring
let rocksDbPtr = rocksdb_open_for_read_only_column_families(
dbOpts.cPtr,
path.cstring,
cfNames.len().cint,
cast[cstringArray](cfNames[0].addr),
cfOpts[0].addr,
columnFamilyHandles[0].addr,
errorIfWalFileExists.uint8,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
let db = RocksDbReadOnlyRef(
lock: createLock(),
cPtr: rocksDbPtr,
path: path,
dbOpts: dbOpts,
readOpts: readOpts,
defaultCfName: DEFAULT_COLUMN_FAMILY_NAME,
cfTable: newColFamilyTable(cfNames.mapIt($it), columnFamilyHandles))
ok(db)
proc isClosed*(db: RocksDbRef): bool {.inline.} =
## Returns `true` if the database has been closed and `false` otherwise.
db.cPtr.isNil()
proc cPtr*(db: RocksDbRef): RocksDbPtr {.inline.} =
## Get the underlying database pointer.
doAssert not db.isClosed()
db.cPtr
proc get*(
db: RocksDbRef,
key: openArray[byte],
onData: DataProc,
columnFamily = db.defaultCfName): RocksDBResult[bool] =
## Get the value for the given key from the specified column family.
## If the value does not exist, `false` will be returned in the result
## and `onData` will not be called. If the value does exist, `true` will be
## returned in the result and `onData` will be called with the value.
## The `onData` callback reduces the number of copies and therefore should be
## preferred if performance is required.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = db.cfTable.get(columnFamily)
if cfHandle.isNil():
return err("rocksdb: unknown column family")
var
len: csize_t
errors: cstring
let data = rocksdb_get_cf(
db.cPtr,
db.readOpts.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
len.addr,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
if data.isNil():
doAssert len == 0
ok(false)
else:
onData(toOpenArrayByte(data, 0, len.int - 1))
rocksdb_free(data)
ok(true)
proc get*(
db: RocksDbRef,
key: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[seq[byte]] =
## Get the value for the given key from the specified column family.
## If the value does not exist, an empty error will be returned in the result.
## If the value does exist, the value will be returned in the result.
var dataRes: RocksDBResult[seq[byte]]
proc onData(data: openArray[byte]) = dataRes.ok(@data)
let res = db.get(key, onData, columnFamily)
if res.isOk():
return dataRes
dataRes.err(res.error())
proc put*(
db: RocksDbReadWriteRef,
key, val: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[void] =
## Put the value for the given key into the specified column family.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = db.cfTable.get(columnFamily)
if cfHandle.isNil():
return err("rocksdb: unknown column family")
var errors: cstring
rocksdb_put_cf(
db.cPtr,
db.writeOpts.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
csize_t(val.len),
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc keyExists*(
db: RocksDbRef,
key: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[bool] =
## Check if the key exists in the specified column family.
## Returns a result containing `true` if the key exists or a result
## containing `false` otherwise.
# TODO: Call rocksdb_key_may_exist_cf to improve performance for the case
# when the key does not exist
db.get(key, proc(data: openArray[byte]) = discard, columnFamily)
proc delete*(
db: RocksDbReadWriteRef,
key: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[void] =
## Delete the value for the given key from the specified column family.
## If the value does not exist, the delete will be a no-op.
## To check if the value exists before or after a delete, use `keyExists`.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = db.cfTable.get(columnFamily)
if cfHandle.isNil:
return err("rocksdb: unknown column family")
var errors: cstring
rocksdb_delete_cf(
db.cPtr,
db.writeOpts.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc openIterator*(
db: RocksDbRef,
columnFamily = db.defaultCfName): RocksDBResult[RocksIteratorRef] =
## Opens an `RocksIteratorRef` for the specified column family.
doAssert not db.isClosed()
let cfHandle = db.cfTable.get(columnFamily)
if cfHandle.isNil():
return err("rocksdb: unknown column family")
let rocksIterPtr = rocksdb_create_iterator_cf(
db.cPtr,
db.readOpts.cPtr,
cfHandle.cPtr)
ok(newRocksIterator(rocksIterPtr))
proc openWriteBatch*(
db: RocksDbReadWriteRef,
columnFamily = db.defaultCfName): WriteBatchRef =
## Opens a `WriteBatchRef` which defaults to using the specified column family.
doAssert not db.isClosed()
newWriteBatch(db.cfTable, columnFamily)
proc write*(
db: RocksDbReadWriteRef,
updates: WriteBatchRef): RocksDBResult[void] =
## Apply the updates in the `WriteBatchRef` to the database.
doAssert not db.isClosed()
var errors: cstring
rocksdb_write(
db.cPtr,
db.writeOpts.cPtr,
updates.cPtr,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc ingestExternalFile*(
db: RocksDbReadWriteRef,
filePath: string,
columnFamily = db.defaultCfName): RocksDbResult[void] =
## Ingest an external sst file into the database. The file will be ingested
## into the specified column family or the default column family if none is
## provided.
doAssert not db.isClosed()
let cfHandle = db.cfTable.get(columnFamily)
if cfHandle.isNil():
return err("rocksdb: unknown column family")
var
sstPath = filePath.cstring
errors: cstring
rocksdb_ingest_external_file_cf(
db.cPtr,
cfHandle.cPtr,
cast[cstringArray](sstPath.addr), csize_t(1),
db.ingestOptsPtr,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc close*(db: RocksDbRef) =
## Close the `RocksDbRef` which will release the connection to the database
## and free the memory associated with it. `close` is idempotent and can
## safely be called multple times. `close` is a no-op if the `RocksDbRef`
## is already closed.
withLock(db.lock):
if not db.isClosed():
db.dbOpts.close()
db.readOpts.close()
db.cfTable.close()
if db of RocksDbReadWriteRef:
let db = RocksDbReadWriteRef(db)
db.writeOpts.close()
rocksdb_ingestexternalfileoptions_destroy(db.ingestOptsPtr)
db.ingestOptsPtr = nil
rocksdb_close(db.cPtr)
db.cPtr = nil

133
rocksdb/rocksiterator.nim Normal file
View File

@ -0,0 +1,133 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## A `RocksIteratorRef` is a reference to a RocksDB iterator which supports
## iterating over the key value pairs in a column family.
{.push raises: [].}
import
./lib/librocksdb,
./internal/utils,
./rocksresult
export
rocksresult
type
RocksIteratorPtr* = ptr rocksdb_iterator_t
RocksIteratorRef* = ref object
cPtr: RocksIteratorPtr
proc newRocksIterator*(cPtr: RocksIteratorPtr): RocksIteratorRef =
doAssert not cPtr.isNil()
RocksIteratorRef(cPtr: cPtr)
proc isClosed*(iter: RocksIteratorRef): bool {.inline.} =
## Returns `true` if the iterator is closed and `false` otherwise.
iter.cPtr.isNil()
proc seekToFirst*(iter: RocksIteratorRef) =
## Seeks to the first entry in the column family.
doAssert not iter.isClosed()
rocksdb_iter_seek_to_first(iter.cPtr)
proc seekToLast*(iter: RocksIteratorRef) =
## Seeks to the last entry in the column family.
doAssert not iter.isClosed()
rocksdb_iter_seek_to_last(iter.cPtr)
proc isValid*(iter: RocksIteratorRef): bool =
## Returns `true` if the iterator is valid and `false` otherwise.
rocksdb_iter_valid(iter.cPtr).bool
proc next*(iter: RocksIteratorRef) =
## Seeks to the next entry in the column family.
rocksdb_iter_next(iter.cPtr)
proc prev*(iter: RocksIteratorRef) =
## Seeks to the previous entry in the column family.
rocksdb_iter_prev(iter.cPtr)
proc key*(iter: RocksIteratorRef, onData: DataProc) =
## Returns the current key using the provided `onData` callback.
var kLen: csize_t
let kData = rocksdb_iter_key(iter.cPtr, kLen.addr)
if kData.isNil or kLen == 0:
onData([])
else:
onData(kData.toOpenArrayByte(0, kLen.int - 1))
proc key*(iter: RocksIteratorRef): seq[byte] =
## Returns the current key.
var res: seq[byte]
proc onData(data: openArray[byte]) =
res = @data
iter.key(onData)
res
proc value*(iter: RocksIteratorRef, onData: DataProc) =
## Returns the current value using the provided `onData` callback.
var vLen: csize_t
let vData = rocksdb_iter_value(iter.cPtr, vLen.addr)
if vData.isNil or vLen == 0:
onData([])
else:
onData(vData.toOpenArrayByte(0, vLen.int - 1))
proc value*(iter: RocksIteratorRef): seq[byte] =
## Returns the current value.
var res: seq[byte]
proc onData(data: openArray[byte]) =
res = @data
iter.value(onData)
res
proc status*(iter: RocksIteratorRef): RocksDBResult[void] =
## Returns the status of the iterator.
doAssert not iter.isClosed()
var errors: cstring
rocksdb_iter_get_error(iter.cPtr, cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc close*(iter: RocksIteratorRef) =
## Closes the `RocksIteratorRef`.
if not iter.isClosed():
rocksdb_iter_destroy(iter.cPtr)
iter.cPtr = nil
iterator pairs*(iter: RocksIteratorRef): tuple[key: seq[byte], value: seq[byte]] =
## Iterates over the key value pairs in the column family yielding them in
## the form of a tuple. The iterator is automatically closed after the
## iteration.
doAssert not iter.isClosed()
defer: iter.close()
iter.seekToFirst()
while iter.isValid():
var
key: seq[byte]
value: seq[byte]
iter.key(proc(data: openArray[byte]) = key = @data)
iter.value(proc(data: openArray[byte]) = value = @data)
iter.next()
yield (key, value)

15
tests/all.nim → rocksdb/rocksresult.nim
View File

@ -1,5 +1,5 @@
# Nim-RocksDB
# Copyright 2018-2019 Status Research & Development GmbH
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
@ -7,6 +7,15 @@
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
./test_rocksdb_c,
./test_rocksdb
results
export
results
type
RocksDBResult*[T] = Result[T, string]
DataProc* = proc(val: openArray[byte]) {.gcsafe, raises: [].}

101
rocksdb/sstfilewriter.nim Normal file
View File

@ -0,0 +1,101 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## A `SstFileWriterRef` is used to create sst files that can be added to the database later.
{.push raises: [].}
import
./lib/librocksdb,
./internal/utils,
./options/dbopts,
./rocksresult
export
rocksresult
type
SstFileWriterPtr* = ptr rocksdb_sstfilewriter_t
EnvOptionsPtr = ptr rocksdb_envoptions_t
SstFileWriterRef* = ref object
cPtr: SstFileWriterPtr
envOptsPtr: EnvOptionsPtr
dbOpts: DbOptionsRef
proc openSstFileWriter*(
filePath: string,
dbOpts = defaultDbOptions()): RocksDBResult[SstFileWriterRef] =
## Creates a new `SstFileWriterRef` and opens the file at the given `filePath`.
doAssert not dbOpts.isClosed()
let envOptsPtr = rocksdb_envoptions_create()
let writer = SstFileWriterRef(
cPtr: rocksdb_sstfilewriter_create(envOptsPtr, dbOpts.cPtr),
envOptsPtr: envOptsPtr,
dbOpts: dbOpts)
var errors: cstring
rocksdb_sstfilewriter_open(
writer.cPtr,
filePath.cstring,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok(writer)
proc isClosed*(writer: SstFileWriterRef): bool {.inline.} =
## Returns `true` if the `SstFileWriterRef` is closed and `false` otherwise.
writer.cPtr.isNil()
proc put*(
writer: SstFileWriterRef,
key: openArray[byte],
val: openArray[byte]): RocksDBResult[void] =
## Add a key-value pair to the sst file.
var errors: cstring
rocksdb_sstfilewriter_put(
writer.cPtr,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
cast[cstring](unsafeAddr val[0]), csize_t(val.len),
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc delete*(writer: SstFileWriterRef, key: openArray[byte]): RocksDBResult[void] =
## Delete a key-value pair from the sst file.
var errors: cstring
rocksdb_sstfilewriter_delete(
writer.cPtr,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc finish*(writer: SstFileWriterRef): RocksDBResult[void] =
## Finish the process and close the sst file.
doAssert not writer.isClosed()
var errors: cstring
rocksdb_sstfilewriter_finish(writer.cPtr, cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc close*(writer: SstFileWriterRef) =
## Closes the `SstFileWriterRef`.
if not writer.isClosed():
rocksdb_envoptions_destroy(writer.envOptsPtr)
writer.envOptsPtr = nil
rocksdb_sstfilewriter_destroy(writer.cPtr)
writer.cPtr = nil

118
rocksdb/transactiondb.nim Normal file
View File

@ -0,0 +1,118 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## A `TransactionDbRef` can be used to open a connection to the RocksDB database
## with support for transactional operations against multiple column families.
## To create a new transaction call `beginTransaction` which will return a
## `TransactionRef`. To commit or rollback the transaction call `commit` or
## `rollback` on the `TransactionRef` type after applying changes to the transaction.
{.push raises: [].}
import
std/[sequtils, locks],
./lib/librocksdb,
./options/[dbopts, readopts, writeopts],
./transactions/[transaction, txdbopts, txopts],
./columnfamily/[cfopts, cfdescriptor, cfhandle],
./internal/[cftable, utils],
./rocksresult
export
dbopts,
txdbopts,
cfdescriptor,
readopts,
writeopts,
txopts,
transaction,
rocksresult
type
TransactionDbPtr* = ptr rocksdb_transactiondb_t
TransactionDbRef* = ref object
lock: Lock
cPtr: TransactionDbPtr
path: string
dbOpts: DbOptionsRef
txDbOpts: TransactionDbOptionsRef
cfTable: ColFamilyTableRef
proc openTransactionDb*(
path: string,
dbOpts = defaultDbOptions(),
txDbOpts = defaultTransactionDbOptions(),
columnFamilies = @[defaultColFamilyDescriptor()]): RocksDBResult[TransactionDbRef] =
## Open a `TransactionDbRef` with the given options and column families.
## If no column families are provided the default column family will be used.
## If no options are provided the default options will be used.
if columnFamilies.len == 0:
return err("rocksdb: no column families")
var
cfNames = columnFamilies.mapIt(it.name().cstring)
cfOpts = columnFamilies.mapIt(it.options.cPtr)
columnFamilyHandles = newSeq[ColFamilyHandlePtr](columnFamilies.len)
errors: cstring
let txDbPtr = rocksdb_transactiondb_open_column_families(
dbOpts.cPtr,
txDbOpts.cPtr,
path.cstring,
cfNames.len().cint,
cast[cstringArray](cfNames[0].addr),
cfOpts[0].addr,
columnFamilyHandles[0].addr,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
let db = TransactionDbRef(
lock: createLock(),
cPtr: txDbPtr,
path: path,
dbOpts: dbOpts,
txDbOpts: txDbOpts,
cfTable: newColFamilyTable(cfNames.mapIt($it), columnFamilyHandles))
ok(db)
proc isClosed*(db: TransactionDbRef): bool {.inline.} =
## Returns `true` if the `TransactionDbRef` has been closed.
db.cPtr.isNil()
proc beginTransaction*(
db: TransactionDbRef,
readOpts = defaultReadOptions(),
writeOpts = defaultWriteOptions(),
txOpts = defaultTransactionOptions(),
columnFamily = DEFAULT_COLUMN_FAMILY_NAME): TransactionRef =
## Begin a new transaction against the database. The transaction will default
## to using the specified column family. If no column family is specified
## then the default column family will be used.
doAssert not db.isClosed()
let txPtr = rocksdb_transaction_begin(
db.cPtr,
writeOpts.cPtr,
txOpts.cPtr,
nil)
newTransaction(txPtr, readOpts, writeOpts, txOpts, columnFamily, db.cfTable)
proc close*(db: TransactionDbRef) =
## Close the `TransactionDbRef`.
withLock(db.lock):
if not db.isClosed():
db.dbOpts.close()
db.txDbOpts.close()
db.cfTable.close()
rocksdb_transactiondb_close(db.cPtr)
db.cPtr = nil

193
rocksdb/transactions/transaction.nim Normal file
View File

@ -0,0 +1,193 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## To use transactions, you must first create a `TransactionDbRef`. Then to
## create a transaction call `beginTransaction` on the `TransactionDbRef`.
## `commit` and `rollback` are used to commit or rollback a transaction.
## The `TransactionDbRef` currently supports `put`, `delete` and `get` operations.
## Keys that have been writen to a transaction but are not yet committed can be
## read from the transaction using `get`. Uncommitted updates will not be visible
## to other transactions until they are committed to the database.
## Multiple column families can be written to and read from in a single transaction
## but a default column family will be used if none is specified in each call.
{.push raises: [].}
import
../lib/librocksdb,
../options/[readopts, writeopts],
../internal/[cftable, utils],
../rocksresult,
./txopts
export
rocksresult
type
TransactionPtr* = ptr rocksdb_transaction_t
TransactionRef* = ref object
cPtr: TransactionPtr
readOpts: ReadOptionsRef
writeOpts: WriteOptionsRef
txOpts: TransactionOptionsRef
defaultCfName: string
cfTable: ColFamilyTableRef
proc newTransaction*(
cPtr: TransactionPtr,
readOpts: ReadOptionsRef,
writeOpts: WriteOptionsRef,
txOpts: TransactionOptionsRef,
defaultCfName: string,
cfTable: ColFamilyTableRef): TransactionRef =
TransactionRef(
cPtr: cPtr,
readOpts: readOpts,
writeOpts: writeOpts,
txOpts: txOpts,
defaultCfName: defaultCfName,
cfTable: cfTable)
proc isClosed*(tx: TransactionRef): bool {.inline.} =
## Returns `true` if the `TransactionRef` has been closed.
tx.cPtr.isNil()
proc get*(
tx: TransactionRef,
key: openArray[byte],
onData: DataProc,
columnFamily = tx.defaultCfName): RocksDBResult[bool] =
## Get the value for a given key from the transaction using the provided
## `onData` callback.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = tx.cfTable.get(columnFamily)
if cfHandle.isNil():
return err("rocksdb: unknown column family")
var
len: csize_t
errors: cstring
let data = rocksdb_transaction_get_cf(
tx.cPtr,
tx.readOpts.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
len.addr,
cast[cstringArray](errors.addr))
bailOnErrors(errors)
if data.isNil():
doAssert len == 0
ok(false)
else:
onData(toOpenArrayByte(data, 0, len.int - 1))
rocksdb_free(data)
ok(true)
proc get*(
tx: TransactionRef,
key: openArray[byte],
columnFamily = tx.defaultCfName): RocksDBResult[seq[byte]] =
## Get the value for a given key from the transaction.
var dataRes: RocksDBResult[seq[byte]]
proc onData(data: openArray[byte]) =
dataRes.ok(@data)
let res = tx.get(key, onData, columnFamily)
if res.isOk():
return dataRes
dataRes.err(res.error())
proc put*(
tx: TransactionRef,
key, val: openArray[byte],
columnFamily = tx.defaultCfName): RocksDBResult[void] =
## Put the value for the given key into the transaction.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = tx.cfTable.get(columnFamily)
if cfHandle.isNil():
return err("rocksdb: unknown column family")
var errors: cstring
rocksdb_transaction_put_cf(
tx.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
csize_t(val.len),
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc delete*(
tx: TransactionRef,
key: openArray[byte],
columnFamily = tx.defaultCfName): RocksDBResult[void] =
## Delete the value for the given key from the transaction.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = tx.cfTable.get(columnFamily)
if cfHandle.isNil:
return err("rocksdb: unknown column family")
var errors: cstring
rocksdb_transaction_delete_cf(
tx.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc commit*(tx: TransactionRef): RocksDBResult[void] =
## Commit the transaction.
doAssert not tx.isClosed()
var errors: cstring
rocksdb_transaction_commit(tx.cPtr, cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc rollback*(tx: TransactionRef): RocksDBResult[void] =
## Rollback the transaction.
doAssert not tx.isClosed()
var errors: cstring
rocksdb_transaction_rollback(tx.cPtr, cast[cstringArray](errors.addr))
bailOnErrors(errors)
ok()
proc close*(tx: TransactionRef) =
## Close the `TransactionRef`.
if not tx.isClosed():
tx.readOpts.close()
tx.writeOpts.close()
tx.txOpts.close()
rocksdb_transaction_destroy(tx.cPtr)
tx.cPtr = nil

40
rocksdb/transactions/txdbopts.nim Normal file
View File

@ -0,0 +1,40 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../lib/librocksdb
type
TransactionDbOptionsPtr* = ptr rocksdb_transactiondb_options_t
TransactionDbOptionsRef* = ref object
cPtr: TransactionDbOptionsPtr
proc newTransactionDbOptions*(): TransactionDbOptionsRef =
TransactionDbOptionsRef(cPtr: rocksdb_transactiondb_options_create())
proc isClosed*(txDbOpts: TransactionDbOptionsRef): bool {.inline.} =
txDbOpts.cPtr.isNil()
proc cPtr*(txDbOpts: TransactionDbOptionsRef): TransactionDbOptionsPtr =
doAssert not txDbOpts.isClosed()
txDbOpts.cPtr
# TODO: Add setters and getters for backup options properties.
proc defaultTransactionDbOptions*(): TransactionDbOptionsRef {.inline.} =
newTransactionDbOptions()
# TODO: set prefered defaults
proc close*(txDbOpts: TransactionDbOptionsRef) =
if not txDbOpts.isClosed():
rocksdb_transactiondb_options_destroy(txDbOpts.cPtr)
txDbOpts.cPtr = nil

40
rocksdb/transactions/txopts.nim Normal file
View File

@ -0,0 +1,40 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
../lib/librocksdb
type
TransactionOptionsPtr* = ptr rocksdb_transaction_options_t
TransactionOptionsRef* = ref object
cPtr: TransactionOptionsPtr
proc newTransactionOptions*(): TransactionOptionsRef =
TransactionOptionsRef(cPtr: rocksdb_transaction_options_create())
proc isClosed*(txOpts: TransactionOptionsRef): bool {.inline.} =
txOpts.cPtr.isNil()
proc cPtr*(txOpts: TransactionOptionsRef): TransactionOptionsPtr =
doAssert not txOpts.isClosed()
txOpts.cPtr
# TODO: Add setters and getters for backup options properties.
proc defaultTransactionOptions*(): TransactionOptionsRef {.inline.} =
newTransactionOptions()
# TODO: set prefered defaults
proc close*(txOpts: TransactionOptionsRef) =
if not txOpts.isClosed():
rocksdb_transaction_options_destroy(txOpts.cPtr)
txOpts.cPtr = nil

103
rocksdb/writebatch.nim Normal file
View File

@ -0,0 +1,103 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
## A `WriteBatchRef` holds a collection of updates to apply atomically to the database.
{.push raises: [].}
import
./lib/librocksdb,
./internal/[cftable, utils],
./rocksresult
export
rocksresult
type
WriteBatchPtr* = ptr rocksdb_writebatch_t
WriteBatchRef* = ref object
cPtr: WriteBatchPtr
defaultCfName: string
cfTable: ColFamilyTableRef
proc newWriteBatch*(cfTable: ColFamilyTableRef, defaultCfName: string): WriteBatchRef =
WriteBatchRef(
cPtr: rocksdb_writebatch_create(),
defaultCfName: defaultCfName,
cfTable: cfTable)
proc isClosed*(batch: WriteBatchRef): bool {.inline.} =
## Returns `true` if the `WriteBatchRef` has been closed and `false` otherwise.
batch.cPtr.isNil()
proc cPtr*(batch: WriteBatchRef): WriteBatchPtr =
## Get the underlying database pointer.
doAssert not batch.isClosed()
batch.cPtr
proc clear*(batch: WriteBatchRef) =
## Clears the write batch.
doAssert not batch.isClosed()
rocksdb_writebatch_clear(batch.cPtr)
proc count*(batch: WriteBatchRef): int =
## Get the number of updates in the write batch.
doAssert not batch.isClosed()
rocksdb_writebatch_count(batch.cPtr).int
proc put*(
batch: WriteBatchRef,
key, val: openArray[byte],
columnFamily = DEFAULT_COLUMN_FAMILY_NAME): RocksDBResult[void] =
## Add a put operation to the write batch.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = batch.cfTable.get(columnFamily)
if cfHandle.isNil:
return err("rocksdb: unknown column family")
rocksdb_writebatch_put_cf(
batch.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
csize_t(val.len))
ok()
proc delete*(
batch: WriteBatchRef,
key: openArray[byte],
columnFamily = DEFAULT_COLUMN_FAMILY_NAME): RocksDBResult[void] =
## Add a delete operation to the write batch.
if key.len() == 0:
return err("rocksdb: key is empty")
let cfHandle = batch.cfTable.get(columnFamily)
if cfHandle.isNil:
return err("rocksdb: unknown column family")
rocksdb_writebatch_delete_cf(
batch.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len))
ok()
proc close*(batch: WriteBatchRef) =
## Close the `WriteBatchRef`.
if not batch.isClosed():
rocksdb_writebatch_destroy(batch.cPtr)
batch.cPtr = nil

59
tests/columnfamily/test_cfdescriptor.nim Normal file
View File

@ -0,0 +1,59 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/internal/utils,
../../rocksdb/columnfamily/cfdescriptor
suite "ColFamilyDescriptor Tests":
const TEST_CF_NAME = "test"
test "Test initColFamilyDescriptor":
var descriptor = initColFamilyDescriptor(TEST_CF_NAME)
check:
descriptor.name() == TEST_CF_NAME
not descriptor.options().isNil()
not descriptor.isDefault()
descriptor.close()
test "Test initColFamilyDescriptor with options":
var descriptor = initColFamilyDescriptor(TEST_CF_NAME, defaultColFamilyOptions())
check:
descriptor.name() == TEST_CF_NAME
not descriptor.options().isNil()
not descriptor.isDefault()
descriptor.close()
test "Test defaultColFamilyDescriptor":
var descriptor = defaultColFamilyDescriptor()
check:
descriptor.name() == DEFAULT_COLUMN_FAMILY_NAME
not descriptor.options().isNil()
descriptor.isDefault()
descriptor.close()
test "Test close":
var descriptor = defaultColFamilyDescriptor()
check not descriptor.isClosed()
descriptor.close()
check descriptor.isClosed()
descriptor.close()
check descriptor.isClosed()

67
tests/columnfamily/test_cfhandle.nim Normal file
View File

@ -0,0 +1,67 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../../rocksdb/lib/librocksdb,
../../rocksdb/columnfamily/cfhandle
suite "ColFamilyHandleRef Tests":
const TEST_CF_NAME = "test"
setup:
let
dbPath = mkdtemp() / "data"
dbOpts = rocksdb_options_create()
cfOpts = rocksdb_options_create()
var
errors: cstring
rocksdb_options_set_create_if_missing(dbOpts, 1);
let db = rocksdb_open(dbOpts, dbPath.cstring, cast[cstringArray](errors.addr))
doAssert errors.isNil()
doAssert not db.isNil()
let cfHandlePtr = rocksdb_create_column_family(
db,
cfOpts,
TEST_CF_NAME.cstring,
cast[cstringArray](errors.addr))
doAssert errors.isNil()
doAssert not cfHandlePtr.isNil()
teardown:
rocksdb_close(db)
removeDir($dbPath)
test "Test newColFamilyHandle":
var cfHandle = newColFamilyHandle(cfHandlePtr)
check:
not cfHandle.cPtr.isNil()
cfHandle.cPtr == cfHandlePtr
cfHandle.close()
test "Test close":
var cfHandle = newColFamilyHandle(cfHandlePtr)
check not cfHandle.isClosed()
cfHandle.close()
check cfHandle.isClosed()
cfHandle.close()
check cfHandle.isClosed()

47
tests/columnfamily/test_cfopts.nim Normal file
View File

@ -0,0 +1,47 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/columnfamily/cfopts
suite "ColFamilyOptionsRef Tests":
test "Test newColFamilyOptions":
var cfOpts = newColFamilyOptions()
check not cfOpts.cPtr.isNil()
# check not cfOpts.getCreateMissingColumnFamilies()
cfOpts.setCreateMissingColumnFamilies(true)
# check cfOpts.getCreateMissingColumnFamilies()
cfOpts.close()
test "Test defaultColFamilyOptions":
var cfOpts = defaultColFamilyOptions()
check not cfOpts.cPtr.isNil()
# check cfOpts.getCreateMissingColumnFamilies()
cfOpts.setCreateMissingColumnFamilies(false)
# check not cfOpts.getCreateMissingColumnFamilies()
cfOpts.close()
test "Test close":
var cfOpts = defaultColFamilyOptions()
check not cfOpts.isClosed()
cfOpts.close()
check cfOpts.isClosed()
cfOpts.close()
check cfOpts.isClosed()

77
tests/internal/test_cftable.nim Normal file
View File

@ -0,0 +1,77 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../../rocksdb/lib/librocksdb,
../../rocksdb/columnfamily/cfhandle,
../../rocksdb/internal/cftable
suite "ColFamilyTableRef Tests":
const TEST_CF_NAME = "test"
setup:
let
dbPath = mkdtemp() / "data"
dbOpts = rocksdb_options_create()
cfOpts = rocksdb_options_create()
var
errors: cstring
rocksdb_options_set_create_if_missing(dbOpts, 1);
let db = rocksdb_open(dbOpts, dbPath.cstring, cast[cstringArray](errors.addr))
doAssert errors.isNil()
doAssert not db.isNil()
let cfHandlePtr = rocksdb_create_column_family(
db,
cfOpts,
TEST_CF_NAME.cstring,
cast[cstringArray](errors.addr))
doAssert errors.isNil()
doAssert not cfHandlePtr.isNil()
teardown:
rocksdb_close(db)
removeDir($dbPath)
test "Test newColFamilyTable":
var cfTable = newColFamilyTable(
@[TEST_CF_NAME, TEST_CF_NAME],
@[cfHandlePtr, cfHandlePtr])
check cfTable.get(TEST_CF_NAME).cPtr() == cfHandlePtr
check not cfTable.isClosed()
# doesn't exist
check cfTable.get("other").isNil()
check not cfTable.isClosed()
cfTable.close()
test "Test close":
var cfTable = newColFamilyTable(@[TEST_CF_NAME], @[cfHandlePtr])
let cfHandle = cfTable.get(TEST_CF_NAME)
check not cfHandle.isClosed()
check not cfTable.isClosed()
cfTable.close()
check cfHandle.isClosed()
check cfTable.isClosed()
cfTable.close()
check cfTable.isClosed()

33
tests/test_rocksdb_c.nim → tests/lib/test_librocksdb.nim
View File

@ -1,5 +1,5 @@
# Nim-RocksDB
# Copyright 2018-2019 Status Research & Development GmbH
# Copyright 2018-2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
@ -10,15 +10,16 @@
{.used.}
import
cpuinfo, os, unittest,
std/[cpuinfo, os],
tempfile,
../rocksdb
unittest2,
../../rocksdb/lib/librocksdb
suite "RocksDB C wrapper tests":
suite "librocksdb C wrapper Tests":
setup:
let
dbPath: cstring = mkdtemp()
dbBackupPath: cstring = mkdtemp()
dbPath = mkdtemp().cstring
dbBackupPath = mkdtemp().cstring
teardown:
removeDir($dbPath)
@ -26,8 +27,8 @@ suite "RocksDB C wrapper tests":
test "Simple create-update-close example":
var
db: rocksdb_t
be: rocksdb_backup_engine_t
db: ptr rocksdb_t
be: ptr rocksdb_backup_engine_t
options = rocksdb_options_create()
let cpus = countProcessors()
@ -39,12 +40,12 @@ suite "RocksDB C wrapper tests":
rocksdb_options_set_create_if_missing(options, 1);
# open DB
var err: cstring # memory leak: example code does not free error string!
db = rocksdb_open(options, dbPath, err.addr)
var err: cstringArray # memory leak: example code does not free error string!
db = rocksdb_open(options, dbPath, err)
check: err.isNil
# open Backup Engine that we will use for backing up our database
be = rocksdb_backup_engine_open(options, dbBackupPath, err.addr)
be = rocksdb_backup_engine_open(options, dbBackupPath, err)
check: err.isNil
# Put key-value
@ -53,14 +54,14 @@ suite "RocksDB C wrapper tests":
let put_value = "value"
rocksdb_put(
db, writeOptions, key.cstring, csize_t(key.len),
put_value.cstring, csize_t(put_value.len), err.addr)
put_value.cstring, csize_t(put_value.len), err)
check: err.isNil
# Get value
var readOptions = rocksdb_readoptions_create()
var len: csize_t
let raw_value = rocksdb_get(
db, readOptions, key, csize_t(key.len), addr len, err.addr) # Important: rocksdb_get is not null-terminated
db, readOptions, key.cstring, csize_t(key.len), addr len, err) # Important: rocksdb_get is not null-terminated
check: err.isNil
# Copy it to a regular Nim string (copyMem workaround because non-null terminated)
@ -70,7 +71,7 @@ suite "RocksDB C wrapper tests":
check: $get_value == $put_value
# create new backup in a directory specified by DBBackupPath
rocksdb_backup_engine_create_new_backup(be, db, err.addr)
rocksdb_backup_engine_create_new_backup(be, db, err)
check: err.isNil
rocksdb_close(db)
@ -78,11 +79,11 @@ suite "RocksDB C wrapper tests":
# If something is wrong, you might want to restore data from last backup
var restoreOptions = rocksdb_restore_options_create()
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbPath, dbPath,
restoreOptions, err.addr)
restoreOptions, err)
check: err.isNil
rocksdb_restore_options_destroy(restore_options)
db = rocksdb_open(options, dbPath, err.addr)
db = rocksdb_open(options, dbPath, err)
check: err.isNil
# cleanup

39
tests/options/test_backupopts.nim Normal file
View File

@ -0,0 +1,39 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/options/backupopts
suite "BackupEngineOptionsRef Tests":
test "Test newBackupEngineOptions":
var backupOpts = newBackupEngineOptions()
check not backupOpts.cPtr.isNil()
backupOpts.close()
test "Test defaultBackupEngineOptions":
var backupOpts = defaultBackupEngineOptions()
check not backupOpts.cPtr.isNil()
backupOpts.close()
test "Test close":
var backupOpts = defaultBackupEngineOptions()
check not backupOpts.isClosed()
backupOpts.close()
check backupOpts.isClosed()
backupOpts.close()
check backupOpts.isClosed()

61
tests/options/test_dbopts.nim Normal file
View File

@ -0,0 +1,61 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/options/dbopts
suite "DbOptionsRef Tests":
test "Test newDbOptions":
var dbOpts = newDbOptions()
check not dbOpts.cPtr.isNil()
dbOpts.setCreateIfMissing(true)
dbOpts.setMaxOpenFiles(10)
dbOpts.setCreateMissingColumnFamilies(false)
# check:
# dbOpts.getCreateIfMissing()
# dbOpts.getMaxOpenFiles() == 10
# not dbOpts.getCreateMissingColumnFamilies()
dbOpts.close()
test "Test defaultDbOptions":
var dbOpts = defaultDbOptions()
check:
not dbOpts.cPtr.isNil()
# dbOpts.getCreateIfMissing()
# dbOpts.getMaxOpenFiles() == -1
# dbOpts.getCreateMissingColumnFamilies()
dbOpts.setCreateIfMissing(false)
dbOpts.setMaxOpenFiles(100)
dbOpts.setCreateMissingColumnFamilies(false)
# check:
# not dbOpts.getCreateIfMissing()
# dbOpts.getMaxOpenFiles() == 100
# not dbOpts.getCreateMissingColumnFamilies()
dbOpts.close()
test "Test close":
var dbOpts = defaultDbOptions()
check not dbOpts.isClosed()
dbOpts.close()
check dbOpts.isClosed()
dbOpts.close()
check dbOpts.isClosed()

39
tests/options/test_readopts.nim Normal file
View File

@ -0,0 +1,39 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/options/readopts
suite "ReadOptionsRef Tests":
test "Test newReadOptions":
var readOpts = newReadOptions()
check not readOpts.cPtr.isNil()
readOpts.close()
test "Test defaultReadOptions":
var readOpts = defaultReadOptions()
check not readOpts.cPtr.isNil()
readOpts.close()
test "Test close":
var readOpts = defaultReadOptions()
check not readOpts.isClosed()
readOpts.close()
check readOpts.isClosed()
readOpts.close()
check readOpts.isClosed()

39
tests/options/test_writeopts.nim Normal file
View File

@ -0,0 +1,39 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/options/writeopts
suite "WriteOptionsRef Tests":
test "Test newWriteOptions":
var writeOpts = newWriteOptions()
check not writeOpts.cPtr.isNil()
writeOpts.close()
test "Test defaultWriteOptions":
var writeOpts = defaultWriteOptions()
check not writeOpts.cPtr.isNil()
writeOpts.close()
test "Test close":
var writeOpts = defaultWriteOptions()
check not writeOpts.isClosed()
writeOpts.close()
check writeOpts.isClosed()
writeOpts.close()
check writeOpts.isClosed()

27
tests/test_all.nim Normal file
View File

@ -0,0 +1,27 @@
# Nim-RocksDB
# Copyright 2018-2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
./columnfamily/test_cfdescriptor,
./columnfamily/test_cfhandle,
./columnfamily/test_cfopts,
./internal/test_cftable,
./lib/test_librocksdb,
./options/test_backupopts,
./options/test_dbopts,
./options/test_readopts,
./options/test_writeopts,
./transactions/test_txdbopts,
./transactions/test_txopts,
./test_backup,
./test_columnfamily,
./test_rocksdb,
./test_rocksiterator,
./test_sstfilewriter,
./test_writebatch

70
tests/test_backup.nim Normal file
View File

@ -0,0 +1,70 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/backup,
./test_helper
suite "BackupEngineRef Tests":
let
key = @[byte(1), 2, 3, 4, 5]
val = @[byte(1), 2, 3, 4, 5]
setup:
let
dbPath = mkdtemp() / "data"
dbBackupPath = mkdtemp() / "backup"
dbRestorePath = mkdtemp() / "restore"
var
db = initReadWriteDb(dbPath)
teardown:
db.close()
removeDir($dbPath)
removeDir($dbBackupPath)
test "Test backup":
var engine = initBackupEngine(dbBackupPath)
check:
db.put(key, val).isOk()
db.keyExists(key).value()
check engine.createNewBackup(db).isOk()
check:
db.delete(key).isOk()
not db.keyExists(key).value()
check engine.restoreDbFromLatestBackup(dbRestorePath).isOk()
let db2 = initReadWriteDb(dbRestorePath)
check db2.keyExists(key).value()
engine.close()
test "Test close":
let res = openBackupEngine(dbPath)
doAssert res.isOk()
var engine = res.get()
check not engine.isClosed()
engine.close()
check engine.isClosed()
engine.close()
check engine.isClosed()

84
tests/test_columnfamily.nim Normal file
View File

@ -0,0 +1,84 @@
# Nim-RocksDB
# Copyright 2018-2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/columnfamily,
./test_helper
suite "ColFamily Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
let
key = @[byte(1), 2, 3, 4, 5]
otherKey = @[byte(1), 2, 3, 4, 5, 6]
val = @[byte(1), 2, 3, 4, 5]
setup:
let
dbPath = mkdtemp() / "data"
db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
teardown:
db.close()
removeDir($dbPath)
test "Basic operations":
let r0 = db.withColFamily(CF_OTHER)
check r0.isOk()
let cf = r0.value()
check cf.put(key, val).isOk()
var bytes: seq[byte]
check cf.get(key, proc(data: openArray[byte]) = bytes = @data)[]
check not cf.get(otherkey, proc(data: openArray[byte]) = bytes = @data)[]
var r1 = cf.get(key)
check r1.isOk() and r1.value == val
var r2 = cf.get(otherKey)
# there's no error string for missing keys
check r2.isOk() == false and r2.error.len == 0
var e1 = cf.keyExists(key)
check e1.isOk() and e1.value == true
var e2 = cf.keyExists(otherKey)
check e2.isOk() and e2.value == false
var d = cf.delete(key)
check d.isOk()
e1 = cf.keyExists(key)
check e1.isOk() and e1.value == false
d = cf.delete(otherKey)
check d.isOk()
cf.db.close()
check db.isClosed()
# Open database in read only mode
block:
var res = initReadOnlyDb(dbPath).withColFamily(CF_DEFAULT)
check res.isOk()
let readOnlyCf = res.value()
let r = readOnlyCf.keyExists(key)
check r.isOk() and r.value == false
readOnlyCf.db.close()
check readOnlyCf.db.isClosed()

59
tests/test_helper.nim Normal file
View File

@ -0,0 +1,59 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/sequtils,
../rocksdb/backup,
../rocksdb/rocksdb,
../rocksdb/transactiondb
proc initReadWriteDb*(
path: string,
columnFamilyNames = @["default"]): RocksDbReadWriteRef =
let res = openRocksDb(
path,
columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)))
if res.isErr():
echo res.error()
doAssert res.isOk()
res.value()
proc initReadOnlyDb*(
path: string,
columnFamilyNames = @["default"]): RocksDbReadOnlyRef =
let res = openRocksDbReadOnly(
path,
columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)))
if res.isErr():
echo res.error()
doAssert res.isOk()
res.value()
proc initBackupEngine*(path: string): BackupEngineRef =
let res = openBackupEngine(path)
doAssert res.isOk()
res.value()
proc initTransactionDb*(
path: string,
columnFamilyNames = @["default"]): TransactionDbRef =
let res = openTransactionDb(
path,
columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)))
if res.isErr():
echo res.error()
doAssert res.isOk()
res.value()

301
tests/test_rocksdb.nim
View File

@ -1,5 +1,5 @@
# Nim-RocksDB
# Copyright 2018-2019 Status Research & Development GmbH
# Copyright 2018-2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
@ -10,68 +10,275 @@
{.used.}
import
os, unittest,
std/os,
tempfile,
../rocksdb
unittest2,
../rocksdb/rocksdb,
./test_helper
type
MyDB = object
rocksdb: RocksDBInstance
suite "RocksDbRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
# TODO no tests for failures / error reporting
proc initMyDb(path: string): MyDB =
let
dataDir = path / "data"
backupsDir = path / "backups"
key = @[byte(1), 2, 3, 4, 5]
otherKey = @[byte(1), 2, 3, 4, 5, 6]
val = @[byte(1), 2, 3, 4, 5]
createDir(dataDir)
createDir(backupsDir)
var s = result.rocksdb.init(dataDir, backupsDir)
doAssert s.isOk, $s
suite "Nim API tests":
setup:
var
dbDir = mkdtemp()
db = initMyDb(dbDir)
let
dbPath = mkdtemp() / "data"
db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
teardown:
close(db.rocksdb)
removeDir(dbDir)
db.close()
removeDir($dbPath)
test "Basic operations":
proc test() =
let key = @[byte(1), 2, 3, 4, 5]
let otherKey = @[byte(1), 2, 3, 4, 5, 6]
let val = @[byte(1), 2, 3, 4, 5]
var s = db.rocksdb.put(key, val)
check s.isok
var s = db.put(key, val)
check s.isOk()
var bytes: seq[byte]
check db.rocksdb.get(key, proc(data: openArray[byte]) = bytes = @data)[]
check not db.rocksdb.get(
otherkey, proc(data: openArray[byte]) = bytes = @data)[]
var bytes: seq[byte]
check db.get(key, proc(data: openArray[byte]) = bytes = @data)[]
check not db.get(otherkey, proc(data: openArray[byte]) = bytes = @data)[]
var r1 = db.rocksdb.getBytes(key)
check r1.isok and r1.value == val
var r1 = db.get(key)
check r1.isOk() and r1.value == val
var r2 = db.rocksdb.getBytes(otherKey)
# there's no error string for missing keys
check r2.isok == false and r2.error.len == 0
var r2 = db.get(otherKey)
# there's no error string for missing keys
check r2.isOk() == false and r2.error.len == 0
var e1 = db.rocksdb.contains(key)
check e1.isok and e1.value == true
var e1 = db.keyExists(key)
check e1.isOk() and e1.value == true
var e2 = db.rocksdb.contains(otherKey)
check e2.isok and e2.value == false
var e2 = db.keyExists(otherKey)
check e2.isOk() and e2.value == false
var d = db.rocksdb.del(key)
check d.isok and d.value == true
var d = db.delete(key)
check d.isOk()
e1 = db.rocksdb.contains(key)
check e1.isok and e1.value == false
e1 = db.keyExists(key)
check e1.isOk() and e1.value == false
test()
d = db.delete(otherKey)
check d.isOk()
close(db)
check db.isClosed()
# Open database in read only mode
block:
var
readOnlyDb = initReadOnlyDb(dbPath)
r = readOnlyDb.keyExists(key)
check r.isOk() and r.value == false
# This won't compile as designed:
# var r2 = readOnlyDb.put(key, @[123.byte])
# check r2.isErr()
readOnlyDb.close()
check readOnlyDb.isClosed()
test "Basic operations - default column family":
var s = db.put(key, val, CF_DEFAULT)
check s.isOk()
var bytes: seq[byte]
check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
check not db.get(otherkey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
var r1 = db.get(key)
check r1.isOk() and r1.value == val
var r2 = db.get(otherKey)
# there's no error string for missing keys
check r2.isOk() == false and r2.error.len == 0
var e1 = db.keyExists(key, CF_DEFAULT)
check e1.isOk() and e1.value == true
var e2 = db.keyExists(otherKey, CF_DEFAULT)
check e2.isOk() and e2.value == false
var d = db.delete(key, CF_DEFAULT)
check d.isOk()
e1 = db.keyExists(key, CF_DEFAULT)
check e1.isOk() and e1.value == false
d = db.delete(otherKey, CF_DEFAULT)
check d.isOk()
close(db)
check db.isClosed()
# Open database in read only mode
block:
var
readOnlyDb = initReadOnlyDb(dbPath, columnFamilyNames = @[CF_DEFAULT])
r = readOnlyDb.keyExists(key, CF_DEFAULT)
check r.isOk() and r.value == false
# Won't compile as designed:
# var r2 = readOnlyDb.put(key, @[123.byte], CF_DEFAULT)
# check r2.isErr()
readOnlyDb.close()
check readOnlyDb.isClosed()
test "Basic operations - multiple column families":
var s = db.put(key, val, CF_DEFAULT)
check s.isOk()
var s2 = db.put(otherKey, val, CF_OTHER)
check s2.isOk()
var bytes: seq[byte]
check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
check not db.get(otherkey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
var bytes2: seq[byte]
check db.get(otherKey, proc(data: openArray[byte]) = bytes2 = @data, CF_OTHER)[]
check not db.get(key, proc(data: openArray[byte]) = bytes2 = @data, CF_OTHER)[]
var e1 = db.keyExists(key, CF_DEFAULT)
check e1.isOk() and e1.value == true
var e2 = db.keyExists(otherKey, CF_DEFAULT)
check e2.isOk() and e2.value == false
var e3 = db.keyExists(key, CF_OTHER)
check e3.isOk() and e3.value == false
var e4 = db.keyExists(otherKey, CF_OTHER)
check e4.isOk() and e4.value == true
var d = db.delete(key, CF_DEFAULT)
check d.isOk()
e1 = db.keyExists(key, CF_DEFAULT)
check e1.isOk() and e1.value == false
d = db.delete(otherKey, CF_DEFAULT)
check d.isOk()
var d2 = db.delete(key, CF_OTHER)
check d2.isOk()
e3 = db.keyExists(key, CF_OTHER)
check e3.isOk() and e3.value == false
d2 = db.delete(otherKey, CF_OTHER)
check d2.isOk()
d2 = db.delete(otherKey, CF_OTHER)
check d2.isOk()
db.close()
check db.isClosed()
# Open database in read only mode
block:
var
readOnlyDb = initReadOnlyDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
var r = readOnlyDb.keyExists(key, CF_OTHER)
check r.isOk() and r.value == false
# Does not compile as designed:
# var r2 = readOnlyDb.put(key, @[123.byte], CF_OTHER)
# check r2.isErr()
readOnlyDb.close()
check readOnlyDb.isClosed()
test "Close multiple times":
check not db.isClosed()
db.close()
check db.isClosed()
db.close()
check db.isClosed()
test "Unknown column family":
const CF_UNKNOWN = "unknown"
let r = db.put(key, val, CF_UNKNOWN)
check r.isErr() and r.error() == "rocksdb: unknown column family"
var bytes: seq[byte]
let r2 = db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_UNKNOWN)
check r2.isErr() and r2.error() == "rocksdb: unknown column family"
let r3 = db.keyExists(key, CF_UNKNOWN)
check r3.isErr() and r3.error() == "rocksdb: unknown column family"
let r4 = db.delete(key, CF_UNKNOWN)
check r4.isErr() and r4.error() == "rocksdb: unknown column family"
test "Test missing key and values":
let
key1 = @[byte(1)] # exists with non empty value
val1 = @[byte(1)]
key2 = @[byte(2)] # exists with empty seq value
val2: seq[byte] = @[]
key3 = @[byte(3)] # exists with empty array value
val3: array[0, byte] = []
key4 = @[byte(4)] # deleted key
key5 = @[byte(5)] # key not created
check:
db.put(key1, val1).isOk()
db.put(key2, val2).isOk()
db.put(key3, val3).isOk()
db.delete(key4).isOk()
db.keyExists(key1).get() == true
db.keyExists(key2).get() == true
db.keyExists(key3).get() == true
db.keyExists(key4).get() == false
db.keyExists(key5).get() == false
block:
var v: seq[byte]
let r = db.get(key1, proc(data: openArray[byte]) = v = @data)
check:
r.isOk()
r.value() == true
v == val1
db.get(key1).isOk()
block:
var v: seq[byte]
let r = db.get(key2, proc(data: openArray[byte]) = v = @data)
check:
r.isOk()
r.value() == true
v.len() == 0
db.get(key2).isOk()
block:
var v: seq[byte]
let r = db.get(key3, proc(data: openArray[byte]) = v = @data)
check:
r.isOk()
r.value() == true
v.len() == 0
db.get(key3).isOk()
block:
var v: seq[byte]
let r = db.get(key4, proc(data: openArray[byte]) = v = @data)
check:
r.isOk()
r.value() == false
v.len() == 0
db.get(key4).isErr()
block:
var v: seq[byte]
let r = db.get(key5, proc(data: openArray[byte]) = v = @data)
check:
r.isOk()
r.value() == false
v.len() == 0
db.get(key5).isErr()

196
tests/test_rocksiterator.nim Normal file
View File

@ -0,0 +1,196 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[rocksdb, rocksiterator],
./test_helper
suite "RocksIteratorRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
CF_EMPTY = "empty"
let
key1 = @[byte(1)]
val1 = @[byte(1)]
key2 = @[byte(2)]
val2 = @[byte(2)]
key3 = @[byte(3)]
val3 = @[byte(3)]
setup:
let
dbPath = mkdtemp() / "data"
db = initReadWriteDb(dbPath,
columnFamilyNames = @[CF_DEFAULT, CF_OTHER, CF_EMPTY])
doAssert db.put(key1, val1).isOk()
doAssert db.put(key2, val2).isOk()
doAssert db.put(key3, val3).isOk()
doAssert db.put(key1, val1, CF_OTHER).isOk()
doAssert db.put(key2, val2, CF_OTHER).isOk()
doAssert db.put(key3, val3, CF_OTHER).isOk()
teardown:
db.close()
removeDir($dbPath)
test "Iterate forwards using default column family":
let res = db.openIterator(CF_DEFAULT)
check res.isOk()
var iter = res.get()
defer: iter.close()
iter.seekToFirst()
check iter.isValid()
var expected = byte(1)
while iter.isValid():
let
key = iter.key()
val = iter.value()
check:
key == @[expected]
val == @[expected]
inc expected
iter.next()
check expected == byte(4)
test "Iterate backwards using other column family":
let res = db.openIterator(CF_OTHER)
check res.isOk()
var iter = res.get()
defer: iter.close()
iter.seekToLast()
check iter.isValid()
var expected = byte(3)
while iter.isValid():
var key: seq[byte]
iter.key(proc(data: openArray[byte]) = key = @data)
var val: seq[byte]
iter.value(proc(data: openArray[byte]) = val = @data)
check:
key == @[expected]
val == @[expected]
dec expected
iter.prev()
check expected == byte(0)
iter.close()
test "Open two iterators on the same column family":
let res1 = db.openIterator(CF_DEFAULT)
check res1.isOk()
var iter1 = res1.get()
defer: iter1.close()
let res2 = db.openIterator(CF_DEFAULT)
check res2.isOk()
var iter2 = res2.get()
defer: iter2.close()
iter1.seekToFirst()
check iter1.isValid()
iter2.seekToLast()
check iter2.isValid()
check:
iter1.key() == @[byte(1)]
iter1.value() == @[byte(1)]
iter2.key() == @[byte(3)]
iter2.value() == @[byte(3)]
test "Open two iterators on different column families":
let res1 = db.openIterator(CF_DEFAULT)
check res1.isOk()
var iter1 = res1.get()
defer: iter1.close()
let res2 = db.openIterator(CF_OTHER)
check res2.isOk()
var iter2 = res2.get()
defer: iter2.close()
iter1.seekToFirst()
check iter1.isValid()
iter2.seekToLast()
check iter2.isValid()
check:
iter1.key() == @[byte(1)]
iter1.value() == @[byte(1)]
iter2.key() == @[byte(3)]
iter2.value() == @[byte(3)]
test "Invalid column family":
let res = db.openIterator("unknown")
check:
res.isErr()
res.error() == "rocksdb: unknown column family"
test "Empty column family":
let res = db.openIterator(CF_EMPTY)
check res.isOk()
var iter = res.get()
defer: iter.close()
iter.seekToFirst()
check not iter.isValid()
iter.seekToLast()
check not iter.isValid()
test "Test status":
let res = db.openIterator(CF_EMPTY)
check res.isOk()
var iter = res.get()
defer: iter.close()
check iter.status().isOk()
iter.seekToLast()
check iter.status().isOk()
test "Test pairs iterator":
let res = db.openIterator(CF_DEFAULT)
check res.isOk()
var iter = res.get()
var expected = byte(1)
for k, v in iter:
check:
k == @[expected]
v == @[expected]
inc expected
check iter.isClosed()
test "Test close":
let res = db.openIterator()
check res.isOk()
var iter = res.get()
check not iter.isClosed()
iter.close()
check iter.isClosed()
iter.close()
check iter.isClosed()

91
tests/test_sstfilewriter.nim Normal file
View File

@ -0,0 +1,91 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[rocksdb, sstfilewriter],
./test_helper
suite "SstFileWriterRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
let
key1 = @[byte(1)]
val1 = @[byte(1)]
key2 = @[byte(2)]
val2 = @[byte(2)]
key3 = @[byte(3)]
val3 = @[byte(3)]
setup:
let
dbPath = mkdtemp() / "data"
sstFilePath = mkdtemp() / "sst"
db = initReadWriteDb(dbPath,
columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
teardown:
db.close()
removeDir($dbPath)
test "Write to sst file then load into db using default column family":
let res = openSstFileWriter(sstFilePath)
check res.isOk()
let writer = res.get()
defer: writer.close()
check:
writer.put(key1, val1).isOk()
writer.put(key2, val2).isOk()
writer.put(key3, val3).isOk()
writer.delete(@[byte(4)]).isOk()
writer.finish().isOk()
db.ingestExternalFile(sstFilePath).isOk()
db.get(key1).get() == val1
db.get(key2).get() == val2
db.get(key3).get() == val3
test "Write to sst file then load into db using specific column family":
let res = openSstFileWriter(sstFilePath)
check res.isOk()
let writer = res.get()
defer: writer.close()
check:
writer.put(key1, val1).isOk()
writer.put(key2, val2).isOk()
writer.put(key3, val3).isOk()
writer.finish().isOk()
db.ingestExternalFile(sstFilePath, CF_OTHER).isOk()
db.keyExists(key1, CF_DEFAULT).get() == false
db.keyExists(key2, CF_DEFAULT).get() == false
db.keyExists(key3, CF_DEFAULT).get() == false
db.get(key1, CF_OTHER).get() == val1
db.get(key2, CF_OTHER).get() == val2
db.get(key3, CF_OTHER).get() == val3
test "Test close":
let res = openSstFileWriter(sstFilePath)
check res.isOk()
let writer = res.get()
check not writer.isClosed()
writer.close()
check writer.isClosed()
writer.close()
check writer.isClosed()

223
tests/test_transactiondb.nim Normal file
View File

@ -0,0 +1,223 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[transactiondb],
./test_helper
suite "TransactionDbRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
let
key1 = @[byte(1)]
val1 = @[byte(1)]
key2 = @[byte(2)]
val2 = @[byte(2)]
key3 = @[byte(3)]
val3 = @[byte(3)]
setup:
let dbPath = mkdtemp() / "data"
var db = initTransactionDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
teardown:
db.close()
removeDir($dbPath)
# test multiple transactions
test "Test rollback using default column family":
var tx = db.beginTransaction()
defer: tx.close()
check not tx.isClosed()
check:
tx.put(key1, val1).isOk()
tx.put(key2, val2).isOk()
tx.put(key3, val3).isOk()
tx.delete(key2).isOk()
not tx.isClosed()
check:
tx.get(key1).get() == val1
tx.get(key2).error() == ""
tx.get(key3).get() == val3
let res = tx.rollback()
check:
res.isOk()
tx.get(key1).error() == ""
tx.get(key2).error() == ""
tx.get(key3).error() == ""
test "Test commit using default column family":
var tx = db.beginTransaction()
defer: tx.close()
check not tx.isClosed()
check:
tx.put(key1, val1).isOk()
tx.put(key2, val2).isOk()
tx.put(key3, val3).isOk()
tx.delete(key2).isOk()
not tx.isClosed()
check:
tx.get(key1).get() == val1
tx.get(key2).error() == ""
tx.get(key3).get() == val3
let res = tx.commit()
check:
res.isOk()
tx.get(key1).get() == val1
tx.get(key2).error() == ""
tx.get(key3).get() == val3
test "Test setting column family in beginTransaction":
var tx = db.beginTransaction(columnFamily = CF_OTHER)
defer: tx.close()
check not tx.isClosed()
check:
tx.put(key1, val1).isOk()
tx.put(key2, val2).isOk()
tx.put(key3, val3).isOk()
tx.delete(key2).isOk()
not tx.isClosed()
check:
tx.get(key1, CF_DEFAULT).error() == ""
tx.get(key2, CF_DEFAULT).error() == ""
tx.get(key3, CF_DEFAULT).error() == ""
tx.get(key1, CF_OTHER).get() == val1
tx.get(key2, CF_OTHER).error() == ""
tx.get(key3, CF_OTHER).get() == val3
test "Test setting column family using withDefaultColFamily":
var tx = db.beginTransaction().withDefaultColFamily(CF_OTHER)
defer: tx.close()
check not tx.isClosed()
check:
tx.put(key1, val1).isOk()
tx.put(key2, val2).isOk()
tx.put(key3, val3).isOk()
tx.delete(key2).isOk()
not tx.isClosed()
check:
tx.get(key1, CF_DEFAULT).error() == ""
tx.get(key2, CF_DEFAULT).error() == ""
tx.get(key3, CF_DEFAULT).error() == ""
tx.get(key1, CF_OTHER).get() == val1
tx.get(key2, CF_OTHER).error() == ""
tx.get(key3, CF_OTHER).get() == val3
test "Test rollback and commit with multiple transactions":
var tx1 = db.beginTransaction(columnFamily = CF_DEFAULT)
defer: tx1.close()
check not tx1.isClosed()
var tx2 = db.beginTransaction(columnFamily = CF_OTHER)
defer: tx2.close()
check not tx2.isClosed()
check:
tx1.put(key1, val1).isOk()
tx1.put(key2, val2).isOk()
tx1.put(key3, val3).isOk()
tx1.delete(key2).isOk()
not tx1.isClosed()
tx2.put(key1, val1).isOk()
tx2.put(key2, val2).isOk()
tx2.put(key3, val3).isOk()
tx2.delete(key2).isOk()
not tx2.isClosed()
check:
tx1.get(key1, CF_DEFAULT).get() == val1
tx1.get(key2, CF_DEFAULT).error() == ""
tx1.get(key3, CF_DEFAULT).get() == val3
tx1.get(key1, CF_OTHER).error() == ""
tx1.get(key2, CF_OTHER).error() == ""
tx1.get(key3, CF_OTHER).error() == ""
tx2.get(key1, CF_DEFAULT).error() == ""
tx2.get(key2, CF_DEFAULT).error() == ""
tx2.get(key3, CF_DEFAULT).error() == ""
tx2.get(key1, CF_OTHER).get() == val1
tx2.get(key2, CF_OTHER).error() == ""
tx2.get(key3, CF_OTHER).get() == val3
block:
let res = tx1.rollback()
check:
res.isOk()
tx1.get(key1, CF_DEFAULT).error() == ""
tx1.get(key2, CF_DEFAULT).error() == ""
tx1.get(key3, CF_DEFAULT).error() == ""
tx1.get(key1, CF_OTHER).error() == ""
tx1.get(key2, CF_OTHER).error() == ""
tx1.get(key3, CF_OTHER).error() == ""
block:
let res = tx2.commit()
check:
res.isOk()
tx2.get(key1, CF_DEFAULT).error() == ""
tx2.get(key2, CF_DEFAULT).error() == ""
tx2.get(key3, CF_DEFAULT).error() == ""
tx2.get(key1, CF_OTHER).get() == val1
tx2.get(key2, CF_OTHER).error() == ""
tx2.get(key3, CF_OTHER).get() == val3
test "Test close":
var tx = db.beginTransaction()
check not tx.isClosed()
tx.close()
check tx.isClosed()
tx.close()
check tx.isClosed()
check not db.isClosed()
db.close()
check db.isClosed()
db.close()
check db.isClosed()
test "Test close multiple tx":
var tx1 = db.beginTransaction()
var tx2 = db.beginTransaction()
check not db.isClosed()
check not tx1.isClosed()
tx1.close()
check tx1.isClosed()
tx1.close()
check tx1.isClosed()
check not db.isClosed()
check not tx2.isClosed()
tx2.close()
check tx2.isClosed()
tx2.close()
check tx2.isClosed()

187
tests/test_writebatch.nim Normal file
View File

@ -0,0 +1,187 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[rocksdb, writebatch],
./test_helper
suite "WriteBatchRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
let
key1 = @[byte(1)]
val1 = @[byte(1)]
key2 = @[byte(2)]
val2 = @[byte(2)]
key3 = @[byte(3)]
val3 = @[byte(3)]
setup:
let dbPath = mkdtemp() / "data"
var db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
teardown:
db.close()
removeDir($dbPath)
test "Test writing batch to the default column family":
var batch = db.openWriteBatch()
defer: batch.close()
check not batch.isClosed()
check:
batch.put(key1, val1).isOk()
batch.put(key2, val2).isOk()
batch.put(key3, val3).isOk()
batch.count() == 3
batch.delete(key2).isOk()
batch.count() == 4
not batch.isClosed()
let res = db.write(batch)
check:
res.isOk()
db.write(batch).isOk() # test that it's idempotent
db.get(key1).get() == val1
db.keyExists(key2).get() == false
db.get(key3).get() == val3
batch.clear()
check:
batch.count() == 0
not batch.isClosed()
test "Test writing batch to column family":
var batch = db.openWriteBatch()
defer: batch.close()
check not batch.isClosed()
check:
batch.put(key1, val1, CF_OTHER).isOk()
batch.put(key2, val2, CF_OTHER).isOk()
batch.put(key3, val3, CF_OTHER).isOk()
batch.count() == 3
batch.delete(key2, CF_OTHER).isOk()
batch.count() == 4
not batch.isClosed()
let res = db.write(batch)
check:
res.isOk()
db.get(key1, CF_OTHER).get() == val1
db.keyExists(key2, CF_OTHER).get() == false
db.get(key3, CF_OTHER).get() == val3
batch.clear()
check:
batch.count() == 0
not batch.isClosed()
test "Test writing to multiple column families in single batch":
var batch = db.openWriteBatch()
defer: batch.close()
check not batch.isClosed()
check:
batch.put(key1, val1).isOk()
batch.put(key1, val1, CF_OTHER).isOk()
batch.put(key2, val2, CF_OTHER).isOk()
batch.put(key3, val3, CF_OTHER).isOk()
batch.count() == 4
batch.delete(key2, CF_OTHER).isOk()
batch.count() == 5
not batch.isClosed()
let res = db.write(batch)
check:
res.isOk()
db.get(key1).get() == val1
db.get(key1, CF_OTHER).get() == val1
db.keyExists(key2, CF_OTHER).get() == false
db.get(key3, CF_OTHER).get() == val3
batch.clear()
check:
batch.count() == 0
not batch.isClosed()
test "Test writing to multiple column families in multiple batches":
var batch1 = db.openWriteBatch()
defer: batch1.close()
check not batch1.isClosed()
var batch2 = db.openWriteBatch()
defer: batch2.close()
check not batch2.isClosed()
check:
batch1.put(key1, val1).isOk()
batch1.delete(key2, CF_OTHER).isOk()
batch1.put(key3, val3, CF_OTHER).isOk()
batch2.put(key1, val1, CF_OTHER).isOk()
batch2.delete(key1, CF_OTHER).isOk()
batch2.put(key3, val3).isOk()
batch1.count() == 3
batch2.count() == 3
let res1 = db.write(batch1)
let res2 = db.write(batch2)
check:
res1.isOk()
res2.isOk()
db.get(key1).get() == val1
db.keyExists(key2).get() == false
db.get(key3).get() == val3
db.keyExists(key1, CF_OTHER).get() == false
db.keyExists(key2, CF_OTHER).get() == false
db.get(key3, CF_OTHER).get() == val3
test "Test unknown column family":
const CF_UNKNOWN = "unknown"
var batch = db.openWriteBatch()
defer: batch.close()
check not batch.isClosed()
let r = batch.put(key1, val1, CF_UNKNOWN)
check r.isErr() and r.error() == "rocksdb: unknown column family"
let r2 = batch.delete(key1, CF_UNKNOWN)
check r2.isErr() and r2.error() == "rocksdb: unknown column family"
test "Test write empty batch":
var batch = db.openWriteBatch()
defer: batch.close()
check not batch.isClosed()
check batch.count() == 0
let res1 = db.write(batch)
check:
res1.isOk()
batch.count() == 0
test "Test close":
var batch = db.openWriteBatch()
check not batch.isClosed()
batch.close()
check batch.isClosed()
batch.close()
check batch.isClosed()

39
tests/transactions/test_txdbopts.nim Normal file
View File

@ -0,0 +1,39 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/transactions/txdbopts
suite "TransactionDbOptionsRef Tests":
test "Test newTransactionDbOptions":
var txDbOpts = newTransactionDbOptions()
check not txDbOpts.cPtr.isNil()
txDbOpts.close()
test "Test defaultTransactionDbOptions":
var txDbOpts = defaultTransactionDbOptions()
check not txDbOpts.cPtr.isNil()
txDbOpts.close()
test "Test close":
var txDbOpts = defaultTransactionDbOptions()
check not txDbOpts.isClosed()
txDbOpts.close()
check txDbOpts.isClosed()
txDbOpts.close()
check txDbOpts.isClosed()

39
tests/transactions/test_txopts.nim Normal file
View File

@ -0,0 +1,39 @@
# Nim-RocksDB
# Copyright 2024 Status Research & Development GmbH
# Licensed under either of
#
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
# * GPL license, version 2.0, ([LICENSE-GPLv2](LICENSE-GPLv2) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html)
#
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.used.}
import
unittest2,
../../rocksdb/transactions/txopts
suite "TransactionOptionsRef Tests":
test "Test newTransactionOptions":
var txOpts = newTransactionOptions()
check not txOpts.cPtr.isNil()
txOpts.close()
test "Test defaultTransactionOptions":
var txOpts = defaultTransactionOptions()
check not txOpts.cPtr.isNil()
txOpts.close()
test "Test close":
var txOpts = defaultTransactionOptions()
check not txOpts.isClosed()
txOpts.close()
check txOpts.isClosed()
txOpts.close()
check txOpts.isClosed()