status-im
/
nim-rocksdb
mirror of https://github.com/status-im/nim-rocksdb.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Format using nph (#54) 

* Format entire project using nph.

* Add nph lint to CI.
This commit is contained in:
web3-developer 2024-06-26 23:00:10 +08:00 committed by GitHub
parent 45f7a9266f
commit a691d5b9d2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
47 changed files with 5749 additions and 3072 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/ci.yml vendored
View File

@ -7,6 +7,26 @@ on:
workflow_dispatch:
jobs:
lint:
name: "nph Lint"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 2 # In PR, has extra merge commit: ^1 = PR, ^2 = base
- name: Check nph formatting
# Pin nph to a specific version to avoid sudden style differences.
# Updating nph version should be accompanied with running the new
# version on the project directory.
run: |
VERSION="v0.5.1"
ARCHIVE="nph-linux_x64.tar.gz"
curl -L "https://github.com/arnetheduck/nph/releases/download/${VERSION}/${ARCHIVE}" -o ${ARCHIVE}
tar -xzf ${ARCHIVE}
./nph .
git diff --exit-code
build:
strategy:
fail-fast: false

6
config.nims
View File

@ -23,5 +23,7 @@ when defined(rocksdb_static_linking):
switch("dynlibOverride", "lz4")
switch("dynlibOverride", "zstd")
--styleCheck:usages
--styleCheck:error
--styleCheck:
usages
--styleCheck:
error

32
examples/simple_example.nim
View File

@ -17,10 +17,10 @@ proc main() =
# snappy support (for example Fedora 28, certain Ubuntu versions)
# rocksdb_options_optimize_level_style_compaction(options, 0);
# create the DB if it's not already present
rocksdb_options_set_create_if_missing(options, 1);
rocksdb_options_set_create_if_missing(options, 1)
# open DB
var err: cstring # memory leak: example code does not free error string!
var # open DB
err: cstring # memory leak: example code does not free error string!
db = rocksdb_open(options, dbPath, cast[cstringArray](err.addr))
doAssert err.isNil, $err
@ -32,15 +32,28 @@ proc main() =
var writeOptions = rocksdb_writeoptions_create()
let key = "key"
let put_value = "value"
rocksdb_put(db, writeOptions, key.cstring, key.len.csize_t, put_value.cstring,
put_value.len.csize_t, cast[cstringArray](err.addr))
rocksdb_put(
db,
writeOptions,
key.cstring,
key.len.csize_t,
put_value.cstring,
put_value.len.csize_t,
cast[cstringArray](err.addr),
)
doAssert err.isNil, $err
# Get value
var readOptions = rocksdb_readoptions_create()
var len: csize_t
let raw_value = rocksdb_get(db, readOptions, key.cstring, key.len.csize_t, addr len,
cast[cstringArray](err.addr)) # Important: rocksdb_get is not null-terminated
let raw_value = rocksdb_get(
db,
readOptions,
key.cstring,
key.len.csize_t,
addr len,
cast[cstringArray](err.addr),
) # Important: rocksdb_get is not null-terminated
doAssert err.isNil, $err
# Copy it to a regular Nim string (copyMem workaround because raw value is NOT null-terminated)
@ -57,8 +70,9 @@ proc main() =
# If something is wrong, you might want to restore data from last backup
var restoreOptions = rocksdb_restore_options_create()
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbPath, dbPath,
restoreOptions, cast[cstringArray](err.addr))
rocksdb_backup_engine_restore_db_from_latest_backup(
be, dbPath, dbPath, restoreOptions, cast[cstringArray](err.addr)
)
doAssert err.isNil, $err
rocksdb_restore_options_destroy(restore_options)

19
rocksdb.nim
View File

@ -8,19 +8,10 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
./rocksdb/[backup,
columnfamily,
rocksdb,
rocksiterator,
sstfilewriter,
transactiondb,
writebatch]
./rocksdb/[
backup, columnfamily, rocksdb, rocksiterator, sstfilewriter, transactiondb,
writebatch,
]
export
backup,
columnfamily,
rocksdb,
rocksiterator,
sstfilewriter,
transactiondb,
writebatch
backup, columnfamily, rocksdb, rocksiterator, sstfilewriter, transactiondb, writebatch

15
rocksdb.nimble
View File

@ -1,16 +1,21 @@
packageName = "rocksdb"
version = "0.4.0"
author = "Status Research & Development GmbH"
description = "A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage"
description =
"A wrapper for Facebook's RocksDB, an embeddable, persistent key-value store for fast storage"
license = "Apache License 2.0 or GPLv2"
skipDirs = @["examples", "tests"]
mode = ScriptMode.Verbose
### Dependencies
requires "nim >= 1.6",
"results",
"tempfile",
"unittest2"
requires "nim >= 1.6", "results", "tempfile", "unittest2"
# Format only works with nim version 2
task format, "Format nim code using nph":
# Using the latest nph commit for now because the latest tagged version
# doesn't work with the latest nim 2 version
exec "nimble install nph@#head"
exec "nph ."
task clean, "Remove temporary files":
exec "rm -rf build"

44
rocksdb/backup.nim
View File

@ -12,16 +12,9 @@
{.push raises: [].}
import
./lib/librocksdb,
./internal/utils,
./options/backupopts,
./rocksdb,
./rocksresult
./lib/librocksdb, ./internal/utils, ./options/backupopts, ./rocksdb, ./rocksresult
export
backupopts,
rocksdb,
rocksresult
export backupopts, rocksdb, rocksresult
type
BackupEnginePtr* = ptr rocksdb_backup_engine_t
@ -32,23 +25,20 @@ type
backupOpts: BackupEngineOptionsRef
proc openBackupEngine*(
path: string,
backupOpts = defaultBackupEngineOptions()): RocksDBResult[BackupEngineRef] =
path: string, backupOpts = defaultBackupEngineOptions()
): RocksDBResult[BackupEngineRef] =
## Create a new backup engine. The `path` parameter is the path of the backup
## directory. Note that the same directory should not be used for both backups
## and the database itself.
var errors: cstring
let backupEnginePtr = rocksdb_backup_engine_open(
backupOpts.cPtr,
path.cstring,
cast[cstringArray](errors.addr))
backupOpts.cPtr, path.cstring, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
let engine = BackupEngineRef(
cPtr: backupEnginePtr,
path: path,
backupOpts: backupOpts)
let engine =
BackupEngineRef(cPtr: backupEnginePtr, path: path, backupOpts: backupOpts)
ok(engine)
proc isClosed*(backupEngine: BackupEngineRef): bool {.inline.} =
@ -56,26 +46,23 @@ proc isClosed*(backupEngine: BackupEngineRef): bool {.inline.} =
backupEngine.cPtr.isNil()
proc createNewBackup*(
backupEngine: BackupEngineRef,
db: RocksDbRef): RocksDBResult[void] =
backupEngine: BackupEngineRef, db: RocksDbRef
): RocksDBResult[void] =
## Create a new backup of the database.
doAssert not backupEngine.isClosed()
doAssert not db.isClosed()
var errors: cstring
rocksdb_backup_engine_create_new_backup(
backupEngine.cPtr,
db.cPtr,
cast[cstringArray](errors.addr))
backupEngine.cPtr, db.cPtr, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
ok()
proc restoreDbFromLatestBackup*(
backupEngine: BackupEngineRef,
dbDir: string,
walDir = dbDir,
keepLogFiles = false): RocksDBResult[void] =
backupEngine: BackupEngineRef, dbDir: string, walDir = dbDir, keepLogFiles = false
): RocksDBResult[void] =
## Restore the database from the latest backup.
doAssert not backupEngine.isClosed()
@ -88,7 +75,8 @@ proc restoreDbFromLatestBackup*(
dbDir.cstring,
walDir.cstring,
restoreOptions,
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
rocksdb_restore_options_destroy(restoreOptions)

39
rocksdb/columnfamily.nim
View File

@ -19,8 +19,7 @@
{.push raises: [].}
import
./rocksdb
import ./rocksdb
export rocksdb
@ -34,8 +33,8 @@ type
name: string
proc withColFamily*(
db: RocksDbReadOnlyRef,
name: string): RocksDBResult[ColFamilyReadOnly] =
db: RocksDbReadOnlyRef, name: string
): RocksDBResult[ColFamilyReadOnly] =
## Creates a new `ColFamilyReadOnly` from the given `RocksDbReadOnlyRef` and
## column family name.
@ -46,8 +45,8 @@ proc withColFamily*(
ok(ColFamilyReadOnly(db: db, name: name))
proc withColFamily*(
db: RocksDbReadWriteRef,
name: string): RocksDBResult[ColFamilyReadWrite] =
db: RocksDbReadWriteRef, name: string
): RocksDBResult[ColFamilyReadWrite] =
## Create a new `ColFamilyReadWrite` from the given `RocksDbReadWriteRef` and
## column family name.
@ -66,39 +65,39 @@ proc name*(cf: ColFamilyReadOnly | ColFamilyReadWrite): string {.inline.} =
cf.name
proc get*(
cf: ColFamilyReadOnly | ColFamilyReadWrite,
key: openArray[byte],
onData: DataProc): RocksDBResult[bool] {.inline.} =
cf: ColFamilyReadOnly | ColFamilyReadWrite, key: openArray[byte], onData: DataProc
): RocksDBResult[bool] {.inline.} =
## Gets the value of the given key from the column family using the `onData`
## callback.
cf.db.get(key, onData, cf.name)
proc get*(
cf: ColFamilyReadOnly | ColFamilyReadWrite,
key: openArray[byte]): RocksDBResult[seq[byte]] {.inline.} =
cf: ColFamilyReadOnly | ColFamilyReadWrite, key: openArray[byte]
): RocksDBResult[seq[byte]] {.inline.} =
## Gets the value of the given key from the column family.
cf.db.get(key, cf.name)
proc put*(
cf: ColFamilyReadWrite,
key, val: openArray[byte]): RocksDBResult[void] {.inline.} =
cf: ColFamilyReadWrite, key, val: openArray[byte]
): RocksDBResult[void] {.inline.} =
## Puts a value for the given key into the column family.
cf.db.put(key, val, cf.name)
proc keyExists*(
cf: ColFamilyReadOnly | ColFamilyReadWrite,
key: openArray[byte]): RocksDBResult[bool] {.inline.} =
cf: ColFamilyReadOnly | ColFamilyReadWrite, key: openArray[byte]
): RocksDBResult[bool] {.inline.} =
## Checks if the given key exists in the column family.
cf.db.keyExists(key, cf.name)
proc delete*(
cf: ColFamilyReadWrite,
key: openArray[byte]): RocksDBResult[void] {.inline.} =
cf: ColFamilyReadWrite, key: openArray[byte]
): RocksDBResult[void] {.inline.} =
## Deletes the given key from the column family.
cf.db.delete(key, cf.name)
proc openIterator*(
cf: ColFamilyReadOnly | ColFamilyReadWrite): RocksDBResult[RocksIteratorRef] {.inline.} =
cf: ColFamilyReadOnly | ColFamilyReadWrite
): RocksDBResult[RocksIteratorRef] {.inline.} =
## Opens an `RocksIteratorRef` for the given column family.
cf.db.openIterator(cf.name)
@ -107,7 +106,7 @@ proc openWriteBatch*(cf: ColFamilyReadWrite): WriteBatchRef {.inline.} =
cf.db.openWriteBatch(cf.name)
proc write*(
cf: ColFamilyReadWrite,
updates: WriteBatchRef): RocksDBResult[void] {.inline.} =
cf: ColFamilyReadWrite, updates: WriteBatchRef
): RocksDBResult[void] {.inline.} =
## Writes the updates in the `WriteBatchRef` to the column family.
cf.db.write(updates)

11
rocksdb/columnfamily/cfdescriptor.nim
View File

@ -9,20 +9,17 @@
{.push raises: [].}
import
../internal/utils,
./cfopts
import ../internal/utils, ./cfopts
export cfopts
type
ColFamilyDescriptor* = object
type ColFamilyDescriptor* = object
name: string
options: ColFamilyOptionsRef
proc initColFamilyDescriptor*(
name: string,
options = defaultColFamilyOptions()): ColFamilyDescriptor =
name: string, options = defaultColFamilyOptions()
): ColFamilyDescriptor =
ColFamilyDescriptor(name: name, options: options)
proc name*(descriptor: ColFamilyDescriptor): string {.inline.} =

3
rocksdb/columnfamily/cfhandle.nim
View File

@ -9,8 +9,7 @@
{.push raises: [].}
import
../lib/librocksdb
import ../lib/librocksdb
type
ColFamilyHandlePtr* = ptr rocksdb_column_family_handle_t

19
rocksdb/columnfamily/cfopts.nim
View File

@ -9,8 +9,7 @@
{.push raises: [].}
import
../lib/librocksdb, ../options/tableopts
import ../lib/librocksdb, ../options/tableopts
type
SlicetransformPtr* = ptr rocksdb_slicetransform_t
@ -127,21 +126,23 @@ proc `setPrefixExtractor`*(cfOpts: ColFamilyOptionsRef, value: SlicetransformRef
doAssert not cfOpts.isClosed()
rocksdb_options_set_prefix_extractor(cfOpts.cPtr, value.cPtr)
proc `blockBasedTableFactory=`*(cfOpts: ColFamilyOptionsRef, tableOpts: TableOptionsRef) =
proc `blockBasedTableFactory=`*(
cfOpts: ColFamilyOptionsRef, tableOpts: TableOptionsRef
) =
doAssert not cfOpts.isClosed()
rocksdb_options_set_block_based_table_factory(cfOpts.cPtr, tableOpts.cPtr)
# https://github.com/facebook/rocksdb/wiki/MemTable
proc setHashSkipListRep*(
cfOpts: ColFamilyOptionsRef, bucketCount, skipListHeight,
skipListBranchingFactor: int) =
cfOpts: ColFamilyOptionsRef,
bucketCount, skipListHeight, skipListBranchingFactor: int,
) =
doAssert not cfOpts.isClosed()
rocksdb_options_set_hash_skip_list_rep(
cfOpts.cPtr, bucketCount.csize_t, skipListHeight.cint,
skipListBranchingFactor.cint)
cfOpts.cPtr, bucketCount.csize_t, skipListHeight.cint, skipListBranchingFactor.cint
)
proc setHashLinkListRep*(
cfOpts: ColFamilyOptionsRef, bucketCount: int) =
proc setHashLinkListRep*(cfOpts: ColFamilyOptionsRef, bucketCount: int) =
doAssert not cfOpts.isClosed()
rocksdb_options_set_hash_link_list_rep(cfOpts.cPtr, bucketCount.csize_t)

14
rocksdb/internal/cftable.nim
View File

@ -9,20 +9,16 @@
{.push raises: [].}
import
std/tables,
../columnfamily/cfhandle
import std/tables, ../columnfamily/cfhandle
export
cfhandle
export cfhandle
type
ColFamilyTableRef* = ref object
type ColFamilyTableRef* = ref object
columnFamilies: TableRef[string, ColFamilyHandleRef]
proc newColFamilyTable*(
names: openArray[string],
handles: openArray[ColFamilyHandlePtr]): ColFamilyTableRef =
names: openArray[string], handles: openArray[ColFamilyHandlePtr]
): ColFamilyTableRef =
doAssert names.len() == handles.len()
let cfTable = newTable[string, ColFamilyHandleRef]()

4
rocksdb/internal/utils.nim
View File

@ -9,9 +9,7 @@
{.push raises: [].}
import
std/locks,
../lib/librocksdb
import std/locks, ../lib/librocksdb
const DEFAULT_COLUMN_FAMILY_NAME* = "default"

7430
rocksdb/lib/rocksdb_gen.nim
View File

File diff suppressed because it is too large Load Diff

4
rocksdb/options/backupopts.nim
View File

@ -9,8 +9,7 @@
{.push raises: [].}
import
../lib/librocksdb
import ../lib/librocksdb
type
BackupEngineOptionsPtr* = ptr rocksdb_options_t
@ -34,7 +33,6 @@ proc defaultBackupEngineOptions*(): BackupEngineOptionsRef {.inline.} =
let opts = newBackupEngineOptions()
opts
proc close*(engineOpts: BackupEngineOptionsRef) =
if not engineOpts.isClosed():
rocksdb_options_destroy(engineOpts.cPtr)

3
rocksdb/options/cache.nim
View File

@ -1,5 +1,4 @@
import
../lib/librocksdb
import ../lib/librocksdb
type
CachePtr* = ptr rocksdb_cache_t

5
rocksdb/options/dbopts.nim
View File

@ -9,10 +9,7 @@
{.push raises: [].}
import
std/cpuinfo,
../lib/librocksdb,
./[cache, tableopts]
import std/cpuinfo, ../lib/librocksdb, ./[cache, tableopts]
export cache, tableopts

3
rocksdb/options/readopts.nim
View File

@ -9,8 +9,7 @@
{.push raises: [].}
import
../lib/librocksdb
import ../lib/librocksdb
type
ReadOptionsPtr* = ptr rocksdb_readoptions_t

17
rocksdb/options/tableopts.nim
View File

@ -1,6 +1,4 @@
import
../lib/librocksdb,
./cache
import ../lib/librocksdb, ./cache
type
# TODO might eventually wrap this
@ -21,13 +19,18 @@ type
DataBlockIndexType* {.pure.} = enum
binarySearch = rocksdb_block_based_table_data_block_index_type_binary_search
binarySearchAndHash = rocksdb_block_based_table_data_block_index_type_binary_search_and_hash
binarySearchAndHash =
rocksdb_block_based_table_data_block_index_type_binary_search_and_hash
proc createRibbon*(bitsPerKey: float): FilterPolicyRef =
FilterPolicyRef(cPtr: rocksdb_filterpolicy_create_ribbon(bitsPerKey))
proc createRibbonHybrid*(bitsPerKey: float, bloomBeforeLevel: int = 0): FilterPolicyRef =
FilterPolicyRef(cPtr: rocksdb_filterpolicy_create_ribbon_hybrid(bitsPerKey, bloomBeforeLevel.cint))
proc createRibbonHybrid*(
bitsPerKey: float, bloomBeforeLevel: int = 0
): FilterPolicyRef =
FilterPolicyRef(
cPtr: rocksdb_filterpolicy_create_ribbon_hybrid(bitsPerKey, bloomBeforeLevel.cint)
)
proc isClosed*(policy: FilterPolicyRef): bool =
isNil(policy.cPtr)
@ -81,7 +84,7 @@ proc `filterPolicy=`*(opts: TableOptionsRef, policy: FilterPolicyRef) =
proc defaultTableOptions*(): TableOptionsRef =
# https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#other-general-options
let opts = createTableOptions()
opts.blockSize = 16*1024
opts.blockSize = 16 * 1024
opts.cacheIndexAndFilterBlocks = true
opts.pinL0FilterAndIndexBlocksInCache = true
opts

3
rocksdb/options/writeopts.nim
View File

@ -9,8 +9,7 @@
{.push raises: [].}
import
../lib/librocksdb
import ../lib/librocksdb
type
WriteOptionsPtr* = ptr rocksdb_writeoptions_t

166
rocksdb/rocksdb.nim
View File

@ -35,14 +35,7 @@ import
./rocksresult,
./writebatch
export
rocksresult,
dbopts,
readopts,
writeopts,
cfdescriptor,
rocksiterator,
writebatch
export rocksresult, dbopts, readopts, writeopts, cfdescriptor, rocksiterator, writebatch
type
RocksDbPtr* = ptr rocksdb_t
@ -64,9 +57,8 @@ type
ingestOptsPtr: IngestExternalFilesOptionsPtr
proc listColumnFamilies*(
path: string;
dbOpts = DbOptionsRef(nil);
): RocksDBResult[seq[string]] =
path: string, dbOpts = DbOptionsRef(nil)
): RocksDBResult[seq[string]] =
## List exisiting column families on disk. This might be used to find out
## whether there were some columns missing with the version on disk.
##
@ -78,30 +70,28 @@ proc listColumnFamilies*(
## above once rocksdb has been upgraded to the latest version, see comments
## at the end of ./columnfamily/cfhandle.nim.
##
let
useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil: useDbOpts.close()
if dbOpts.isNil:
useDbOpts.close()
var
lencf: csize_t
errors: cstring
let
cList = rocksdb_list_column_families(
useDbOpts.cPtr,
path.cstring,
addr lencf,
cast[cstringArray](errors.addr))
let cList = rocksdb_list_column_families(
useDbOpts.cPtr, path.cstring, addr lencf, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
var cfs: seq[string]
if not cList.isNil:
defer: rocksdb_free(cList)
defer:
rocksdb_free(cList)
for n in 0 ..< lencf:
if cList[n].isNil:
# Clean up the rest
for z in n+1 ..< lencf:
for z in n + 1 ..< lencf:
if not cList[z].isNil:
rocksdb_free(cList[z])
return err("short reply")
@ -112,22 +102,22 @@ proc listColumnFamilies*(
ok cfs
proc openRocksDb*(
path: string;
dbOpts = DbOptionsRef(nil);
readOpts = ReadOptionsRef(nil);
writeOpts = WriteOptionsRef(nil);
columnFamilies: openArray[ColFamilyDescriptor] = [];
): RocksDBResult[RocksDbReadWriteRef] =
path: string,
dbOpts = DbOptionsRef(nil),
readOpts = ReadOptionsRef(nil),
writeOpts = WriteOptionsRef(nil),
columnFamilies: openArray[ColFamilyDescriptor] = [],
): RocksDBResult[RocksDbReadWriteRef] =
## Open a RocksDB instance in read-write mode. If `columnFamilies` is empty
## then it will open the default column family. If `dbOpts`, `readOpts`, or
## `writeOpts` are not supplied then the default options will be used.
## By default, column families will be created if they don't yet exist.
## All existing column families must be specified if the database has
## previously created any column families.
let
useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil: useDbOpts.close()
if dbOpts.isNil:
useDbOpts.close()
var cfs = columnFamilies.toSeq()
if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
@ -145,7 +135,8 @@ proc openRocksDb*(
cast[cstringArray](cfNames[0].addr),
cfOpts[0].addr,
cfHandles[0].addr,
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
let
@ -161,26 +152,27 @@ proc openRocksDb*(
writeOpts: writeOpts,
ingestOptsPtr: rocksdb_ingestexternalfileoptions_create(),
defaultCfName: DEFAULT_COLUMN_FAMILY_NAME,
cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles))
cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles),
)
ok(db)
proc openRocksDbReadOnly*(
path: string;
dbOpts = DbOptionsRef(nil);
readOpts = ReadOptionsRef(nil);
columnFamilies: openArray[ColFamilyDescriptor] = [];
errorIfWalFileExists = false;
): RocksDBResult[RocksDbReadOnlyRef] =
path: string,
dbOpts = DbOptionsRef(nil),
readOpts = ReadOptionsRef(nil),
columnFamilies: openArray[ColFamilyDescriptor] = [],
errorIfWalFileExists = false,
): RocksDBResult[RocksDbReadOnlyRef] =
## Open a RocksDB instance in read-only mode. If `columnFamilies` is empty
## then it will open the default column family. If `dbOpts` or `readOpts` are
## not supplied then the default options will be used. By default, column
## families will be created if they don't yet exist. If the database already
## contains any column families, then all or a subset of the existing column
## families can be opened for reading.
let
useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil: useDbOpts.close()
if dbOpts.isNil:
useDbOpts.close()
var cfs = columnFamilies.toSeq()
if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
@ -199,7 +191,8 @@ proc openRocksDbReadOnly*(
cfOpts[0].addr,
cfHandles[0].addr,
errorIfWalFileExists.uint8,
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
let
@ -212,7 +205,8 @@ proc openRocksDbReadOnly*(
dbOpts: dbOpts,
readOpts: readOpts,
defaultCfName: DEFAULT_COLUMN_FAMILY_NAME,
cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles))
cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles),
)
ok(db)
proc isClosed*(db: RocksDbRef): bool {.inline.} =
@ -228,7 +222,8 @@ proc get*(
db: RocksDbRef,
key: openArray[byte],
onData: DataProc,
columnFamily = db.defaultCfName): RocksDBResult[bool] =
columnFamily = db.defaultCfName,
): RocksDBResult[bool] =
## Get the value for the given key from the specified column family.
## If the value does not exist, `false` will be returned in the result
## and `onData` will not be called. If the value does exist, `true` will be
@ -253,7 +248,8 @@ proc get*(
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
len.addr,
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
if data.isNil():
@ -265,15 +261,15 @@ proc get*(
ok(true)
proc get*(
db: RocksDbRef,
key: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[seq[byte]] =
db: RocksDbRef, key: openArray[byte], columnFamily = db.defaultCfName
): RocksDBResult[seq[byte]] =
## Get the value for the given key from the specified column family.
## If the value does not exist, an empty error will be returned in the result.
## If the value does exist, the value will be returned in the result.
var dataRes: RocksDBResult[seq[byte]]
proc onData(data: openArray[byte]) = dataRes.ok(@data)
proc onData(data: openArray[byte]) =
dataRes.ok(@data)
let res = db.get(key, onData, columnFamily)
if res.isOk():
@ -282,9 +278,8 @@ proc get*(
dataRes.err(res.error())
proc put*(
db: RocksDbReadWriteRef,
key, val: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[void] =
db: RocksDbReadWriteRef, key, val: openArray[byte], columnFamily = db.defaultCfName
): RocksDBResult[void] =
## Put the value for the given key into the specified column family.
if key.len() == 0:
@ -301,17 +296,21 @@ proc put*(
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
cast[cstring](if val.len > 0:
unsafeAddr val[0]
else:
nil
),
csize_t(val.len),
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
ok()
proc keyExists*(
db: RocksDbRef,
key: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[bool] =
db: RocksDbRef, key: openArray[byte], columnFamily = db.defaultCfName
): RocksDBResult[bool] =
## Check if the key exists in the specified column family.
## Returns a result containing `true` if the key exists or a result
## containing `false` otherwise.
@ -319,12 +318,17 @@ proc keyExists*(
# TODO: Call rocksdb_key_may_exist_cf to improve performance for the case
# when the key does not exist
db.get(key, proc(data: openArray[byte]) = discard, columnFamily)
db.get(
key,
proc(data: openArray[byte]) =
discard
,
columnFamily,
)
proc delete*(
db: RocksDbReadWriteRef,
key: openArray[byte],
columnFamily = db.defaultCfName): RocksDBResult[void] =
db: RocksDbReadWriteRef, key: openArray[byte], columnFamily = db.defaultCfName
): RocksDBResult[void] =
## Delete the value for the given key from the specified column family.
## If the value does not exist, the delete will be a no-op.
## To check if the value exists before or after a delete, use `keyExists`.
@ -343,14 +347,15 @@ proc delete*(
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
ok()
proc openIterator*(
db: RocksDbRef,
columnFamily = db.defaultCfName): RocksDBResult[RocksIteratorRef] =
db: RocksDbRef, columnFamily = db.defaultCfName
): RocksDBResult[RocksIteratorRef] =
## Opens an `RocksIteratorRef` for the specified column family.
doAssert not db.isClosed()
@ -358,41 +363,34 @@ proc openIterator*(
if cfHandle.isNil():
return err("rocksdb: unknown column family")
let rocksIterPtr = rocksdb_create_iterator_cf(
db.cPtr,
db.readOpts.cPtr,
cfHandle.cPtr)
let rocksIterPtr =
rocksdb_create_iterator_cf(db.cPtr, db.readOpts.cPtr, cfHandle.cPtr)
ok(newRocksIterator(rocksIterPtr))
proc openWriteBatch*(
db: RocksDbReadWriteRef,
columnFamily = db.defaultCfName): WriteBatchRef =
db: RocksDbReadWriteRef, columnFamily = db.defaultCfName
): WriteBatchRef =
## Opens a `WriteBatchRef` which defaults to using the specified column family.
doAssert not db.isClosed()
newWriteBatch(db.cfTable, columnFamily)
proc write*(
db: RocksDbReadWriteRef,
updates: WriteBatchRef): RocksDBResult[void] =
proc write*(db: RocksDbReadWriteRef, updates: WriteBatchRef): RocksDBResult[void] =
## Apply the updates in the `WriteBatchRef` to the database.
doAssert not db.isClosed()
var errors: cstring
rocksdb_write(
db.cPtr,
db.writeOpts.cPtr,
updates.cPtr,
cast[cstringArray](errors.addr))
db.cPtr, db.writeOpts.cPtr, updates.cPtr, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
ok()
proc ingestExternalFile*(
db: RocksDbReadWriteRef,
filePath: string,
columnFamily = db.defaultCfName): RocksDBResult[void] =
db: RocksDbReadWriteRef, filePath: string, columnFamily = db.defaultCfName
): RocksDBResult[void] =
## Ingest an external sst file into the database. The file will be ingested
## into the specified column family or the default column family if none is
## provided.
@ -408,9 +406,11 @@ proc ingestExternalFile*(
rocksdb_ingest_external_file_cf(
db.cPtr,
cfHandle.cPtr,
cast[cstringArray](sstPath.addr), csize_t(1),
cast[cstringArray](sstPath.addr),
csize_t(1),
db.ingestOptsPtr,
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
ok()

21
rocksdb/rocksiterator.nim
View File

@ -12,13 +12,9 @@
{.push raises: [].}
import
./lib/librocksdb,
./internal/utils,
./rocksresult
import ./lib/librocksdb, ./internal/utils, ./rocksresult
export
rocksresult
export rocksresult
type
RocksIteratorPtr* = ptr rocksdb_iterator_t
@ -138,15 +134,22 @@ iterator pairs*(iter: RocksIteratorRef): tuple[key: seq[byte], value: seq[byte]]
## the form of a tuple. The iterator is automatically closed after the
## iteration.
doAssert not iter.isClosed()
defer: iter.close()
defer:
iter.close()
iter.seekToFirst()
while iter.isValid():
var
key: seq[byte]
value: seq[byte]
iter.key(proc(data: openArray[byte]) = key = @data)
iter.value(proc(data: openArray[byte]) = value = @data)
iter.key(
proc(data: openArray[byte]) =
key = @data
)
iter.value(
proc(data: openArray[byte]) =
value = @data
)
iter.next()
yield (key, value)

6
rocksdb/rocksresult.nim
View File

@ -9,11 +9,9 @@
{.push raises: [].}
import
results
import results
export
results
export results
type
RocksDBResult*[T] = Result[T, string]

42
rocksdb/sstfilewriter.nim
View File

@ -11,14 +11,9 @@
{.push raises: [].}
import
./lib/librocksdb,
./internal/utils,
./options/dbopts,
./rocksresult
import ./lib/librocksdb, ./internal/utils, ./options/dbopts, ./rocksresult
export
rocksresult
export rocksresult
type
SstFileWriterPtr* = ptr rocksdb_sstfilewriter_t
@ -30,9 +25,8 @@ type
dbOpts: DbOptionsRef
proc openSstFileWriter*(
filePath: string;
dbOpts = DbOptionsRef(nil);
): RocksDBResult[SstFileWriterRef] =
filePath: string, dbOpts = DbOptionsRef(nil)
): RocksDBResult[SstFileWriterRef] =
## Creates a new `SstFileWriterRef` and opens the file at the given `filePath`.
let dbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
doAssert not dbOpts.isClosed()
@ -41,13 +35,13 @@ proc openSstFileWriter*(
let writer = SstFileWriterRef(
cPtr: rocksdb_sstfilewriter_create(envOptsPtr, dbOpts.cPtr),
envOptsPtr: envOptsPtr,
dbOpts: dbOpts)
dbOpts: dbOpts,
)
var errors: cstring
rocksdb_sstfilewriter_open(
writer.cPtr,
filePath.cstring,
cast[cstringArray](errors.addr))
writer.cPtr, filePath.cstring, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
ok(writer)
@ -57,17 +51,19 @@ proc isClosed*(writer: SstFileWriterRef): bool {.inline.} =
writer.cPtr.isNil()
proc put*(
writer: SstFileWriterRef,
key: openArray[byte],
val: openArray[byte]): RocksDBResult[void] =
writer: SstFileWriterRef, key: openArray[byte], val: openArray[byte]
): RocksDBResult[void] =
## Add a key-value pair to the sst file.
var errors: cstring
rocksdb_sstfilewriter_put(
writer.cPtr,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
cast[cstring](unsafeAddr val[0]), csize_t(val.len),
cast[cstringArray](errors.addr))
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstring](unsafeAddr val[0]),
csize_t(val.len),
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
ok()
@ -78,8 +74,10 @@ proc delete*(writer: SstFileWriterRef, key: openArray[byte]): RocksDBResult[void
var errors: cstring
rocksdb_sstfilewriter_delete(
writer.cPtr,
cast[cstring](unsafeAddr key[0]), csize_t(key.len),
cast[cstringArray](errors.addr))
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
ok()

57
rocksdb/transactiondb.nim
View File

@ -25,14 +25,7 @@ import
./rocksresult
export
dbopts,
txdbopts,
cfdescriptor,
readopts,
writeopts,
txopts,
transaction,
rocksresult
dbopts, txdbopts, cfdescriptor, readopts, writeopts, txopts, transaction, rocksresult
type
TransactionDbPtr* = ptr rocksdb_transactiondb_t
@ -47,17 +40,17 @@ type
proc openTransactionDb*(
path: string,
dbOpts = DbOptionsRef(nil);
txDbOpts = TransactionDbOptionsRef(nil);
columnFamilies: openArray[ColFamilyDescriptor] = [];
): RocksDBResult[TransactionDbRef] =
dbOpts = DbOptionsRef(nil),
txDbOpts = TransactionDbOptionsRef(nil),
columnFamilies: openArray[ColFamilyDescriptor] = [],
): RocksDBResult[TransactionDbRef] =
## Open a `TransactionDbRef` with the given options and column families.
## If no column families are provided the default column family will be used.
## If no options are provided the default options will be used.
let
useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil: useDbOpts.close()
if dbOpts.isNil:
useDbOpts.close()
var cfs = columnFamilies.toSeq()
if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
@ -77,19 +70,23 @@ proc openTransactionDb*(
cast[cstringArray](cfNames[0].addr),
cfOpts[0].addr,
cfHandles[0].addr,
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
let
dbOpts = useDbOpts # don't close on exit
txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() else: txDbOpts)
txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions()
else: txDbOpts
)
db = TransactionDbRef(
lock: createLock(),
cPtr: txDbPtr,
path: path,
dbOpts: dbOpts,
txDbOpts: txDbOpts,
cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles))
cfTable: newColFamilyTable(cfNames.mapIt($it), cfHandles),
)
ok(db)
proc isClosed*(db: TransactionDbRef): bool {.inline.} =
@ -97,27 +94,25 @@ proc isClosed*(db: TransactionDbRef): bool {.inline.} =
db.cPtr.isNil()
proc beginTransaction*(
db: TransactionDbRef;
readOpts = ReadOptionsRef(nil);
writeOpts = WriteOptionsRef(nil);
txDbOpts = TransactionDbOptionsRef(nil);
txOpts = defaultTransactionOptions();
columnFamily = DEFAULT_COLUMN_FAMILY_NAME;
): TransactionRef =
db: TransactionDbRef,
readOpts = ReadOptionsRef(nil),
writeOpts = WriteOptionsRef(nil),
txDbOpts = TransactionDbOptionsRef(nil),
txOpts = defaultTransactionOptions(),
columnFamily = DEFAULT_COLUMN_FAMILY_NAME,
): TransactionRef =
## Begin a new transaction against the database. The transaction will default
## to using the specified column family. If no column family is specified
## then the default column family will be used.
doAssert not db.isClosed()
let
txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() else: txDbOpts)
txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions()
else: txDbOpts
)
readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts)
writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts)
let txPtr = rocksdb_transaction_begin(
db.cPtr,
writeOpts.cPtr,
txOpts.cPtr,
nil)
let txPtr = rocksdb_transaction_begin(db.cPtr, writeOpts.cPtr, txOpts.cPtr, nil)
newTransaction(txPtr, readOpts, writeOpts, txOpts, columnFamily, db.cfTable)

43
rocksdb/transactions/transaction.nim
View File

@ -26,8 +26,7 @@ import
../rocksresult,
./txopts
export
rocksresult
export rocksresult
type
TransactionPtr* = ptr rocksdb_transaction_t
@ -46,15 +45,16 @@ proc newTransaction*(
writeOpts: WriteOptionsRef,
txOpts: TransactionOptionsRef,
defaultCfName: string,
cfTable: ColFamilyTableRef): TransactionRef =
cfTable: ColFamilyTableRef,
): TransactionRef =
TransactionRef(
cPtr: cPtr,
readOpts: readOpts,
writeOpts: writeOpts,
txOpts: txOpts,
defaultCfName: defaultCfName,
cfTable: cfTable)
cfTable: cfTable,
)
proc isClosed*(tx: TransactionRef): bool {.inline.} =
## Returns `true` if the `TransactionRef` has been closed.
@ -64,7 +64,8 @@ proc get*(
tx: TransactionRef,
key: openArray[byte],
onData: DataProc,
columnFamily = tx.defaultCfName): RocksDBResult[bool] =
columnFamily = tx.defaultCfName,
): RocksDBResult[bool] =
## Get the value for a given key from the transaction using the provided
## `onData` callback.
@ -85,7 +86,8 @@ proc get*(
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
len.addr,
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
if data.isNil():
@ -97,9 +99,8 @@ proc get*(
ok(true)
proc get*(
tx: TransactionRef,
key: openArray[byte],
columnFamily = tx.defaultCfName): RocksDBResult[seq[byte]] =
tx: TransactionRef, key: openArray[byte], columnFamily = tx.defaultCfName
): RocksDBResult[seq[byte]] =
## Get the value for a given key from the transaction.
var dataRes: RocksDBResult[seq[byte]]
@ -113,9 +114,8 @@ proc get*(
dataRes.err(res.error())
proc put*(
tx: TransactionRef,
key, val: openArray[byte],
columnFamily = tx.defaultCfName): RocksDBResult[void] =
tx: TransactionRef, key, val: openArray[byte], columnFamily = tx.defaultCfName
): RocksDBResult[void] =
## Put the value for the given key into the transaction.
if key.len() == 0:
@ -131,17 +131,21 @@ proc put*(
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
cast[cstring](if val.len > 0:
unsafeAddr val[0]
else:
nil
),
csize_t(val.len),
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
ok()
proc delete*(
tx: TransactionRef,
key: openArray[byte],
columnFamily = tx.defaultCfName): RocksDBResult[void] =
tx: TransactionRef, key: openArray[byte], columnFamily = tx.defaultCfName
): RocksDBResult[void] =
## Delete the value for the given key from the transaction.
if key.len() == 0:
@ -157,7 +161,8 @@ proc delete*(
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstringArray](errors.addr))
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
ok()

3
rocksdb/transactions/txdbopts.nim
View File

@ -9,8 +9,7 @@
{.push raises: [].}
import
../lib/librocksdb
import ../lib/librocksdb
type
TransactionDbOptionsPtr* = ptr rocksdb_transactiondb_options_t

3
rocksdb/transactions/txopts.nim
View File

@ -9,8 +9,7 @@
{.push raises: [].}
import
../lib/librocksdb
import ../lib/librocksdb
type
TransactionOptionsPtr* = ptr rocksdb_transaction_options_t

34
rocksdb/writebatch.nim
View File

@ -11,13 +11,9 @@
{.push raises: [].}
import
./lib/librocksdb,
./internal/[cftable, utils],
./rocksresult
import ./lib/librocksdb, ./internal/[cftable, utils], ./rocksresult
export
rocksresult
export rocksresult
type
WriteBatchPtr* = ptr rocksdb_writebatch_t
@ -29,9 +25,8 @@ type
proc newWriteBatch*(cfTable: ColFamilyTableRef, defaultCfName: string): WriteBatchRef =
WriteBatchRef(
cPtr: rocksdb_writebatch_create(),
defaultCfName: defaultCfName,
cfTable: cfTable)
cPtr: rocksdb_writebatch_create(), defaultCfName: defaultCfName, cfTable: cfTable
)
proc isClosed*(batch: WriteBatchRef): bool {.inline.} =
## Returns `true` if the `WriteBatchRef` has been closed and `false` otherwise.
@ -55,7 +50,8 @@ proc count*(batch: WriteBatchRef): int =
proc put*(
batch: WriteBatchRef,
key, val: openArray[byte],
columnFamily = DEFAULT_COLUMN_FAMILY_NAME): RocksDBResult[void] =
columnFamily = DEFAULT_COLUMN_FAMILY_NAME,
): RocksDBResult[void] =
## Add a put operation to the write batch.
if key.len() == 0:
@ -70,15 +66,21 @@ proc put*(
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len),
cast[cstring](if val.len > 0: unsafeAddr val[0] else: nil),
csize_t(val.len))
cast[cstring](if val.len > 0:
unsafeAddr val[0]
else:
nil
),
csize_t(val.len),
)
ok()
proc delete*(
batch: WriteBatchRef,
key: openArray[byte],
columnFamily = DEFAULT_COLUMN_FAMILY_NAME): RocksDBResult[void] =
columnFamily = DEFAULT_COLUMN_FAMILY_NAME,
): RocksDBResult[void] =
## Add a delete operation to the write batch.
if key.len() == 0:
@ -89,10 +91,8 @@ proc delete*(
return err("rocksdb: unknown column family")
rocksdb_writebatch_delete_cf(
batch.cPtr,
cfHandle.cPtr,
cast[cstring](unsafeAddr key[0]),
csize_t(key.len))
batch.cPtr, cfHandle.cPtr, cast[cstring](unsafeAddr key[0]), csize_t(key.len)
)
ok()

7
tests/columnfamily/test_cfdescriptor.nim
View File

@ -9,13 +9,9 @@
{.used.}
import
unittest2,
../../rocksdb/internal/utils,
../../rocksdb/columnfamily/cfdescriptor
import unittest2, ../../rocksdb/internal/utils, ../../rocksdb/columnfamily/cfdescriptor
suite "ColFamilyDescriptor Tests":
const TEST_CF_NAME = "test"
test "Test initColFamilyDescriptor":
@ -56,4 +52,3 @@ suite "ColFamilyDescriptor Tests":
check descriptor.isClosed()
descriptor.close()
check descriptor.isClosed()

13
tests/columnfamily/test_cfhandle.nim
View File

@ -17,7 +17,6 @@ import
../../rocksdb/columnfamily/cfhandle
suite "ColFamilyHandleRef Tests":
const TEST_CF_NAME = "test"
setup:
@ -26,20 +25,17 @@ suite "ColFamilyHandleRef Tests":
dbOpts = rocksdb_options_create()
cfOpts = rocksdb_options_create()
var
errors: cstring
var errors: cstring
rocksdb_options_set_create_if_missing(dbOpts, 1);
rocksdb_options_set_create_if_missing(dbOpts, 1)
let db = rocksdb_open(dbOpts, dbPath.cstring, cast[cstringArray](errors.addr))
doAssert errors.isNil()
doAssert not db.isNil()
let cfHandlePtr = rocksdb_create_column_family(
db,
cfOpts,
TEST_CF_NAME.cstring,
cast[cstringArray](errors.addr))
db, cfOpts, TEST_CF_NAME.cstring, cast[cstringArray](errors.addr)
)
doAssert errors.isNil()
doAssert not cfHandlePtr.isNil()
@ -64,4 +60,3 @@ suite "ColFamilyHandleRef Tests":
check cfHandle.isClosed()
cfHandle.close()
check cfHandle.isClosed()

4
tests/columnfamily/test_cfopts.nim
View File

@ -9,9 +9,7 @@
{.used.}
import
unittest2,
../../rocksdb/columnfamily/cfopts
import unittest2, ../../rocksdb/columnfamily/cfopts
suite "ColFamilyOptionsRef Tests":
test "Test close":

18
tests/internal/test_cftable.nim
View File

@ -18,7 +18,6 @@ import
../../rocksdb/internal/cftable
suite "ColFamilyTableRef Tests":
const TEST_CF_NAME = "test"
setup:
@ -27,20 +26,17 @@ suite "ColFamilyTableRef Tests":
dbOpts = rocksdb_options_create()
cfOpts = rocksdb_options_create()
var
errors: cstring
var errors: cstring
rocksdb_options_set_create_if_missing(dbOpts, 1);
rocksdb_options_set_create_if_missing(dbOpts, 1)
let db = rocksdb_open(dbOpts, dbPath.cstring, cast[cstringArray](errors.addr))
doAssert errors.isNil()
doAssert not db.isNil()
let cfHandlePtr = rocksdb_create_column_family(
db,
cfOpts,
TEST_CF_NAME.cstring,
cast[cstringArray](errors.addr))
db, cfOpts, TEST_CF_NAME.cstring, cast[cstringArray](errors.addr)
)
doAssert errors.isNil()
doAssert not cfHandlePtr.isNil()
@ -48,11 +44,9 @@ suite "ColFamilyTableRef Tests":
rocksdb_close(db)
removeDir($dbPath)
test "Test newColFamilyTable":
var cfTable = newColFamilyTable(
@[TEST_CF_NAME, TEST_CF_NAME],
@[cfHandlePtr, cfHandlePtr])
var cfTable =
newColFamilyTable(@[TEST_CF_NAME, TEST_CF_NAME], @[cfHandlePtr, cfHandlePtr])
check cfTable.get(TEST_CF_NAME).cPtr() == cfHandlePtr
check not cfTable.isClosed()

56
tests/lib/test_librocksdb.nim
View File

@ -9,11 +9,7 @@
{.used.}
import
std/[cpuinfo, os],
tempfile,
unittest2,
../../rocksdb/lib/librocksdb
import std/[cpuinfo, os], tempfile, unittest2, ../../rocksdb/lib/librocksdb
suite "librocksdb C wrapper Tests":
setup:
@ -37,54 +33,70 @@ suite "librocksdb C wrapper Tests":
# snappy support (for example Fedora 28, certain Ubuntu versions)
# rocksdb_options_optimize_level_style_compaction(options, 0);
# create the DB if it's not already present
rocksdb_options_set_create_if_missing(options, 1);
rocksdb_options_set_create_if_missing(options, 1)
# open DB
var err: cstringArray # memory leak: example code does not free error string!
var # open DB
err: cstringArray # memory leak: example code does not free error string!
db = rocksdb_open(options, dbPath, err)
check: err.isNil
check:
err.isNil
# open Backup Engine that we will use for backing up our database
be = rocksdb_backup_engine_open(options, dbBackupPath, err)
check: err.isNil
check:
err.isNil
# Put key-value
var writeOptions = rocksdb_writeoptions_create()
let key = "key"
let put_value = "value"
rocksdb_put(
db, writeOptions, key.cstring, csize_t(key.len),
put_value.cstring, csize_t(put_value.len), err)
check: err.isNil
db,
writeOptions,
key.cstring,
csize_t(key.len),
put_value.cstring,
csize_t(put_value.len),
err,
)
check:
err.isNil
# Get value
var readOptions = rocksdb_readoptions_create()
var len: csize_t
let raw_value = rocksdb_get(
db, readOptions, key.cstring, csize_t(key.len), addr len, err) # Important: rocksdb_get is not null-terminated
check: err.isNil
let raw_value =
rocksdb_get(db, readOptions, key.cstring, csize_t(key.len), addr len, err)
# Important: rocksdb_get is not null-terminated
check:
err.isNil
# Copy it to a regular Nim string (copyMem workaround because non-null terminated)
var get_value = newString(int(len))
copyMem(addr get_value[0], unsafeAddr raw_value[0], int(len) * sizeof(char))
check: $get_value == $put_value
check:
$get_value == $put_value
# create new backup in a directory specified by DBBackupPath
rocksdb_backup_engine_create_new_backup(be, db, err)
check: err.isNil
check:
err.isNil
rocksdb_close(db)
# If something is wrong, you might want to restore data from last backup
var restoreOptions = rocksdb_restore_options_create()
rocksdb_backup_engine_restore_db_from_latest_backup(be, dbPath, dbPath,
restoreOptions, err)
check: err.isNil
rocksdb_backup_engine_restore_db_from_latest_backup(
be, dbPath, dbPath, restoreOptions, err
)
check:
err.isNil
rocksdb_restore_options_destroy(restoreOptions)
db = rocksdb_open(options, dbPath, err)
check: err.isNil
check:
err.isNil
# cleanup
rocksdb_writeoptions_destroy(writeOptions)

5
tests/options/test_backupopts.nim
View File

@ -9,12 +9,9 @@
{.used.}
import
unittest2,
../../rocksdb/options/backupopts
import unittest2, ../../rocksdb/options/backupopts
suite "BackupEngineOptionsRef Tests":
test "Test newBackupEngineOptions":
var backupOpts = newBackupEngineOptions()

5
tests/options/test_dbopts.nim
View File

@ -9,12 +9,9 @@
{.used.}
import
unittest2,
../../rocksdb/options/dbopts
import unittest2, ../../rocksdb/options/dbopts
suite "DbOptionsRef Tests":
test "Test newDbOptions":
var dbOpts = newDbOptions()

5
tests/options/test_readopts.nim
View File

@ -9,12 +9,9 @@
{.used.}
import
unittest2,
../../rocksdb/options/readopts
import unittest2, ../../rocksdb/options/readopts
suite "ReadOptionsRef Tests":
test "Test newReadOptions":
var readOpts = newReadOptions()

5
tests/options/test_writeopts.nim
View File

@ -9,12 +9,9 @@
{.used.}
import
unittest2,
../../rocksdb/options/writeopts
import unittest2, ../../rocksdb/options/writeopts
suite "WriteOptionsRef Tests":
test "Test newWriteOptions":
var writeOpts = newWriteOptions()

13
tests/test_backup.nim
View File

@ -9,15 +9,9 @@
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/backup,
./test_helper
import std/os, tempfile, unittest2, ../rocksdb/backup, ./test_helper
suite "BackupEngineRef Tests":
let
key = @[byte(1), 2, 3, 4, 5]
val = @[byte(1), 2, 3, 4, 5]
@ -28,16 +22,13 @@ suite "BackupEngineRef Tests":
dbBackupPath = mkdtemp() / "backup"
dbRestorePath = mkdtemp() / "restore"
var
db = initReadWriteDb(dbPath)
var db = initReadWriteDb(dbPath)
teardown:
db.close()
removeDir($dbPath)
removeDir($dbBackupPath)
test "Test backup":
var engine = initBackupEngine(dbBackupPath)

21
tests/test_columnfamily.nim
View File

@ -9,12 +9,7 @@
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/columnfamily,
./test_helper
import std/os, tempfile, unittest2, ../rocksdb/columnfamily, ./test_helper
suite "ColFamily Tests":
const
@ -43,8 +38,18 @@ suite "ColFamily Tests":
check cf.put(key, val).isOk()
var bytes: seq[byte]
check cf.get(key, proc(data: openArray[byte]) = bytes = @data)[]
check not cf.get(otherKey, proc(data: openArray[byte]) = bytes = @data)[]
check cf.get(
key,
proc(data: openArray[byte]) =
bytes = @data
,
)[]
check not cf.get(
otherKey,
proc(data: openArray[byte]) =
bytes = @data
,
)[]
var r1 = cf.get(key)
check r1.isOk() and r1.value == val

35
tests/test_helper.nim
View File

@ -9,50 +9,41 @@
{.used.}
import
std/sequtils,
../rocksdb/backup,
../rocksdb/rocksdb,
../rocksdb/transactiondb
import std/sequtils, ../rocksdb/backup, ../rocksdb/rocksdb, ../rocksdb/transactiondb
proc initReadWriteDb*(
path: string,
columnFamilyNames: openArray[string] = @[]): RocksDbReadWriteRef =
path: string, columnFamilyNames: openArray[string] = @[]
): RocksDbReadWriteRef =
let res = openRocksDb(
path,
columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)))
path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it))
)
if res.isErr():
echo res.error()
doAssert res.isOk()
res.value()
proc initReadOnlyDb*(
path: string,
columnFamilyNames: openArray[string] = @[]): RocksDbReadOnlyRef =
path: string, columnFamilyNames: openArray[string] = @[]
): RocksDbReadOnlyRef =
let res = openRocksDbReadOnly(
path,
columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)))
path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it))
)
if res.isErr():
echo res.error()
doAssert res.isOk()
res.value()
proc initBackupEngine*(path: string): BackupEngineRef =
let res = openBackupEngine(path)
doAssert res.isOk()
res.value()
proc initTransactionDb*(
path: string,
columnFamilyNames: openArray[string] = @[]): TransactionDbRef =
path: string, columnFamilyNames: openArray[string] = @[]
): TransactionDbRef =
let res = openTransactionDb(
path,
columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)))
path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it))
)
if res.isErr():
echo res.error()
doAssert res.isOk()

120
tests/test_rocksdb.nim
View File

@ -9,12 +9,7 @@
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/rocksdb,
./test_helper
import std/os, tempfile, unittest2, ../rocksdb/rocksdb, ./test_helper
suite "RocksDbRef Tests":
const
@ -36,13 +31,22 @@ suite "RocksDbRef Tests":
removeDir($dbPath)
test "Basic operations":
var s = db.put(key, val)
check s.isOk()
var bytes: seq[byte]
check db.get(key, proc(data: openArray[byte]) = bytes = @data)[]
check not db.get(otherKey, proc(data: openArray[byte]) = bytes = @data)[]
check db.get(
key,
proc(data: openArray[byte]) =
bytes = @data
,
)[]
check not db.get(
otherKey,
proc(data: openArray[byte]) =
bytes = @data
,
)[]
var r1 = db.get(key)
check r1.isOk() and r1.value == val
@ -84,13 +88,24 @@ suite "RocksDbRef Tests":
check readOnlyDb.isClosed()
test "Basic operations - default column family":
var s = db.put(key, val, CF_DEFAULT)
check s.isOk()
var bytes: seq[byte]
check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
check not db.get(otherKey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
check db.get(
key,
proc(data: openArray[byte]) =
bytes = @data
,
CF_DEFAULT,
)[]
check not db.get(
otherKey,
proc(data: openArray[byte]) =
bytes = @data
,
CF_DEFAULT,
)[]
var r1 = db.get(key)
check r1.isOk() and r1.value == val
@ -132,7 +147,6 @@ suite "RocksDbRef Tests":
check readOnlyDb.isClosed()
test "Basic operations - multiple column families":
var s = db.put(key, val, CF_DEFAULT)
check s.isOk()
@ -140,12 +154,36 @@ suite "RocksDbRef Tests":
check s2.isOk()
var bytes: seq[byte]
check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
check not db.get(otherKey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[]
check db.get(
key,
proc(data: openArray[byte]) =
bytes = @data
,
CF_DEFAULT,
)[]
check not db.get(
otherKey,
proc(data: openArray[byte]) =
bytes = @data
,
CF_DEFAULT,
)[]
var bytes2: seq[byte]
check db.get(otherKey, proc(data: openArray[byte]) = bytes2 = @data, CF_OTHER)[]
check not db.get(key, proc(data: openArray[byte]) = bytes2 = @data, CF_OTHER)[]
check db.get(
otherKey,
proc(data: openArray[byte]) =
bytes2 = @data
,
CF_OTHER,
)[]
check not db.get(
key,
proc(data: openArray[byte]) =
bytes2 = @data
,
CF_OTHER,
)[]
var e1 = db.keyExists(key, CF_DEFAULT)
check e1.isOk() and e1.value == true
@ -178,8 +216,8 @@ suite "RocksDbRef Tests":
# Open database in read only mode
block:
var
readOnlyDb = initReadOnlyDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
var readOnlyDb =
initReadOnlyDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
var r = readOnlyDb.keyExists(key, CF_OTHER)
check r.isOk() and r.value == false
@ -192,7 +230,6 @@ suite "RocksDbRef Tests":
check readOnlyDb.isClosed()
test "Close multiple times":
check not db.isClosed()
db.close()
check db.isClosed()
@ -206,7 +243,13 @@ suite "RocksDbRef Tests":
check r.isErr() and r.error() == "rocksdb: unknown column family"
var bytes: seq[byte]
let r2 = db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_UNKNOWN)
let r2 = db.get(
key,
proc(data: openArray[byte]) =
bytes = @data
,
CF_UNKNOWN,
)
check r2.isErr() and r2.error() == "rocksdb: unknown column family"
let r3 = db.keyExists(key, CF_UNKNOWN)
@ -240,7 +283,12 @@ suite "RocksDbRef Tests":
block:
var v: seq[byte]
let r = db.get(key1, proc(data: openArray[byte]) = v = @data)
let r = db.get(
key1,
proc(data: openArray[byte]) =
v = @data
,
)
check:
r.isOk()
r.value() == true
@ -249,7 +297,12 @@ suite "RocksDbRef Tests":
block:
var v: seq[byte]
let r = db.get(key2, proc(data: openArray[byte]) = v = @data)
let r = db.get(
key2,
proc(data: openArray[byte]) =
v = @data
,
)
check:
r.isOk()
r.value() == true
@ -258,7 +311,12 @@ suite "RocksDbRef Tests":
block:
var v: seq[byte]
let r = db.get(key3, proc(data: openArray[byte]) = v = @data)
let r = db.get(
key3,
proc(data: openArray[byte]) =
v = @data
,
)
check:
r.isOk()
r.value() == true
@ -267,7 +325,12 @@ suite "RocksDbRef Tests":
block:
var v: seq[byte]
let r = db.get(key4, proc(data: openArray[byte]) = v = @data)
let r = db.get(
key4,
proc(data: openArray[byte]) =
v = @data
,
)
check:
r.isOk()
r.value() == false
@ -276,7 +339,12 @@ suite "RocksDbRef Tests":
block:
var v: seq[byte]
let r = db.get(key5, proc(data: openArray[byte]) = v = @data)
let r = db.get(
key5,
proc(data: openArray[byte]) =
v = @data
,
)
check:
r.isOk()
r.value() == false

47
tests/test_rocksiterator.nim
View File

@ -9,15 +9,9 @@
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[rocksdb, rocksiterator],
./test_helper
import std/os, tempfile, unittest2, ../rocksdb/[rocksdb, rocksiterator], ./test_helper
suite "RocksIteratorRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
@ -34,8 +28,8 @@ suite "RocksIteratorRef Tests":
setup:
let
dbPath = mkdtemp() / "data"
db = initReadWriteDb(dbPath,
columnFamilyNames = @[CF_DEFAULT, CF_OTHER, CF_EMPTY])
db =
initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER, CF_EMPTY])
doAssert db.put(key1, val1).isOk()
doAssert db.put(key2, val2).isOk()
@ -53,7 +47,8 @@ suite "RocksIteratorRef Tests":
check res.isOk()
var iter = res.get()
defer: iter.close()
defer:
iter.close()
iter.seekToFirst()
check iter.isValid()
@ -78,18 +73,24 @@ suite "RocksIteratorRef Tests":
check res.isOk()
var iter = res.get()
defer: iter.close()
defer:
iter.close()
iter.seekToLast()
check iter.isValid()
var expected = byte(3)
while iter.isValid():
var key: seq[byte]
iter.key(proc(data: openArray[byte]) = key = @data)
iter.key(
proc(data: openArray[byte]) =
key = @data
)
var val: seq[byte]
iter.value(proc(data: openArray[byte]) = val = @data)
iter.value(
proc(data: openArray[byte]) =
val = @data
)
check:
key == @[expected]
@ -105,11 +106,13 @@ suite "RocksIteratorRef Tests":
let res1 = db.openIterator(CF_DEFAULT)
check res1.isOk()
var iter1 = res1.get()
defer: iter1.close()
defer:
iter1.close()
let res2 = db.openIterator(CF_DEFAULT)
check res2.isOk()
var iter2 = res2.get()
defer: iter2.close()
defer:
iter2.close()
iter1.seekToFirst()
check iter1.isValid()
@ -126,11 +129,13 @@ suite "RocksIteratorRef Tests":
let res1 = db.openIterator(CF_DEFAULT)
check res1.isOk()
var iter1 = res1.get()
defer: iter1.close()
defer:
iter1.close()
let res2 = db.openIterator(CF_OTHER)
check res2.isOk()
var iter2 = res2.get()
defer: iter2.close()
defer:
iter2.close()
iter1.seekToFirst()
check iter1.isValid()
@ -153,7 +158,8 @@ suite "RocksIteratorRef Tests":
let res = db.openIterator(CF_EMPTY)
check res.isOk()
var iter = res.get()
defer: iter.close()
defer:
iter.close()
iter.seekToFirst()
check not iter.isValid()
@ -165,7 +171,8 @@ suite "RocksIteratorRef Tests":
let res = db.openIterator(CF_EMPTY)
check res.isOk()
var iter = res.get()
defer: iter.close()
defer:
iter.close()
check iter.status().isOk()
iter.seekToLast()

17
tests/test_sstfilewriter.nim
View File

@ -9,15 +9,9 @@
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[rocksdb, sstfilewriter],
./test_helper
import std/os, tempfile, unittest2, ../rocksdb/[rocksdb, sstfilewriter], ./test_helper
suite "SstFileWriterRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
@ -34,8 +28,7 @@ suite "SstFileWriterRef Tests":
let
dbPath = mkdtemp() / "data"
sstFilePath = mkdtemp() / "sst"
db = initReadWriteDb(dbPath,
columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
db = initReadWriteDb(dbPath, columnFamilyNames = @[CF_DEFAULT, CF_OTHER])
teardown:
db.close()
@ -45,7 +38,8 @@ suite "SstFileWriterRef Tests":
let res = openSstFileWriter(sstFilePath)
check res.isOk()
let writer = res.get()
defer: writer.close()
defer:
writer.close()
check:
writer.put(key1, val1).isOk()
@ -63,7 +57,8 @@ suite "SstFileWriterRef Tests":
let res = openSstFileWriter(sstFilePath)
check res.isOk()
let writer = res.get()
defer: writer.close()
defer:
writer.close()
check:
writer.put(key1, val1).isOk()

27
tests/test_transactiondb.nim
View File

@ -9,15 +9,9 @@
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[transactiondb],
./test_helper
import std/os, tempfile, unittest2, ../rocksdb/[transactiondb], ./test_helper
suite "TransactionDbRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
@ -38,11 +32,11 @@ suite "TransactionDbRef Tests":
db.close()
removeDir($dbPath)
# test multiple transactions
# test multiple transactions
test "Test rollback using default column family":
var tx = db.beginTransaction()
defer: tx.close()
defer:
tx.close()
check not tx.isClosed()
check:
@ -67,7 +61,8 @@ suite "TransactionDbRef Tests":
test "Test commit using default column family":
var tx = db.beginTransaction()
defer: tx.close()
defer:
tx.close()
check not tx.isClosed()
check:
@ -92,7 +87,8 @@ suite "TransactionDbRef Tests":
test "Test setting column family in beginTransaction":
var tx = db.beginTransaction(columnFamily = CF_OTHER)
defer: tx.close()
defer:
tx.close()
check not tx.isClosed()
check:
@ -111,13 +107,14 @@ suite "TransactionDbRef Tests":
tx.get(key2, CF_OTHER).error() == ""
tx.get(key3, CF_OTHER).get() == val3
test "Test rollback and commit with multiple transactions":
var tx1 = db.beginTransaction(columnFamily = CF_DEFAULT)
defer: tx1.close()
defer:
tx1.close()
check not tx1.isClosed()
var tx2 = db.beginTransaction(columnFamily = CF_OTHER)
defer: tx2.close()
defer:
tx2.close()
check not tx2.isClosed()
check:

29
tests/test_writebatch.nim
View File

@ -9,15 +9,9 @@
{.used.}
import
std/os,
tempfile,
unittest2,
../rocksdb/[rocksdb, writebatch],
./test_helper
import std/os, tempfile, unittest2, ../rocksdb/[rocksdb, writebatch], ./test_helper
suite "WriteBatchRef Tests":
const
CF_DEFAULT = "default"
CF_OTHER = "other"
@ -40,7 +34,8 @@ suite "WriteBatchRef Tests":
test "Test writing batch to the default column family":
var batch = db.openWriteBatch()
defer: batch.close()
defer:
batch.close()
check not batch.isClosed()
check:
@ -68,7 +63,8 @@ suite "WriteBatchRef Tests":
test "Test writing batch to column family":
var batch = db.openWriteBatch()
defer: batch.close()
defer:
batch.close()
check not batch.isClosed()
check:
@ -95,7 +91,8 @@ suite "WriteBatchRef Tests":
test "Test writing to multiple column families in single batch":
var batch = db.openWriteBatch()
defer: batch.close()
defer:
batch.close()
check not batch.isClosed()
check:
@ -124,11 +121,13 @@ suite "WriteBatchRef Tests":
test "Test writing to multiple column families in multiple batches":
var batch1 = db.openWriteBatch()
defer: batch1.close()
defer:
batch1.close()
check not batch1.isClosed()
var batch2 = db.openWriteBatch()
defer: batch2.close()
defer:
batch2.close()
check not batch2.isClosed()
check:
@ -157,7 +156,8 @@ suite "WriteBatchRef Tests":
const CF_UNKNOWN = "unknown"
var batch = db.openWriteBatch()
defer: batch.close()
defer:
batch.close()
check not batch.isClosed()
let r = batch.put(key1, val1, CF_UNKNOWN)
@ -168,7 +168,8 @@ suite "WriteBatchRef Tests":
test "Test write empty batch":
var batch = db.openWriteBatch()
defer: batch.close()
defer:
batch.close()
check not batch.isClosed()
check batch.count() == 0

5
tests/transactions/test_txdbopts.nim
View File

@ -9,12 +9,9 @@
{.used.}
import
unittest2,
../../rocksdb/transactions/txdbopts
import unittest2, ../../rocksdb/transactions/txdbopts
suite "TransactionDbOptionsRef Tests":
test "Test newTransactionDbOptions":
var txDbOpts = newTransactionDbOptions()

5
tests/transactions/test_txopts.nim
View File

@ -9,12 +9,9 @@
{.used.}
import
unittest2,
../../rocksdb/transactions/txopts
import unittest2, ../../rocksdb/transactions/txopts
suite "TransactionOptionsRef Tests":
test "Test newTransactionOptions":
var txOpts = newTransactionOptions()