From 6b7de5730b6b3bcbd4b2ff89d412f8d771bf3b3e Mon Sep 17 00:00:00 2001 From: web3-developer <51288821+web3-developer@users.noreply.github.com> Date: Wed, 3 Jul 2024 23:46:42 +0800 Subject: [PATCH] Implement missing getter and setters for option types. (#62) * Update readme. * Add additional opts getters and setters for ReadOptionsRef and WriteOptionsRef. Updated BackupEngineOptionsRef to use newer c library type. * Add majority of option type getter and setters. --- README.md | 37 ++++++++++---------- rocksdb/backup.nim | 14 ++++++-- rocksdb/options/backupopts.nim | 56 +++++++++++++++++++++---------- rocksdb/options/dbopts.nim | 18 +++++----- rocksdb/options/readopts.nim | 27 +++++++++++++-- rocksdb/options/writeopts.nim | 27 +++++++++++++-- rocksdb/transactions/txdbopts.nim | 14 ++++++-- rocksdb/transactions/txopts.nim | 16 +++++++-- tests/options/test_backupopts.nim | 6 ++-- tests/test_all.nim | 6 ++-- tests/test_backup.nim | 15 ++++----- tests/test_rocksdb.nim | 21 ++++++++++++ 12 files changed, 187 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 8aec7c7..f59b6e0 100644 --- a/README.md +++ b/README.md @@ -12,38 +12,37 @@ A Nim wrapper for [Facebook's RocksDB](https://github.com/facebook/rocksdb), a p Nim-RocksDB provides a wrapper for the low-level functions in the librocksdb c library. -## Requirements +## Installation -A RocksDB installation that provides `librocksdb.so`. This means that on Debian, and possibly on other Linux distros, you need "librocksdb-dev", not just a versioned "librocksdbX.Y" package that only provides `librocksdb.so.X.Y.Z`. +Nim-RocksDB requires Nim and the Nimble package manager. For Windows you will need Visual Studio 2015 Update 3 or greater with the English language pack. -## Usage +To get started run: +``` +nimble install +``` -See [simple_example](examples/simple_example.nim) +This will download and install the RocksDB dynamic libraries for your platform and copy them into the `build/` directory of the project. When including this library in your application you may want to copy these libraries into another location or set the LD_LIBRARY_PATH environment variable (DYLD_LIBRARY_PATH on MacOS, PATH on Windows) to include the `build/` directory so that your application can find them on startup. + +Alternatively you can use the `rocksdb_static_linking` flag to statically link the library into your application. ### Static linking -To statically link librocksdb, you would do something like: +To build the RocksDB static libraries run: +``` +./scripts/build_static_deps.sh +``` -```nim +To statically link RocksDB, you would do something like: + +``` nim c -d:rocksdb_static_linking --threads:on your_program.nim ``` See the config.nims file which contains the static linking configuration which is switched on with the `rocksdb_static_linking` flag. Note that static linking is currently not supported on windows. -### Building Windows DLLs +## Usage -Prerequisites: -- Windows 7 or newer -- Git -- Visual Studio 2015 Update 3 or greater with the English language pack - -To build RocksDB for Windows, run the following: - -``` -.\scripts\build_dlls_windows.bat -``` - -After the build completes the built RocksDB DLLs will be located in the `build` directory. +See [simple_example](examples/simple_example.nim) ### Contribution diff --git a/rocksdb/backup.nim b/rocksdb/backup.nim index 98301b4..2c251d9 100644 --- a/rocksdb/backup.nim +++ b/rocksdb/backup.nim @@ -18,14 +18,16 @@ export backupopts, rocksdb, rocksresult type BackupEnginePtr* = ptr rocksdb_backup_engine_t + EnvPtr = ptr rocksdb_env_t BackupEngineRef* = ref object cPtr: BackupEnginePtr + env: EnvPtr path: string backupOpts: BackupEngineOptionsRef proc openBackupEngine*( - path: string, backupOpts = defaultBackupEngineOptions(autoClose = true) + path: string, backupOpts = defaultBackupEngineOptions(path, autoClose = true) ): RocksDBResult[BackupEngineRef] = ## Create a new backup engine. The `path` parameter is the path of the backup ## directory. Note that the same directory should not be used for both backups @@ -35,12 +37,14 @@ proc openBackupEngine*( ## default backup options will be closed when the backup engine is closed. ## If `backupOpts` are provided, they will need to be closed manually. + let env = rocksdb_create_default_env() var errors: cstring - let backupEnginePtr = rocksdb_backup_engine_open( - backupOpts.cPtr, path.cstring, cast[cstringArray](errors.addr) + let backupEnginePtr = rocksdb_backup_engine_open_opts( + backupOpts.cPtr, env, cast[cstringArray](errors.addr) ) bailOnErrorsWithCleanup(errors): autoCloseNonNil(backupOpts) + rocksdb_env_destroy(env) let engine = BackupEngineRef(cPtr: backupEnginePtr, path: path, backupOpts: backupOpts) @@ -94,4 +98,8 @@ proc close*(backupEngine: BackupEngineRef) = rocksdb_backup_engine_close(backupEngine.cPtr) backupEngine.cPtr = nil + if not backupEngine.env.isNil(): + rocksdb_env_destroy(backupEngine.env) + backupEngine.env = nil + autoCloseNonNil(backupEngine.backupOpts) diff --git a/rocksdb/options/backupopts.nim b/rocksdb/options/backupopts.nim index 7d01294..18c5659 100644 --- a/rocksdb/options/backupopts.nim +++ b/rocksdb/options/backupopts.nim @@ -12,32 +12,54 @@ import ../lib/librocksdb type - BackupEngineOptionsPtr* = ptr rocksdb_options_t + BackupEngineOptionsPtr* = ptr rocksdb_backup_engine_options_t BackupEngineOptionsRef* = ref object cPtr: BackupEngineOptionsPtr autoClose*: bool # if true then close will be called when the backup engine is closed -proc createBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef = - BackupEngineOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose) +proc createBackupEngineOptions*( + backupDir: string, autoClose = false +): BackupEngineOptionsRef = + BackupEngineOptionsRef( + cPtr: rocksdb_backup_engine_options_create(backupDir.cstring), autoClose: autoClose + ) -proc isClosed*(engineOpts: BackupEngineOptionsRef): bool {.inline.} = - engineOpts.cPtr.isNil() +proc isClosed*(backupOpts: BackupEngineOptionsRef): bool {.inline.} = + backupOpts.cPtr.isNil() -proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr = - doAssert not engineOpts.isClosed() - engineOpts.cPtr +proc cPtr*(backupOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr = + doAssert not backupOpts.isClosed() + backupOpts.cPtr -# TODO: Add setters and getters for backup options properties. +template opt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(backupOpts: BackupEngineOptionsRef, value: ntyp) = + doAssert not backupOpts.isClosed() + `rocksdb_backup_engine_options_set nname`(backupOpts.cPtr, value.ctyp) -proc defaultBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef {.inline.} = - let opts = createBackupEngineOptions(autoClose) + proc `nname`*(backupOpts: BackupEngineOptionsRef): ntyp = + doAssert not backupOpts.isClosed() + ntyp `rocksdb_backup_engine_options_get nname`(backupOpts.cPtr) + +opt shareTableFiles, bool, uint8 +opt sync, bool, uint8 +opt destroyOldData, bool, uint8 +opt backupLogFiles, bool, uint8 +opt backupRateLimit, int, uint64 +opt restoreRateLimit, int, uint64 +opt shareFilesWithChecksumNaming, bool, cint +opt maxBackgroundOperations, int, cint +opt callbackTriggerIntervalSize, int, uint64 + +proc defaultBackupEngineOptions*( + backupDir: string, autoClose = false +): BackupEngineOptionsRef {.inline.} = + let backupOpts = createBackupEngineOptions(backupDir, autoClose) # TODO: set defaults here + backupOpts - opts - -proc close*(engineOpts: BackupEngineOptionsRef) = - if not engineOpts.isClosed(): - rocksdb_options_destroy(engineOpts.cPtr) - engineOpts.cPtr = nil +proc close*(backupOpts: BackupEngineOptionsRef) = + if not backupOpts.isClosed(): + rocksdb_backup_engine_options_destroy(backupOpts.cPtr) + backupOpts.cPtr = nil diff --git a/rocksdb/options/dbopts.nim b/rocksdb/options/dbopts.nim index 2349f08..3bc3ae3 100644 --- a/rocksdb/options/dbopts.nim +++ b/rocksdb/options/dbopts.nim @@ -40,11 +40,11 @@ proc increaseParallelism*(dbOpts: DbOptionsRef, totalThreads: int) = template opt(nname, ntyp, ctyp: untyped) = proc `nname=`*(dbOpts: DbOptionsRef, value: ntyp) = - doAssert not dbOpts.isClosed + doAssert not dbOpts.isClosed() `rocksdb_options_set nname`(dbOpts.cPtr, value.ctyp) proc `nname`*(dbOpts: DbOptionsRef): ntyp = - doAssert not dbOpts.isClosed + doAssert not dbOpts.isClosed() ntyp `rocksdb_options_get nname`(dbOpts.cPtr) opt createIfMissing, bool, uint8 @@ -102,22 +102,22 @@ proc `rowCache=`*(dbOpts: DbOptionsRef, cache: CacheRef) = dbOpts.cache = cache proc defaultDbOptions*(autoClose = false): DbOptionsRef = - let opts: DbOptionsRef = createDbOptions(autoClose) + let dbOpts: DbOptionsRef = createDbOptions(autoClose) # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: - opts.increaseParallelism(countProcessors()) - opts.createIfMissing = true + dbOpts.increaseParallelism(countProcessors()) + dbOpts.createIfMissing = true # Enable creating column families if they do not exist - opts.createMissingColumnFamilies = true + dbOpts.createMissingColumnFamilies = true # Options recommended by rocksdb devs themselves, for new databases # https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#other-general-options - opts.maxBackgroundJobs = 6 - opts.bytesPerSync = 1048576 + dbOpts.maxBackgroundJobs = 6 + dbOpts.bytesPerSync = 1048576 - opts + dbOpts proc close*(dbOpts: DbOptionsRef) = if not dbOpts.isClosed(): diff --git a/rocksdb/options/readopts.nim b/rocksdb/options/readopts.nim index f60269e..976836f 100644 --- a/rocksdb/options/readopts.nim +++ b/rocksdb/options/readopts.nim @@ -28,11 +28,34 @@ proc cPtr*(readOpts: ReadOptionsRef): ReadOptionsPtr = doAssert not readOpts.isClosed() readOpts.cPtr -# TODO: Add setters and getters for read options properties. +template opt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(readOpts: ReadOptionsRef, value: ntyp) = + doAssert not readOpts.isClosed() + `rocksdb_readoptions_set nname`(readOpts.cPtr, value.ctyp) + + proc `nname`*(readOpts: ReadOptionsRef): ntyp = + doAssert not readOpts.isClosed() + ntyp `rocksdb_readoptions_get nname`(readOpts.cPtr) + +opt verifyChecksums, bool, uint8 +opt fillCache, bool, uint8 +opt readTier, int, cint +opt tailing, bool, uint8 +opt totalOrderSeek, bool, uint8 +opt prefixSameAsStart, bool, uint8 +opt pinData, bool, uint8 +opt backgroundPurgeOnIteratorCleanup, bool, uint8 +opt readaheadSize, int, csize_t +opt maxSkippableInternalKeys, int, csize_t +opt ignoreRangeDeletions, bool, uint8 +opt deadline, int, uint64 +opt ioTimeout, int, uint64 proc defaultReadOptions*(autoClose = false): ReadOptionsRef {.inline.} = - createReadOptions(autoClose) + let readOpts = createReadOptions(autoClose) + # TODO: set prefered defaults + readOpts proc close*(readOpts: ReadOptionsRef) = if not readOpts.isClosed(): diff --git a/rocksdb/options/writeopts.nim b/rocksdb/options/writeopts.nim index f621e7f..62c3f48 100644 --- a/rocksdb/options/writeopts.nim +++ b/rocksdb/options/writeopts.nim @@ -28,11 +28,34 @@ proc cPtr*(writeOpts: WriteOptionsRef): WriteOptionsPtr = doAssert not writeOpts.isClosed() writeOpts.cPtr -# TODO: Add setters and getters for write options properties. +template opt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(writeOpts: WriteOptionsRef, value: ntyp) = + doAssert not writeOpts.isClosed() + `rocksdb_writeoptions_set nname`(writeOpts.cPtr, value.ctyp) + + proc `nname`*(writeOpts: WriteOptionsRef): ntyp = + doAssert not writeOpts.isClosed() + ntyp `rocksdb_writeoptions_get nname`(writeOpts.cPtr) + +opt sync, bool, uint8 +opt ignoreMissingColumnFamilies, bool, uint8 +opt noSlowdown, bool, uint8 +opt lowPri, bool, uint8 +opt memtableInsertHintPerBatch, bool, uint8 + +proc `disableWAL=`*(writeOpts: WriteOptionsRef, value: bool) = + doAssert not writeOpts.isClosed() + rocksdb_writeoptions_disable_WAL(writeOpts.cPtr, value.cint) + +proc disableWAL*(writeOpts: WriteOptionsRef): bool = + doAssert not writeOpts.isClosed() + rocksdb_writeoptions_get_disable_WAL(writeOpts.cPtr).bool proc defaultWriteOptions*(autoClose = false): WriteOptionsRef {.inline.} = - createWriteOptions(autoClose) + let writeOpts = createWriteOptions(autoClose) + # TODO: set prefered defaults + writeOpts proc close*(writeOpts: WriteOptionsRef) = if not writeOpts.isClosed(): diff --git a/rocksdb/transactions/txdbopts.nim b/rocksdb/transactions/txdbopts.nim index b2d934b..aa6b8c5 100644 --- a/rocksdb/transactions/txdbopts.nim +++ b/rocksdb/transactions/txdbopts.nim @@ -30,13 +30,23 @@ proc cPtr*(txDbOpts: TransactionDbOptionsRef): TransactionDbOptionsPtr = doAssert not txDbOpts.isClosed() txDbOpts.cPtr -# TODO: Add setters and getters for backup options properties. +template setOpt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(txDbOpts: TransactionDbOptionsRef, value: ntyp) = + doAssert not txDbOpts.isClosed() + `rocksdb_transactiondb_options_set nname`(txDbOpts.cPtr, value.ctyp) + +setOpt maxNumLocks, int, int64 +setOpt numStripes, int, csize_t +setOpt transactionLockTimeout, int, int64 +setOpt defaultLockTimeout, int, int64 proc defaultTransactionDbOptions*( autoClose = false ): TransactionDbOptionsRef {.inline.} = - createTransactionDbOptions(autoClose) + let txDbOpts = createTransactionDbOptions(autoClose) + # TODO: set prefered defaults + txDbOpts proc close*(txDbOpts: TransactionDbOptionsRef) = if not txDbOpts.isClosed(): diff --git a/rocksdb/transactions/txopts.nim b/rocksdb/transactions/txopts.nim index a22f2fa..a312f86 100644 --- a/rocksdb/transactions/txopts.nim +++ b/rocksdb/transactions/txopts.nim @@ -30,11 +30,23 @@ proc cPtr*(txOpts: TransactionOptionsRef): TransactionOptionsPtr = doAssert not txOpts.isClosed() txOpts.cPtr -# TODO: Add setters and getters for backup options properties. +template setOpt(nname, ntyp, ctyp: untyped) = + proc `nname=`*(txOpts: TransactionOptionsRef, value: ntyp) = + doAssert not txOpts.isClosed() + `rocksdb_transaction_options_set nname`(txOpts.cPtr, value.ctyp) + +setOpt setSnapshot, bool, uint8 +setOpt deadlockDetect, bool, uint8 +setOpt lockTimeout, int, int64 +setOpt deadlockDetectDepth, int, int64 +setOpt maxWriteBatchSize, int, csize_t +setOpt skipPrepare, bool, uint8 proc defaultTransactionOptions*(autoClose = false): TransactionOptionsRef {.inline.} = - createTransactionOptions(autoClose) + let txOpts = createTransactionOptions(autoClose) + # TODO: set prefered defaults + txOpts proc close*(txOpts: TransactionOptionsRef) = if not txOpts.isClosed(): diff --git a/tests/options/test_backupopts.nim b/tests/options/test_backupopts.nim index 9d0e21a..aa85f8d 100644 --- a/tests/options/test_backupopts.nim +++ b/tests/options/test_backupopts.nim @@ -13,21 +13,21 @@ import unittest2, ../../rocksdb/options/backupopts suite "BackupEngineOptionsRef Tests": test "Test newBackupEngineOptions": - var backupOpts = createBackupEngineOptions() + let backupOpts = createBackupEngineOptions(".") check not backupOpts.cPtr.isNil() backupOpts.close() test "Test defaultBackupEngineOptions": - var backupOpts = defaultBackupEngineOptions() + let backupOpts = defaultBackupEngineOptions(".") check not backupOpts.cPtr.isNil() backupOpts.close() test "Test close": - var backupOpts = defaultBackupEngineOptions() + let backupOpts = defaultBackupEngineOptions(".") check not backupOpts.isClosed() backupOpts.close() diff --git a/tests/test_all.nim b/tests/test_all.nim index 0360269..62d2fa1 100644 --- a/tests/test_all.nim +++ b/tests/test_all.nim @@ -16,8 +16,8 @@ import ./options/test_backupopts, ./options/test_dbopts, ./options/test_readopts, - ./options/test_writeopts, ./options/test_tableopts, + ./options/test_writeopts, ./transactions/test_txdbopts, ./transactions/test_txopts, ./test_backup, @@ -25,5 +25,5 @@ import ./test_rocksdb, ./test_rocksiterator, ./test_sstfilewriter, - ./test_writebatch, - ./test_transactiondb + ./test_transactiondb, + ./test_writebatch diff --git a/tests/test_backup.nim b/tests/test_backup.nim index 4e50650..7b4ab85 100644 --- a/tests/test_backup.nim +++ b/tests/test_backup.nim @@ -21,16 +21,16 @@ suite "BackupEngineRef Tests": dbPath = mkdtemp() / "data" dbBackupPath = mkdtemp() / "backup" dbRestorePath = mkdtemp() / "restore" - - var db = initReadWriteDb(dbPath) + db = initReadWriteDb(dbPath) teardown: db.close() removeDir($dbPath) removeDir($dbBackupPath) + removeDir($dbRestorePath) test "Test backup": - var engine = initBackupEngine(dbBackupPath) + let engine = initBackupEngine(dbBackupPath) check: db.put(key, val).isOk() @@ -46,13 +46,12 @@ suite "BackupEngineRef Tests": let db2 = initReadWriteDb(dbRestorePath) check db2.keyExists(key).value() + db2.close() engine.close() test "Test close": - let res = openBackupEngine(dbPath) - doAssert res.isOk() - var engine = res.get() + let engine = openBackupEngine(dbPath).get() check not engine.isClosed() engine.close() @@ -62,7 +61,7 @@ suite "BackupEngineRef Tests": test "Test auto close enabled": let - backupOpts = defaultBackupEngineOptions(autoClose = true) + backupOpts = defaultBackupEngineOptions(dbPath, autoClose = true) backupEngine = openBackupEngine(dbPath, backupOpts).get() check: @@ -77,7 +76,7 @@ suite "BackupEngineRef Tests": test "Test auto close disabled": let - backupOpts = defaultBackupEngineOptions(autoClose = false) + backupOpts = defaultBackupEngineOptions(dbPath, autoClose = false) backupEngine = openBackupEngine(dbPath, backupOpts).get() check: diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim index 160609a..d0a8d14 100644 --- a/tests/test_rocksdb.nim +++ b/tests/test_rocksdb.nim @@ -407,3 +407,24 @@ suite "RocksDbRef Tests": writeOpts.isClosed() == false columnFamilies[0].isClosed() == false db.isClosed() == true + + test "Test compression libraries linked": + let + dbPath = mkdtemp() / "compression" + cfOpts = defaultColFamilyOptions(autoClose = false) + cfDescriptor = initColFamilyDescriptor(CF_DEFAULT, cfOpts) + + cfOpts.compression = lz4Compression + check cfOpts.compression == lz4Compression + let db1 = openRocksDb(dbPath, columnFamilies = @[cfDescriptor]).get() + check db1.put(key, val).isOk() + db1.close() + + cfOpts.compression = zstdCompression + check cfOpts.compression == zstdCompression + let db2 = openRocksDb(dbPath, columnFamilies = @[cfDescriptor]).get() + check db2.put(key, val).isOk() + db2.close() + + cfOpts.close() + removeDir($dbPath)