diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a79970d..1a0ae08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,7 +136,7 @@ jobs: curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip 7z x -y external/windeps.zip -o"${DLLPATH}" # ROCKSDB - curl -L "https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/nimbus-deps.zip" -o external/nimbus-deps.zip + curl -L "https://github.com/status-im/nimbus-deps/releases/download/rocksdb-9.1.0/nimbus-deps.zip" -o external/nimbus-deps.zip 7z x -y external/nimbus-deps.zip cp -a "./${ROCKSDBSUB}/librocksdb.dll" "${DLLPATH}/librocksdb.dll" diff --git a/config.nims b/config.nims index 4850a6e..5a1c6a8 100644 --- a/config.nims +++ b/config.nims @@ -22,3 +22,6 @@ when defined(rocksdb_static_linking): switch("dynlibOverride", "rocksdb") switch("dynlibOverride", "lz4") switch("dynlibOverride", "zstd") + +--styleCheck:usages +--styleCheck:error diff --git a/rocksdb/columnfamily/cfopts.nim b/rocksdb/columnfamily/cfopts.nim index 9a989eb..82c528e 100644 --- a/rocksdb/columnfamily/cfopts.nim +++ b/rocksdb/columnfamily/cfopts.nim @@ -50,6 +50,10 @@ proc defaultColFamilyOptions*(): ColFamilyOptionsRef = # doAssert not cfOpts.isClosed() # rocksdb_options_get_create_missing_column_families(cfOpts.cPtr).bool +proc setWriteBufferSize*(dbOpts: ColFamilyOptionsRef, maxBufferSize: int) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_write_buffer_size(dbOpts.cPtr, maxBufferSize.csize_t) + proc close*(cfOpts: ColFamilyOptionsRef) = if not cfOpts.isClosed(): rocksdb_options_destroy(cfOpts.cPtr) diff --git a/rocksdb/lib/librocksdb.nim b/rocksdb/lib/librocksdb.nim index 8850f73..308f62f 100644 --- a/rocksdb/lib/librocksdb.nim +++ b/rocksdb/lib/librocksdb.nim @@ -120,17 +120,17 @@ when defined(rocksdb_static_linking): topLevelPath = currentSourcePath.parentDir().parentDir().parentDir() libsDir = topLevelPath.replace('\\', '/') & "/build/lib" - {.passL: libsDir & "/librocksdb.a".} - {.passL: libsDir & "/liblz4.a".} - {.passL: libsDir & "/libzstd.a".} + {.passl: libsDir & "/librocksdb.a".} + {.passl: libsDir & "/liblz4.a".} + {.passl: libsDir & "/libzstd.a".} when defined(windows): - {.passL: "-lshlwapi -lrpcrt4".} + {.passl: "-lshlwapi -lrpcrt4".} else: when shouldUseNativeLinking(): {.pragma: importrocks, importc, cdecl.} - {.passL: "-lrocksdb".} + {.passl: "-lrocksdb".} else: {.pragma: importrocks, importc, cdecl, dynlib: librocksdb.} diff --git a/rocksdb/options/cache.nim b/rocksdb/options/cache.nim new file mode 100644 index 0000000..93d7e57 --- /dev/null +++ b/rocksdb/options/cache.nim @@ -0,0 +1,16 @@ +import + ../lib/librocksdb + +type + CachePtr* = ptr rocksdb_cache_t + + CacheRef* = ref object + cPtr*: CachePtr + +proc cacheCreateLRU*(size: int): CacheRef = + CacheRef(cPtr: rocksdb_cache_create_lru(size.csize_t)) + +proc close*(cache: CacheRef) = + if cache.cPtr != nil: + rocksdb_cache_destroy(cache.cPtr) + cache.cPtr = nil diff --git a/rocksdb/options/dbopts.nim b/rocksdb/options/dbopts.nim index 963a903..7fc9b57 100644 --- a/rocksdb/options/dbopts.nim +++ b/rocksdb/options/dbopts.nim @@ -11,7 +11,10 @@ import std/cpuinfo, - ../lib/librocksdb + ../lib/librocksdb, + ./[cache, tableopts] + +export cache, tableopts type DbOptionsPtr* = ptr rocksdb_options_t @@ -47,24 +50,58 @@ proc setCreateMissingColumnFamilies*(dbOpts: DbOptionsRef, flag: bool) = doAssert not dbOpts.isClosed() rocksdb_options_set_create_missing_column_families(dbOpts.cPtr, flag.uint8) +proc setWriteBufferSize*(dbOpts: DbOptionsRef, maxBufferSize: int) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_write_buffer_size(dbOpts.cPtr, maxBufferSize.csize_t) + +proc setRowCache*(dbOpts: DbOptionsRef, cache: CacheRef) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_row_cache(dbOpts.cPtr, cache.cPtr) + +proc setMaxBackgroundJobs*(dbOpts: DbOptionsRef, jobs: int) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_max_background_jobs(dbOpts.cPtr, jobs.cint) + +proc setBytesPerSync*(dbOpts: DbOptionsRef, bytes: int64) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_bytes_per_sync(dbOpts.cPtr, bytes.csize_t) + +proc setBlockBasedTableFactory*(dbOpts: DbOptionsRef, tableOpts: TableOptionsRef) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_block_based_table_factory(dbOpts.cPtr, tableOpts.cPtr) + +proc setTargetFileSizeBase*(dbOpts: DbOptionsRef, bytes: int64) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_target_file_size_base(dbOpts.cPtr, bytes.csize_t) + +proc setMaxBytesForLevelBase*(dbOpts: DbOptionsRef, bytes: int64) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_max_bytes_for_level_base(dbOpts.cPtr, bytes.csize_t) + +proc setMaxBytesForLevelMultiplier*(dbOpts: DbOptionsRef, multiplier: float) = + doAssert not dbOpts.isClosed() + rocksdb_options_set_max_bytes_for_level_multiplier(dbOpts.cPtr, multiplier.cdouble) + proc defaultDbOptions*(): DbOptionsRef = let opts: DbOptionsRef = newDbOptions() - # rocksdb_options_set_compression(opts.cPtr, rocksdb_lz4_compression) - # rocksdb_options_set_bottommost_compression(opts.cPtr, rocksdb_zstd_compression) - # Optimize RocksDB. This is the easiest way to get RocksDB to perform well: opts.setIncreaseParallelism(countProcessors()) - # This requires snappy - disabled because rocksdb is not always compiled with - # snappy support (for example Fedora 28, certain Ubuntu versions) - # rocksdb_options_optimize_level_style_compaction(options, 0); opts.setCreateIfMissing(true) + # default set to keep all files open (-1), allow setting it to a specific # value, e.g. in case the application limit would be reached. opts.setMaxOpenFiles(-1) # Enable creating column families if they do not exist opts.setCreateMissingColumnFamilies(true) - return opts + + # Options recommended by rocksdb devs themselves, for new databases + # https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#other-general-options + + opts.setMaxBackgroundJobs(6) + opts.setBytesPerSync(1048576) + + opts # TODO: These procs below will not work unless using the latest version of rocksdb # Currently, when installing librocksdb-dev on linux the RocksDb version used is 6.11.4 diff --git a/rocksdb/options/tableopts.nim b/rocksdb/options/tableopts.nim new file mode 100644 index 0000000..7811710 --- /dev/null +++ b/rocksdb/options/tableopts.nim @@ -0,0 +1,47 @@ +import + ../lib/librocksdb, + ./cache + +type + # TODO might eventually wrap this + TableOptionsPtr* = ptr rocksdb_block_based_table_options_t + + TableOptionsRef* = ref object + cPtr*: TableOptionsPtr + +proc createTableOptions*(): TableOptionsRef = + TableOptionsRef(cPtr: rocksdb_block_based_options_create()) + +proc isClosed*(opts: TableOptionsRef): bool = + isNil(opts.cPtr) + +proc close*(opts: TableOptionsRef) = + if not isClosed(opts): + rocksdb_block_based_options_destroy(opts.cPtr) + opts.cPtr = nil + +# TODO there's _a lot_ of options to set - here we expose a select few.. + +proc setBlockSize*(opts: TableOptionsRef, size: int) = + rocksdb_block_based_options_set_block_size(opts.cPtr, size.csize_t) + +proc setBlockCache*(opts: TableOptionsRef, cache: CacheRef) = + rocksdb_block_based_options_set_block_cache(opts.cPtr, cache.cPtr) + +proc setFormatVersion*(opts: TableOptionsRef, version: int) = + rocksdb_block_based_options_set_format_version(opts.cPtr, version.cint) + +proc setCacheIndexAndFilterBlocks*(opts: TableOptionsRef, value: bool) = + rocksdb_block_based_options_set_cache_index_and_filter_blocks(opts.cPtr, value.uint8) + +proc setPinL0FilterAndIndexBlocksInCache*(opts: TableOptionsRef, value: bool) = + rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(opts.cPtr, value.uint8) + +proc defaultTableOptions*(): TableOptionsRef = + # https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning#other-general-options + let opts = createTableOptions() + opts.setBlockSize(16*1024) + opts.setCacheIndexAndFilterBlocks(true) + opts.setPinL0FilterAndIndexBlocksInCache(true) + opts.setFormatVersion(5) + opts diff --git a/rocksdb/rocksdb.nim b/rocksdb/rocksdb.nim index 0a4eb41..531387f 100644 --- a/rocksdb/rocksdb.nim +++ b/rocksdb/rocksdb.nim @@ -327,7 +327,7 @@ proc write*( proc ingestExternalFile*( db: RocksDbReadWriteRef, filePath: string, - columnFamily = db.defaultCfName): RocksDbResult[void] = + columnFamily = db.defaultCfName): RocksDBResult[void] = ## Ingest an external sst file into the database. The file will be ingested ## into the specified column family or the default column family if none is ## provided. diff --git a/tests/lib/test_librocksdb.nim b/tests/lib/test_librocksdb.nim index 916fc1d..249250c 100644 --- a/tests/lib/test_librocksdb.nim +++ b/tests/lib/test_librocksdb.nim @@ -81,7 +81,7 @@ suite "librocksdb C wrapper Tests": rocksdb_backup_engine_restore_db_from_latest_backup(be, dbPath, dbPath, restoreOptions, err) check: err.isNil - rocksdb_restore_options_destroy(restore_options) + rocksdb_restore_options_destroy(restoreOptions) db = rocksdb_open(options, dbPath, err) check: err.isNil diff --git a/tests/test_columnfamily.nim b/tests/test_columnfamily.nim index 33f0c9b..dfd6fd2 100644 --- a/tests/test_columnfamily.nim +++ b/tests/test_columnfamily.nim @@ -44,7 +44,7 @@ suite "ColFamily Tests": var bytes: seq[byte] check cf.get(key, proc(data: openArray[byte]) = bytes = @data)[] - check not cf.get(otherkey, proc(data: openArray[byte]) = bytes = @data)[] + check not cf.get(otherKey, proc(data: openArray[byte]) = bytes = @data)[] var r1 = cf.get(key) check r1.isOk() and r1.value == val diff --git a/tests/test_rocksdb.nim b/tests/test_rocksdb.nim index e290973..4f3d7f4 100644 --- a/tests/test_rocksdb.nim +++ b/tests/test_rocksdb.nim @@ -42,7 +42,7 @@ suite "RocksDbRef Tests": var bytes: seq[byte] check db.get(key, proc(data: openArray[byte]) = bytes = @data)[] - check not db.get(otherkey, proc(data: openArray[byte]) = bytes = @data)[] + check not db.get(otherKey, proc(data: openArray[byte]) = bytes = @data)[] var r1 = db.get(key) check r1.isOk() and r1.value == val @@ -90,7 +90,7 @@ suite "RocksDbRef Tests": var bytes: seq[byte] check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] - check not db.get(otherkey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] + check not db.get(otherKey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] var r1 = db.get(key) check r1.isOk() and r1.value == val @@ -141,7 +141,7 @@ suite "RocksDbRef Tests": var bytes: seq[byte] check db.get(key, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] - check not db.get(otherkey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] + check not db.get(otherKey, proc(data: openArray[byte]) = bytes = @data, CF_DEFAULT)[] var bytes2: seq[byte] check db.get(otherKey, proc(data: openArray[byte]) = bytes2 = @data, CF_OTHER)[]