status-im
/
nim-rocksdb
mirror of https://github.com/status-im/nim-rocksdb.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Auto close feature to improve memory management of option types. (#57) 

* Revert some changes from prior PR https://github.com/status-im/nim-rocksdb/pull/48 which enable memory leak of options when not cleaned up manually.

* Clean up, add more tests and use correct free function for listColumnFamilies.

* Close opt types when opening database fails.

* Add autoClose flag to each opt type.

* Finish autoClose changes to prevent memory leaks.
This commit is contained in:
web3-developer 2024-06-28 10:04:37 +08:00 committed by GitHub
parent ee15ce027b
commit 03313d8068
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 327 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
rocksdb/backup.nim
View File

@ -25,17 +25,21 @@ type
backupOpts: BackupEngineOptionsRef
proc openBackupEngine*(
path: string, backupOpts = defaultBackupEngineOptions()
path: string, backupOpts = defaultBackupEngineOptions(autoClose = true)
): RocksDBResult[BackupEngineRef] =
## Create a new backup engine. The `path` parameter is the path of the backup
## directory. Note that the same directory should not be used for both backups
## and the database itself.
##
## If no `backupOpts` are provided, the default options will be used. These
## default backup options will be closed when the backup engine is closed.
## If `backupOpts` are provided, they will need to be closed manually.
var errors: cstring
let backupEnginePtr = rocksdb_backup_engine_open(
backupOpts.cPtr, path.cstring, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
bailOnErrors(errors, backupOpts = backupOpts)
let engine =
BackupEngineRef(cPtr: backupEnginePtr, path: path, backupOpts: backupOpts)
@ -88,3 +92,6 @@ proc close*(backupEngine: BackupEngineRef) =
if not backupEngine.isClosed():
rocksdb_backup_engine_close(backupEngine.cPtr)
backupEngine.cPtr = nil
if backupEngine.backupOpts.autoClose:
backupEngine.backupOpts.close()

15
rocksdb/columnfamily/cfdescriptor.nim
View File

@ -9,16 +9,18 @@
{.push raises: [].}
import ../internal/utils, ./cfopts
import ./cfopts
export cfopts
const DEFAULT_COLUMN_FAMILY_NAME* = "default"
type ColFamilyDescriptor* = object
name: string
options: ColFamilyOptionsRef
proc initColFamilyDescriptor*(
name: string, options = defaultColFamilyOptions()
name: string, options: ColFamilyOptionsRef
): ColFamilyDescriptor =
ColFamilyDescriptor(name: name, options: options)
@ -28,11 +30,16 @@ proc name*(descriptor: ColFamilyDescriptor): string {.inline.} =
proc options*(descriptor: ColFamilyDescriptor): ColFamilyOptionsRef {.inline.} =
descriptor.options
proc autoClose*(descriptor: ColFamilyDescriptor): bool {.inline.} =
descriptor.options.autoClose
proc isDefault*(descriptor: ColFamilyDescriptor): bool {.inline.} =
descriptor.name == DEFAULT_COLUMN_FAMILY_NAME
proc defaultColFamilyDescriptor*(): ColFamilyDescriptor {.inline.} =
initColFamilyDescriptor(DEFAULT_COLUMN_FAMILY_NAME)
proc defaultColFamilyDescriptor*(autoClose = false): ColFamilyDescriptor {.inline.} =
initColFamilyDescriptor(
DEFAULT_COLUMN_FAMILY_NAME, defaultColFamilyOptions(autoClose = autoClose)
)
proc isClosed*(descriptor: ColFamilyDescriptor): bool {.inline.} =
descriptor.options.isClosed()

9
rocksdb/columnfamily/cfopts.nim
View File

@ -23,6 +23,7 @@ type
# type - CF options are a subset of rocksdb_options_t - when in doubt, check:
# https://github.com/facebook/rocksdb/blob/b8c9a2576af6a1d0ffcfbb517dfcb7e7037bd460/include/rocksdb/options.h#L66
cPtr: ColFamilyOptionsPtr
autoClose*: bool # if true then close will be called when the database is closed
Compression* {.pure.} = enum
# Use a slightly clunky name here to avoid global symbol conflicts
@ -50,8 +51,8 @@ proc close*(s: SlicetransformRef) =
rocksdb_slicetransform_destroy(s.cPtr)
s.cPtr = nil
proc newColFamilyOptions*(): ColFamilyOptionsRef =
ColFamilyOptionsRef(cPtr: rocksdb_options_create())
proc newColFamilyOptions*(autoClose = false): ColFamilyOptionsRef =
ColFamilyOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose)
proc isClosed*(cfOpts: ColFamilyOptionsRef): bool {.inline.} =
cfOpts.cPtr.isNil()
@ -114,8 +115,8 @@ opt blobGCForceThreshold, float, cdouble
opt blobCompactionReadaheadSize, int, uint64
opt blobFileStartingLevel, int, cint
proc defaultColFamilyOptions*(): ColFamilyOptionsRef =
newColFamilyOptions()
proc defaultColFamilyOptions*(autoClose = false): ColFamilyOptionsRef =
newColFamilyOptions(autoClose)
# proc setFixedPrefixExtractor*(dbOpts: ColFamilyOptionsRef, length: int) =
# doAssert not dbOpts.isClosed()

33
rocksdb/internal/utils.nim
View File

@ -9,17 +9,42 @@
{.push raises: [].}
import std/locks, ../lib/librocksdb
const DEFAULT_COLUMN_FAMILY_NAME* = "default"
import
std/locks,
../lib/librocksdb,
../options/[dbopts, readopts, writeopts, backupopts],
../transactions/txdbopts,
../columnfamily/cfdescriptor
proc createLock*(): Lock =
var lock = Lock()
initLock(lock)
lock
template bailOnErrors*(errors: cstring): auto =
template autoCloseNonNil*(opts: typed) =
if not opts.isNil and opts.autoClose:
opts.close()
template bailOnErrors*(
errors: cstring,
dbOpts: DbOptionsRef = nil,
readOpts: ReadOptionsRef = nil,
writeOpts: WriteOptionsRef = nil,
txDbOpts: TransactionDbOptionsRef = nil,
backupOpts: BackupEngineOptionsRef = nil,
cfDescriptors: openArray[ColFamilyDescriptor] = @[],
): auto =
if not errors.isNil:
autoCloseNonNil(dbOpts)
autoCloseNonNil(readOpts)
autoCloseNonNil(writeOpts)
autoCloseNonNil(txDbOpts)
autoCloseNonNil(backupOpts)
for cfDesc in cfDescriptors:
if cfDesc.autoClose:
cfDesc.close()
let res = err($(errors))
rocksdb_free(errors)
return res

12
rocksdb/options/backupopts.nim
View File

@ -16,9 +16,10 @@ type
BackupEngineOptionsRef* = ref object
cPtr: BackupEngineOptionsPtr
autoClose*: bool # if true then close will be called when the backup engine is closed
proc newBackupEngineOptions*(): BackupEngineOptionsRef =
BackupEngineOptionsRef(cPtr: rocksdb_options_create())
proc newBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef =
BackupEngineOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose)
proc isClosed*(engineOpts: BackupEngineOptionsRef): bool {.inline.} =
engineOpts.cPtr.isNil()
@ -29,8 +30,11 @@ proc cPtr*(engineOpts: BackupEngineOptionsRef): BackupEngineOptionsPtr =
# TODO: Add setters and getters for backup options properties.
proc defaultBackupEngineOptions*(): BackupEngineOptionsRef {.inline.} =
let opts = newBackupEngineOptions()
proc defaultBackupEngineOptions*(autoClose = false): BackupEngineOptionsRef {.inline.} =
let opts = newBackupEngineOptions(autoClose)
# TODO: set defaults here
opts
proc close*(engineOpts: BackupEngineOptionsRef) =

9
rocksdb/options/dbopts.nim
View File

@ -18,9 +18,10 @@ type
DbOptionsRef* = ref object
cPtr: DbOptionsPtr
autoClose*: bool # if true then close will be called when the database is closed
proc newDbOptions*(): DbOptionsRef =
DbOptionsRef(cPtr: rocksdb_options_create())
proc newDbOptions*(autoClose = false): DbOptionsRef =
DbOptionsRef(cPtr: rocksdb_options_create(), autoClose: autoClose)
proc isClosed*(dbOpts: DbOptionsRef): bool {.inline.} =
dbOpts.cPtr.isNil()
@ -95,8 +96,8 @@ proc `rowCache=`*(dbOpts: DbOptionsRef, cache: CacheRef) =
doAssert not dbOpts.isClosed()
rocksdb_options_set_row_cache(dbOpts.cPtr, cache.cPtr)
proc defaultDbOptions*(): DbOptionsRef =
let opts: DbOptionsRef = newDbOptions()
proc defaultDbOptions*(autoClose = false): DbOptionsRef =
let opts: DbOptionsRef = newDbOptions(autoClose)
# Optimize RocksDB. This is the easiest way to get RocksDB to perform well:
opts.increaseParallelism(countProcessors())

9
rocksdb/options/readopts.nim
View File

@ -16,9 +16,10 @@ type
ReadOptionsRef* = ref object
cPtr: ReadOptionsPtr
autoClose*: bool # if true then close will be called when the database is closed
proc newReadOptions*(): ReadOptionsRef =
ReadOptionsRef(cPtr: rocksdb_readoptions_create())
proc newReadOptions*(autoClose = false): ReadOptionsRef =
ReadOptionsRef(cPtr: rocksdb_readoptions_create(), autoClose: autoClose)
proc isClosed*(readOpts: ReadOptionsRef): bool {.inline.} =
readOpts.cPtr.isNil()
@ -29,8 +30,8 @@ proc cPtr*(readOpts: ReadOptionsRef): ReadOptionsPtr =
# TODO: Add setters and getters for read options properties.
proc defaultReadOptions*(): ReadOptionsRef {.inline.} =
newReadOptions()
proc defaultReadOptions*(autoClose = false): ReadOptionsRef {.inline.} =
newReadOptions(autoClose)
# TODO: set prefered defaults
proc close*(readOpts: ReadOptionsRef) =

9
rocksdb/options/writeopts.nim
View File

@ -16,9 +16,10 @@ type
WriteOptionsRef* = ref object
cPtr: WriteOptionsPtr
autoClose*: bool # if true then close will be called when the database is closed
proc newWriteOptions*(): WriteOptionsRef =
WriteOptionsRef(cPtr: rocksdb_writeoptions_create())
proc newWriteOptions*(autoClose = false): WriteOptionsRef =
WriteOptionsRef(cPtr: rocksdb_writeoptions_create(), autoClose: autoClose)
proc isClosed*(writeOpts: WriteOptionsRef): bool {.inline.} =
writeOpts.cPtr.isNil()
@ -29,8 +30,8 @@ proc cPtr*(writeOpts: WriteOptionsRef): WriteOptionsPtr =
# TODO: Add setters and getters for write options properties.
proc defaultWriteOptions*(): WriteOptionsRef {.inline.} =
newWriteOptions()
proc defaultWriteOptions*(autoClose = false): WriteOptionsRef {.inline.} =
newWriteOptions(autoClose)
# TODO: set prefered defaults
proc close*(writeOpts: WriteOptionsRef) =

123
rocksdb/rocksdb.nim
View File

@ -47,6 +47,7 @@ type
path: string
dbOpts: DbOptionsRef
readOpts: ReadOptionsRef
cfDescriptors: seq[ColFamilyDescriptor]
defaultCfHandle: ColFamilyHandleRef
cfTable: ColFamilyTableRef
@ -56,9 +57,7 @@ type
writeOpts: WriteOptionsRef
ingestOptsPtr: IngestExternalFilesOptionsPtr
proc listColumnFamilies*(
path: string, dbOpts = DbOptionsRef(nil)
): RocksDBResult[seq[string]] =
proc listColumnFamilies*(path: string): RocksDBResult[seq[string]] =
## List exisiting column families on disk. This might be used to find out
## whether there were some columns missing with the version on disk.
##
@ -69,59 +68,50 @@ proc listColumnFamilies*(
## Note that the on-the-fly adding might not be needed in the way described
## above once rocksdb has been upgraded to the latest version, see comments
## at the end of ./columnfamily/cfhandle.nim.
##
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil:
useDbOpts.close()
var
lencf: csize_t
cfLen: csize_t
errors: cstring
let cList = rocksdb_list_column_families(
useDbOpts.cPtr, path.cstring, addr lencf, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
let
dbOpts = defaultDbOptions(autoClose = true)
cfList = rocksdb_list_column_families(
dbOpts.cPtr, path.cstring, addr cfLen, cast[cstringArray](errors.addr)
)
bailOnErrors(errors, dbOpts)
var cfs: seq[string]
if not cList.isNil:
defer:
rocksdb_free(cList)
if cfList.isNil or cfLen == 0:
return ok(newSeqOfCap[string](0))
for n in 0 ..< lencf:
if cList[n].isNil:
# Clean up the rest
for z in n + 1 ..< lencf:
if not cList[z].isNil:
rocksdb_free(cList[z])
return err("short reply")
defer:
rocksdb_list_column_families_destroy(cfList, cfLen)
dbOpts.close()
cfs.add $cList[n]
rocksdb_free(cList[n])
var colFamilyNames = newSeqOfCap[string](cfLen)
for i in 0 ..< cfLen:
colFamilyNames.add($cfList[i])
ok cfs
ok(colFamilyNames)
proc openRocksDb*(
path: string,
dbOpts = DbOptionsRef(nil),
readOpts = ReadOptionsRef(nil),
writeOpts = WriteOptionsRef(nil),
dbOpts = defaultDbOptions(autoClose = true),
readOpts = defaultReadOptions(autoClose = true),
writeOpts = defaultWriteOptions(autoClose = true),
columnFamilies: openArray[ColFamilyDescriptor] = [],
): RocksDBResult[RocksDbReadWriteRef] =
## Open a RocksDB instance in read-write mode. If `columnFamilies` is empty
## then it will open the default column family. If `dbOpts`, `readOpts`, or
## `writeOpts` are not supplied then the default options will be used.
## These default options will be closed when the database is closed.
## If any options are provided, they will need to be closed manually.
##
## By default, column families will be created if they don't yet exist.
## All existing column families must be specified if the database has
## previously created any column families.
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil:
useDbOpts.close()
var cfs = columnFamilies.toSeq()
if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
cfs.add(defaultColFamilyDescriptor())
cfs.add(defaultColFamilyDescriptor(autoClose = true))
var
cfNames = cfs.mapIt(it.name().cstring)
@ -129,7 +119,7 @@ proc openRocksDb*(
cfHandles = newSeq[ColFamilyHandlePtr](cfs.len)
errors: cstring
let rocksDbPtr = rocksdb_open_column_families(
useDbOpts.cPtr,
dbOpts.cPtr,
path.cstring,
cfNames.len().cint,
cast[cstringArray](cfNames[0].addr),
@ -137,12 +127,9 @@ proc openRocksDb*(
cfHandles[0].addr,
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
bailOnErrors(errors, dbOpts, readOpts, writeOpts, cfDescriptors = cfs)
let
dbOpts = useDbOpts # don't close on exit
readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts)
writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts)
cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles)
db = RocksDbReadWriteRef(
lock: createLock(),
@ -151,6 +138,7 @@ proc openRocksDb*(
dbOpts: dbOpts,
readOpts: readOpts,
writeOpts: writeOpts,
cfDescriptors: cfs,
ingestOptsPtr: rocksdb_ingestexternalfileoptions_create(),
defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME),
cfTable: cfTable,
@ -159,25 +147,24 @@ proc openRocksDb*(
proc openRocksDbReadOnly*(
path: string,
dbOpts = DbOptionsRef(nil),
readOpts = ReadOptionsRef(nil),
dbOpts = defaultDbOptions(autoClose = true),
readOpts = defaultReadOptions(autoClose = true),
columnFamilies: openArray[ColFamilyDescriptor] = [],
errorIfWalFileExists = false,
): RocksDBResult[RocksDbReadOnlyRef] =
## Open a RocksDB instance in read-only mode. If `columnFamilies` is empty
## then it will open the default column family. If `dbOpts` or `readOpts` are
## not supplied then the default options will be used. By default, column
## families will be created if they don't yet exist. If the database already
## contains any column families, then all or a subset of the existing column
## families can be opened for reading.
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil:
useDbOpts.close()
## not supplied then the default options will be used.
## These default options will be closed when the database is closed.
## If any options are provided, they will need to be closed manually.
##
## By default, column families will be created if they don't yet exist.
## If the database already contains any column families, then all or
## a subset of the existing column families can be opened for reading.
var cfs = columnFamilies.toSeq()
if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
cfs.add(defaultColFamilyDescriptor())
cfs.add(defaultColFamilyDescriptor(autoClose = true))
var
cfNames = cfs.mapIt(it.name().cstring)
@ -185,7 +172,7 @@ proc openRocksDbReadOnly*(
cfHandles = newSeq[ColFamilyHandlePtr](cfs.len)
errors: cstring
let rocksDbPtr = rocksdb_open_for_read_only_column_families(
useDbOpts.cPtr,
dbOpts.cPtr,
path.cstring,
cfNames.len().cint,
cast[cstringArray](cfNames[0].addr),
@ -194,11 +181,9 @@ proc openRocksDbReadOnly*(
errorIfWalFileExists.uint8,
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
bailOnErrors(errors, dbOpts, readOpts, cfDescriptors = cfs)
let
dbOpts = useDbOpts # don't close on exit
readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts)
cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles)
db = RocksDbReadOnlyRef(
lock: createLock(),
@ -206,6 +191,7 @@ proc openRocksDbReadOnly*(
path: path,
dbOpts: dbOpts,
readOpts: readOpts,
cfDescriptors: cfs,
defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME),
cfTable: cfTable,
)
@ -414,15 +400,26 @@ proc close*(db: RocksDbRef) =
withLock(db.lock):
if not db.isClosed():
db.dbOpts.close()
db.readOpts.close()
# the column families should be closed before the database
db.cfTable.close()
if db of RocksDbReadWriteRef:
let db = RocksDbReadWriteRef(db)
db.writeOpts.close()
rocksdb_ingestexternalfileoptions_destroy(db.ingestOptsPtr)
db.ingestOptsPtr = nil
rocksdb_close(db.cPtr)
db.cPtr = nil
# opts should be closed after the database is closed
if db.dbOpts.autoClose:
db.dbOpts.close()
if db.readOpts.autoClose:
db.readOpts.close()
for cfDesc in db.cfDescriptors:
if cfDesc.autoClose:
cfDesc.close()
if db of RocksDbReadWriteRef:
let db = RocksDbReadWriteRef(db)
if db.writeOpts.autoClose:
db.writeOpts.close()
rocksdb_ingestexternalfileoptions_destroy(db.ingestOptsPtr)
db.ingestOptsPtr = nil

3
rocksdb/rocksiterator.nim
View File

@ -46,8 +46,7 @@ proc seekToKey*(iter: RocksIteratorRef, key: openArray[byte]) =
## invalid.
##
doAssert not iter.isClosed()
let (cKey, cLen) = (cast[cstring](unsafeAddr key[0]), csize_t(key.len))
rocksdb_iter_seek(iter.cPtr, cKey, cLen)
rocksdb_iter_seek(iter.cPtr, cast[cstring](unsafeAddr key[0]), csize_t(key.len))
proc seekToFirst*(iter: RocksIteratorRef) =
## Seeks to the first entry in the column family.

11
rocksdb/sstfilewriter.nim
View File

@ -25,10 +25,12 @@ type
dbOpts: DbOptionsRef
proc openSstFileWriter*(
filePath: string, dbOpts = DbOptionsRef(nil)
filePath: string, dbOpts = defaultDbOptions(autoClose = true)
): RocksDBResult[SstFileWriterRef] =
## Creates a new `SstFileWriterRef` and opens the file at the given `filePath`.
let dbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
## If `dbOpts` is not supplied then the default options will be used.
## These default options will be closed when the file writer is closed.
## If any options are provided, they will need to be closed manually.
doAssert not dbOpts.isClosed()
let envOptsPtr = rocksdb_envoptions_create()
@ -42,7 +44,7 @@ proc openSstFileWriter*(
rocksdb_sstfilewriter_open(
writer.cPtr, filePath.cstring, cast[cstringArray](errors.addr)
)
bailOnErrors(errors)
bailOnErrors(errors, dbOpts)
ok(writer)
@ -99,3 +101,6 @@ proc close*(writer: SstFileWriterRef) =
writer.envOptsPtr = nil
rocksdb_sstfilewriter_destroy(writer.cPtr)
writer.cPtr = nil
if writer.dbOpts.autoClose:
writer.dbOpts.close()

51
rocksdb/transactiondb.nim
View File

@ -36,26 +36,25 @@ type
path: string
dbOpts: DbOptionsRef
txDbOpts: TransactionDbOptionsRef
cfDescriptors: seq[ColFamilyDescriptor]
defaultCfHandle: ColFamilyHandleRef
cfTable: ColFamilyTableRef
proc openTransactionDb*(
path: string,
dbOpts = DbOptionsRef(nil),
txDbOpts = TransactionDbOptionsRef(nil),
dbOpts = defaultDbOptions(autoClose = true),
txDbOpts = defaultTransactionDbOptions(autoClose = true),
columnFamilies: openArray[ColFamilyDescriptor] = [],
): RocksDBResult[TransactionDbRef] =
## Open a `TransactionDbRef` with the given options and column families.
## If no column families are provided the default column family will be used.
## If no options are provided the default options will be used.
let useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
defer:
if dbOpts.isNil:
useDbOpts.close()
## These default options will be closed when the database is closed.
## If any options are provided, they will need to be closed manually.
var cfs = columnFamilies.toSeq()
if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
cfs.add(defaultColFamilyDescriptor())
cfs.add(defaultColFamilyDescriptor(autoClose = true))
var
cfNames = cfs.mapIt(it.name().cstring)
@ -64,7 +63,7 @@ proc openTransactionDb*(
errors: cstring
let txDbPtr = rocksdb_transactiondb_open_column_families(
useDbOpts.cPtr,
dbOpts.cPtr,
txDbOpts.cPtr,
path.cstring,
cfNames.len().cint,
@ -73,13 +72,9 @@ proc openTransactionDb*(
cfHandles[0].addr,
cast[cstringArray](errors.addr),
)
bailOnErrors(errors)
bailOnErrors(errors, dbOpts, txDbOpts = txDbOpts, cfDescriptors = cfs)
let
dbOpts = useDbOpts # don't close on exit
txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions()
else: txDbOpts
)
cfTable = newColFamilyTable(cfNames.mapIt($it), cfHandles)
db = TransactionDbRef(
lock: createLock(),
@ -87,6 +82,7 @@ proc openTransactionDb*(
path: path,
dbOpts: dbOpts,
txDbOpts: txDbOpts,
cfDescriptors: cfs,
defaultCfHandle: cfTable.get(DEFAULT_COLUMN_FAMILY_NAME),
cfTable: cfTable,
)
@ -107,22 +103,17 @@ proc isClosed*(db: TransactionDbRef): bool {.inline.} =
proc beginTransaction*(
db: TransactionDbRef,
readOpts = ReadOptionsRef(nil),
writeOpts = WriteOptionsRef(nil),
txDbOpts = TransactionDbOptionsRef(nil),
txOpts = defaultTransactionOptions(),
readOpts = defaultReadOptions(autoClose = true),
writeOpts = defaultWriteOptions(autoClose = true),
txOpts = defaultTransactionOptions(autoClose = true),
cfHandle = db.defaultCfHandle,
): TransactionRef =
## Begin a new transaction against the database. The transaction will default
## to using the specified column family. If no column family is specified
## then the default column family will be used.
##
##
doAssert not db.isClosed()
let
txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions()
else: txDbOpts
)
readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts)
writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts)
let txPtr = rocksdb_transaction_begin(db.cPtr, writeOpts.cPtr, txOpts.cPtr, nil)
@ -130,11 +121,21 @@ proc beginTransaction*(
proc close*(db: TransactionDbRef) =
## Close the `TransactionDbRef`.
withLock(db.lock):
if not db.isClosed():
db.dbOpts.close()
db.txDbOpts.close()
# the column families should be closed before the database
db.cfTable.close()
rocksdb_transactiondb_close(db.cPtr)
db.cPtr = nil
# opts should be closed after the database is closed
if db.dbOpts.autoClose:
db.dbOpts.close()
if db.txDbOpts.autoClose:
db.txDbOpts.close()
for cfDesc in db.cfDescriptors:
if cfDesc.autoClose:
cfDesc.close()

12
rocksdb/transactions/transaction.nim
View File

@ -175,9 +175,13 @@ proc rollback*(tx: TransactionRef): RocksDBResult[void] =
proc close*(tx: TransactionRef) =
## Close the `TransactionRef`.
if not tx.isClosed():
tx.readOpts.close()
tx.writeOpts.close()
tx.txOpts.close()
rocksdb_transaction_destroy(tx.cPtr)
tx.cPtr = nil
# opts should be closed after the transaction is closed
if tx.readOpts.autoClose:
tx.readOpts.close()
if tx.writeOpts.autoClose:
tx.writeOpts.close()
if tx.txOpts.autoClose:
tx.txOpts.close()

13
rocksdb/transactions/txdbopts.nim
View File

@ -16,9 +16,12 @@ type
TransactionDbOptionsRef* = ref object
cPtr: TransactionDbOptionsPtr
autoClose*: bool # if true then close will be called when the database is closed
proc newTransactionDbOptions*(): TransactionDbOptionsRef =
TransactionDbOptionsRef(cPtr: rocksdb_transactiondb_options_create())
proc newTransactionDbOptions*(autoClose = false): TransactionDbOptionsRef =
TransactionDbOptionsRef(
cPtr: rocksdb_transactiondb_options_create(), autoClose: autoClose
)
proc isClosed*(txDbOpts: TransactionDbOptionsRef): bool {.inline.} =
txDbOpts.cPtr.isNil()
@ -29,8 +32,10 @@ proc cPtr*(txDbOpts: TransactionDbOptionsRef): TransactionDbOptionsPtr =
# TODO: Add setters and getters for backup options properties.
proc defaultTransactionDbOptions*(): TransactionDbOptionsRef {.inline.} =
newTransactionDbOptions()
proc defaultTransactionDbOptions*(
autoClose = false
): TransactionDbOptionsRef {.inline.} =
newTransactionDbOptions(autoClose)
# TODO: set prefered defaults
proc close*(txDbOpts: TransactionDbOptionsRef) =

11
rocksdb/transactions/txopts.nim
View File

@ -16,9 +16,12 @@ type
TransactionOptionsRef* = ref object
cPtr: TransactionOptionsPtr
autoClose*: bool # if true then close will be called when the transaction is closed
proc newTransactionOptions*(): TransactionOptionsRef =
TransactionOptionsRef(cPtr: rocksdb_transaction_options_create())
proc newTransactionOptions*(autoClose = false): TransactionOptionsRef =
TransactionOptionsRef(
cPtr: rocksdb_transaction_options_create(), autoClose: autoClose
)
proc isClosed*(txOpts: TransactionOptionsRef): bool {.inline.} =
txOpts.cPtr.isNil()
@ -29,8 +32,8 @@ proc cPtr*(txOpts: TransactionOptionsRef): TransactionOptionsPtr =
# TODO: Add setters and getters for backup options properties.
proc defaultTransactionOptions*(): TransactionOptionsRef {.inline.} =
newTransactionOptions()
proc defaultTransactionOptions*(autoClose = false): TransactionOptionsRef {.inline.} =
newTransactionOptions(autoClose)
# TODO: set prefered defaults
proc close*(txOpts: TransactionOptionsRef) =

12
tests/columnfamily/test_cfdescriptor.nim
View File

@ -9,21 +9,11 @@
{.used.}
import unittest2, ../../rocksdb/internal/utils, ../../rocksdb/columnfamily/cfdescriptor
import unittest2, ../../rocksdb/columnfamily/cfdescriptor
suite "ColFamilyDescriptor Tests":
const TEST_CF_NAME = "test"
test "Test initColFamilyDescriptor":
var descriptor = initColFamilyDescriptor(TEST_CF_NAME)
check:
descriptor.name() == TEST_CF_NAME
not descriptor.options().isNil()
not descriptor.isDefault()
descriptor.close()
test "Test initColFamilyDescriptor with options":
var descriptor = initColFamilyDescriptor(TEST_CF_NAME, defaultColFamilyOptions())

15
tests/test_helper.nim
View File

@ -15,7 +15,10 @@ proc initReadWriteDb*(
path: string, columnFamilyNames: openArray[string] = @[]
): RocksDbReadWriteRef =
let res = openRocksDb(
path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it))
path,
columnFamilies = columnFamilyNames.mapIt(
initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true))
),
)
if res.isErr():
echo res.error()
@ -26,7 +29,10 @@ proc initReadOnlyDb*(
path: string, columnFamilyNames: openArray[string] = @[]
): RocksDbReadOnlyRef =
let res = openRocksDbReadOnly(
path, columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it))
path,
columnFamilies = columnFamilyNames.mapIt(
initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true))
),
)
if res.isErr():
echo res.error()
@ -43,8 +49,9 @@ proc initTransactionDb*(
): TransactionDbRef =
let res = openTransactionDb(
path,
txDbOpts = defaultTransactionDbOptions(),
columnFamilies = columnFamilyNames.mapIt(initColFamilyDescriptor(it)),
columnFamilies = columnFamilyNames.mapIt(
initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true))
),
)
if res.isErr():
echo res.error()

57
tests/test_rocksdb.nim
View File

@ -231,18 +231,6 @@ suite "RocksDbRef Tests":
readOnlyDb.close()
check readOnlyDb.isClosed()
test "Close multiple times":
check not db.isClosed()
db.close()
check db.isClosed()
db.close()
check db.isClosed()
test "Unknown column family":
const CF_UNKNOWN = "unknown"
let cfHandleRes = db.getColFamilyHandle(CF_UNKNOWN)
check cfHandleRes.isErr() and cfHandleRes.error() == "rocksdb: unknown column family"
test "Test missing key and values":
let
key1 = @[byte(1)] # exists with non empty value
@ -335,3 +323,48 @@ suite "RocksDbRef Tests":
r.value() == false
v.len() == 0
db.get(key5).isErr()
test "List column familes":
let cfRes1 = listColumnFamilies(dbPath)
check:
cfRes1.isOk()
cfRes1.value() == @[CF_DEFAULT, CF_OTHER]
let
dbPath2 = dbPath & "2"
db2 = initReadWriteDb(dbPath2, columnFamilyNames = @[CF_DEFAULT])
cfRes2 = listColumnFamilies(dbPath2)
check:
cfRes2.isOk()
cfRes2.value() == @[CF_DEFAULT]
test "Unknown column family":
const CF_UNKNOWN = "unknown"
let cfHandleRes = db.getColFamilyHandle(CF_UNKNOWN)
check cfHandleRes.isErr() and cfHandleRes.error() == "rocksdb: unknown column family"
test "Close multiple times":
check not db.isClosed()
db.close()
check db.isClosed()
db.close()
check db.isClosed()
test "Test auto close":
let
dbPath = mkdtemp() / "autoclose"
dbOpts = defaultDbOptions(autoClose = false)
readOpts = defaultReadOptions(autoClose = true)
db = openRocksDb(dbPath, dbOpts, readOpts).get()
check:
dbOpts.isClosed() == false
readOpts.isClosed() == false
db.isClosed() == false
db.close()
check:
dbOpts.isClosed() == false
readOpts.isClosed() == true
db.isClosed() == true

14
tests/test_rocksiterator.nim
View File

@ -151,6 +151,20 @@ suite "RocksIteratorRef Tests":
iter2.key() == @[byte(3)]
iter2.value() == @[byte(3)]
test "Iterate forwards using seek to key":
let res = db.openIterator(defaultCfHandle)
check res.isOk()
var iter = res.get()
defer:
iter.close()
iter.seekToKey(key2)
check:
iter.isValid()
iter.key() == key2
iter.value() == val2
test "Empty column family":
let res = db.openIterator(emptyCfHandle)
check res.isOk()

19
tests/test_transactiondb.nim
View File

@ -201,3 +201,22 @@ suite "TransactionDbRef Tests":
check tx2.isClosed()
tx2.close()
check tx2.isClosed()
test "Test auto close":
let
dbPath = mkdtemp() / "autoclose"
dbOpts = defaultDbOptions(autoClose = false)
txDbOpts = defaultTransactionDbOptions(autoClose = true)
db = openTransactionDb(dbPath, dbOpts, txDbOpts).get()
check:
dbOpts.isClosed() == false
txDbOpts.isClosed() == false
db.isClosed() == false
db.close()
check:
dbOpts.isClosed() == false
txDbOpts.isClosed() == true
db.isClosed() == true

54
tests/test_writebatch.nim
View File

@ -65,18 +65,18 @@ suite "WriteBatchRef Tests":
not batch.isClosed()
test "Test writing batch to column family":
var batch = db.openWriteBatch(otherCfHandle)
var batch = db.openWriteBatch()
defer:
batch.close()
check not batch.isClosed()
check:
batch.put(key1, val1).isOk()
batch.put(key2, val2).isOk()
batch.put(key3, val3).isOk()
batch.put(key1, val1, otherCfHandle).isOk()
batch.put(key2, val2, otherCfHandle).isOk()
batch.put(key3, val3, otherCfHandle).isOk()
batch.count() == 3
batch.delete(key2).isOk()
batch.delete(key2, otherCfHandle).isOk()
batch.count() == 4
not batch.isClosed()
@ -92,26 +92,54 @@ suite "WriteBatchRef Tests":
batch.count() == 0
not batch.isClosed()
test "Test writing to multiple column families in single batch":
var batch = db.openWriteBatch()
defer:
batch.close()
check not batch.isClosed()
check:
batch.put(key1, val1, defaultCfHandle).isOk()
batch.put(key1, val1, otherCfHandle).isOk()
batch.put(key2, val2, otherCfHandle).isOk()
batch.put(key3, val3, otherCfHandle).isOk()
batch.count() == 4
batch.delete(key2, otherCfHandle).isOk()
batch.count() == 5
not batch.isClosed()
let res = db.write(batch)
check:
res.isOk()
db.get(key1, defaultCfHandle).get() == val1
db.get(key1, otherCfHandle).get() == val1
db.keyExists(key2, otherCfHandle).get() == false
db.get(key3, otherCfHandle).get() == val3
batch.clear()
check:
batch.count() == 0
not batch.isClosed()
test "Test writing to multiple column families in multiple batches":
var batch1 = db.openWriteBatch(defaultCfHandle)
var batch1 = db.openWriteBatch()
defer:
batch1.close()
check not batch1.isClosed()
var batch2 = db.openWriteBatch(otherCfHandle)
var batch2 = db.openWriteBatch()
defer:
batch2.close()
check not batch2.isClosed()
check:
batch1.put(key1, val1).isOk()
batch1.delete(key2).isOk()
batch1.put(key3, val3).isOk()
batch2.put(key1, val1).isOk()
batch2.delete(key1).isOk()
batch1.delete(key2, otherCfHandle).isOk()
batch1.put(key3, val3, otherCfHandle).isOk()
batch2.put(key1, val1, otherCfHandle).isOk()
batch2.delete(key1, otherCfHandle).isOk()
batch2.put(key3, val3).isOk()
batch1.count() == 3
batch2.count() == 3