Expose function for listing existing column families (#48)

* Expose function for listing existing column families

* Free CF list after use

* Update, fix error case

* Rename listRocksDbCFs -> listColumnFamilies

* Need to free each item separately

* Handle default option objects, close after use

rocksdb/rocksdb.nim

@@ -63,18 +63,71 @@ type
     writeOpts: WriteOptionsRef
     ingestOptsPtr: IngestExternalFilesOptionsPtr
 
+proc listColumnFamilies*(
+    path: string;
+    dbOpts = DbOptionsRef(nil);
+      ): RocksDBResult[seq[string]] =
+  ## List exisiting column families on disk. This might be used to find out
+  ## whether there were some columns missing with the version on disk.
+  ##
+  ## Column families previously used must be declared when re-opening an
+  ## existing database. So this function can be used to add some CFs
+  ## on-the-fly to the opener list of CFs
+  ##
+  ## Note that the on-the-fly adding might not be needed in the way described
+  ## above once rocksdb has been upgraded to the latest version, see comments
+  ## at the end of ./columnfamily/cfhandle.nim.
+  ##
+  let
+    useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
+  defer:
+    if dbOpts.isNil: useDbOpts.close()
+
+  var
+    lencf: csize_t
+    errors: cstring
+  let
+    cList = rocksdb_list_column_families(
+    useDbOpts.cPtr,
+    path.cstring,
+    addr lencf,
+    cast[cstringArray](errors.addr))
+  bailOnErrors(errors)
+
+  var cfs: seq[string]
+  if not cList.isNil:
+    defer: rocksdb_free(cList)
+
+    for n in 0 ..< lencf:
+      if cList[n].isNil:
+        # Clean up the rest
+        for z in n+1 ..< lencf:
+          if not cList[z].isNil:
+            rocksdb_free(cList[z])
+        return err("short reply")
+
+      cfs.add $cList[n]
+      rocksdb_free(cList[n])
+
+  ok cfs
+
 proc openRocksDb*(
-    path: string,
-    dbOpts = defaultDbOptions(),
-    readOpts = defaultReadOptions(),
-    writeOpts = defaultWriteOptions(),
-    columnFamilies: openArray[ColFamilyDescriptor] = []): RocksDBResult[RocksDbReadWriteRef] =
+    path: string;
+    dbOpts = DbOptionsRef(nil);
+    readOpts = ReadOptionsRef(nil);
+    writeOpts = WriteOptionsRef(nil);
+    columnFamilies: openArray[ColFamilyDescriptor] = [];
+      ): RocksDBResult[RocksDbReadWriteRef] =
   ## Open a RocksDB instance in read-write mode. If `columnFamilies` is empty
   ## then it will open the default column family. If `dbOpts`, `readOpts`, or
   ## `writeOpts` are not supplied then the default options will be used.
   ## By default, column families will be created if they don't yet exist.
   ## All existing column families must be specified if the database has
   ## previously created any column families.
+  let
+    useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
+  defer:
+    if dbOpts.isNil: useDbOpts.close()
 
   var cfs = columnFamilies.toSeq()
   if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
@@ -86,7 +139,7 @@ proc openRocksDb*(
     cfHandles = newSeq[ColFamilyHandlePtr](cfs.len)
     errors: cstring
   let rocksDbPtr = rocksdb_open_column_families(
-        dbOpts.cPtr,
+        useDbOpts.cPtr,
         path.cstring,
         cfNames.len().cint,
         cast[cstringArray](cfNames[0].addr),
@@ -95,7 +148,11 @@ proc openRocksDb*(
         cast[cstringArray](errors.addr))
   bailOnErrors(errors)
 
-  let db = RocksDbReadWriteRef(
+  let
+    dbOpts = useDbOpts # don't close on exit
+    readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts)
+    writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts)
+    db = RocksDbReadWriteRef(
       lock: createLock(),
       cPtr: rocksDbPtr,
       path: path,
@@ -108,17 +165,22 @@ proc openRocksDb*(
   ok(db)
 
 proc openRocksDbReadOnly*(
-    path: string,
-    dbOpts = defaultDbOptions(),
-    readOpts = defaultReadOptions(),
-    columnFamilies: openArray[ColFamilyDescriptor] = [],
-    errorIfWalFileExists = false): RocksDBResult[RocksDbReadOnlyRef] =
+    path: string;
+    dbOpts = DbOptionsRef(nil);
+    readOpts = ReadOptionsRef(nil);
+    columnFamilies: openArray[ColFamilyDescriptor] = [];
+    errorIfWalFileExists = false;
+      ): RocksDBResult[RocksDbReadOnlyRef] =
   ## Open a RocksDB instance in read-only mode. If `columnFamilies` is empty
   ## then it will open the default column family. If `dbOpts` or `readOpts` are
   ## not supplied then the default options will be used. By default, column
   ## families will be created if they don't yet exist. If the database already
   ## contains any column families, then all or a subset of the existing column
   ## families can be opened for reading.
+  let
+    useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
+  defer:
+    if dbOpts.isNil: useDbOpts.close()
 
   var cfs = columnFamilies.toSeq()
   if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
@@ -130,7 +192,7 @@ proc openRocksDbReadOnly*(
     cfHandles = newSeq[ColFamilyHandlePtr](cfs.len)
     errors: cstring
   let rocksDbPtr = rocksdb_open_for_read_only_column_families(
-        dbOpts.cPtr,
+        useDbOpts.cPtr,
         path.cstring,
         cfNames.len().cint,
         cast[cstringArray](cfNames[0].addr),
@@ -140,7 +202,10 @@ proc openRocksDbReadOnly*(
         cast[cstringArray](errors.addr))
   bailOnErrors(errors)
 
-  let db = RocksDbReadOnlyRef(
+  let
+    dbOpts = useDbOpts # don't close on exit
+    readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts)
+    db = RocksDbReadOnlyRef(
       lock: createLock(),
       cPtr: rocksDbPtr,
       path: path,

rocksdb/sstfilewriter.nim

@@ -30,9 +30,11 @@ type
     dbOpts: DbOptionsRef
 
 proc openSstFileWriter*(
-    filePath: string,
-    dbOpts = defaultDbOptions()): RocksDBResult[SstFileWriterRef] =
+    filePath: string;
+    dbOpts = DbOptionsRef(nil);
+      ): RocksDBResult[SstFileWriterRef] =
   ## Creates a new `SstFileWriterRef` and opens the file at the given `filePath`.
+  let dbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
   doAssert not dbOpts.isClosed()
 
   let envOptsPtr = rocksdb_envoptions_create()

rocksdb/transactiondb.nim

@@ -47,12 +47,17 @@ type
 
 proc openTransactionDb*(
     path: string,
-    dbOpts = defaultDbOptions(),
-    txDbOpts = defaultTransactionDbOptions(),
-    columnFamilies: openArray[ColFamilyDescriptor] = []): RocksDBResult[TransactionDbRef] =
+    dbOpts = DbOptionsRef(nil);
+    txDbOpts = TransactionDbOptionsRef(nil);
+    columnFamilies: openArray[ColFamilyDescriptor] = [];
+      ): RocksDBResult[TransactionDbRef] =
   ## Open a `TransactionDbRef` with the given options and column families.
   ## If no column families are provided the default column family will be used.
   ## If no options are provided the default options will be used.
+  let
+    useDbOpts = (if dbOpts.isNil: defaultDbOptions() else: dbOpts)
+  defer:
+    if dbOpts.isNil: useDbOpts.close()
 
   var cfs = columnFamilies.toSeq()
   if DEFAULT_COLUMN_FAMILY_NAME notin columnFamilies.mapIt(it.name()):
@@ -65,7 +70,7 @@ proc openTransactionDb*(
     errors: cstring
 
   let txDbPtr = rocksdb_transactiondb_open_column_families(
-        dbOpts.cPtr,
+        useDbOpts.cPtr,
         txDbOpts.cPtr,
         path.cstring,
         cfNames.len().cint,
@@ -75,7 +80,10 @@ proc openTransactionDb*(
         cast[cstringArray](errors.addr))
   bailOnErrors(errors)
 
-  let db = TransactionDbRef(
+  let
+    dbOpts = useDbOpts # don't close on exit
+    txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() else: txDbOpts)
+    db = TransactionDbRef(
       lock: createLock(),
       cPtr: txDbPtr,
       path: path,
@@ -89,15 +97,21 @@ proc isClosed*(db: TransactionDbRef): bool {.inline.} =
   db.cPtr.isNil()
 
 proc beginTransaction*(
-    db: TransactionDbRef,
-    readOpts = defaultReadOptions(),
-    writeOpts = defaultWriteOptions(),
-    txOpts = defaultTransactionOptions(),
-    columnFamily = DEFAULT_COLUMN_FAMILY_NAME): TransactionRef =
+    db: TransactionDbRef;
+    readOpts = ReadOptionsRef(nil);
+    writeOpts = WriteOptionsRef(nil);
+    txDbOpts = TransactionDbOptionsRef(nil);
+    txOpts = defaultTransactionOptions();
+    columnFamily = DEFAULT_COLUMN_FAMILY_NAME;
+      ): TransactionRef =
   ## Begin a new transaction against the database. The transaction will default
   ## to using the specified column family. If no column family is specified
   ## then the default column family will be used.
   doAssert not db.isClosed()
+  let
+    txDbOpts = (if txDbOpts.isNil: defaultTransactionDbOptions() else: txDbOpts)
+    readOpts = (if readOpts.isNil: defaultReadOptions() else: readOpts)
+    writeOpts = (if writeOpts.isNil: defaultWriteOptions() else: writeOpts)
 
   let txPtr = rocksdb_transaction_begin(
         db.cPtr,