diff --git a/.gitignore b/.gitignore index cbbffb0..854fcb5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ nimcache/ tests/packagetest/packagetest src/leveldb tests/test +src/htmldocs/ diff --git a/README.md b/README.md index d54a1e1..8ced596 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,46 @@ # leveldb.nim -LevelDB wrapper for Nim +A LevelDB wrapper for Nim in a Nim friendly way. + +Create a database: +```Nim + import leveldb + import options + + var db = leveldb.open("/tmp/mydata") +``` + +Read or modify the database content: +```Nim + assert db.getOrDefault("nothing", "") == "" + + db.put("hello", "world") + db.put("bin", "GIF89a\1\0") + echo db.get("hello") + assert db.get("hello").isSome() + + var key, val = "" + for key, val in db.iter(): + echo key, ": ", repr(val) + + db.delete("hello") + assert db.get("hello").isNone() +``` + +Batch writes: +```Nim + let batch = newBatch() + for i in 1..10: + batch.put("key" & $i, $i) + batch.delete("bin") + db.write(batch) +``` + +Iterate over subset of database content: +```Nim + for key, val in db.iterPrefix(prefix = "key1"): + echo key, ": ", val + for key, val in db.iter(seek = "key3", reverse = true): + echo key, ": ", val + + db.close() +``` diff --git a/src/leveldb.nim b/src/leveldb.nim index 1303259..b09794d 100644 --- a/src/leveldb.nim +++ b/src/leveldb.nim @@ -1,3 +1,56 @@ +## A LevelDB_ wrapper for Nim in a Nim friendly way. +## +## LevelDB is a fast and simple key/value data storage library built +## by Google that provides an ordered mapping from string keys to +## string values. +## +## .. _LevelDB: https://github.com/google/leveldb +## +## Create a database: +## +## .. code-block:: Nim +## import leveldb +## import options +## +## var db = leveldb.open("/tmp/mydata") +## +## Read or modify the database content: +## +## .. code-block:: Nim +## +## assert db.getOrDefault("nothing", "") == "" +## +## db.put("hello", "world") +## db.put("bin", "GIF89a\1\0") +## echo db.get("hello") +## assert db.get("hello").isSome() +## +## var key, val = "" +## for key, val in db.iter(): +## echo key, ": ", repr(val) +## +## db.delete("hello") +## assert db.get("hello").isNone() +## +## Batch writes: +## +## .. code-block:: Nim +## let batch = newBatch() +## for i in 1..10: +## batch.put("key" & $i, $i) +## batch.delete("bin") +## db.write(batch) +## +## Iterate over subset of database content: +## +## .. code-block:: Nim +## for key, val in db.iterPrefix(prefix = "key1"): +## echo key, ": ", val +## for key, val in db.iter(seek = "key3", reverse = true): +## echo key, ": ", val +## +## db.close() + import options, os, strutils import leveldb/raw @@ -5,15 +58,19 @@ type LevelDb* = ref object path*: string db: ptr leveldb_t + cache: ptr leveldb_cache_t + readOptions: ptr leveldb_readoptions_t syncWriteOptions: ptr leveldb_writeoptions_t asyncWriteOptions: ptr leveldb_writeoptions_t - readOptions: ptr leveldb_readoptions_t - cache: ptr leveldb_cache_t - LevelDbBatch* = ref object + LevelDbWriteBatch* = ref object + ## Write batches for bulk data modification. batch: ptr leveldb_writebatch_t CompressionType* = enum + ## No compression or using Snappy_ algorithm (default). + ## + ## .. _Snappy: http://google.github.io/snappy/ ctNoCompression = leveldb_no_compression, ctSnappyCompression = leveldb_snappy_compression @@ -49,10 +106,15 @@ proc checkError(errPtr: cstring) = raise newException(LevelDbException, $errPtr) proc getLibVersion*(): (int, int) = + ## Get the version of leveldb C library. result[0] = leveldb_major_version() result[1] = leveldb_minor_version() proc close*(self: LevelDb) = + ## Closes the database. + ## + ## See also: + ## * `open proc <#open%2Cstring%2Cint%2Cint%2Cint>`_ if self.db == nil: return leveldb_close(self.db) @@ -69,6 +131,12 @@ proc open*(path: string, create = true, reuse = true, paranoidChecks = true, cacheCapacity = 0, blockSize = 4 * 1024, writeBufferSize = 4*1024*1024, maxOpenFiles = 1000, maxFileSize = 2 * 1024 * 1024, blockRestartInterval = 16): LevelDb = + ## Opens a database. + ## + ## Raises `LevelDbException` if corruption detected in the database. + ## + ## See also: + ## * `close proc <#close%2CLevelDb>`_ new(result, close) let options = leveldb_options_create() @@ -111,7 +179,20 @@ proc open*(path: string, create = true, reuse = true, paranoidChecks = true, result.db = leveldb_open(options, path, addr errPtr) checkError(errPtr) -proc put*(self: LevelDb, key: string, value: string, sync = true) = +proc put*(self: LevelDb, key: string, value: string, sync = false) = + ## Set a `value` for the specified `key`. + ## + ## By default, `sync` is turned off, each write to leveldb is asynchronous. + ## Unless reboot, a crash of just the writing process will not cause any + ## loss since even when `sync` is false. + ## + ## See also: + ## * `put proc <#put%2CLevelDbWriteBatch%2Cstring%2Cstring>`_ + runnableExamples: + let db = leveldb.open("/tmp/test") + db.put("hello", "world") + db.close() + assert self.db != nil var errPtr: cstring = nil let writeOptions = if sync: self.syncWriteOptions else: self.asyncWriteOptions @@ -127,6 +208,16 @@ proc newString(cstr: cstring, length: csize): string = result = "" proc get*(self: LevelDb, key: string): Option[string] = + ## Get the value for the specified `key`. + ## + ## See also: + ## * `getOrDefault proc <#getOrDefault%2CLevelDb%2Cstring>`_ + runnableExamples: + let db = leveldb.open("/tmp/test") + db.put("hello", "world") + echo db.get("hello") + db.close() + var size: csize var errPtr: cstring = nil let s = leveldb_get(self.db, self.readOptions, key, key.len, addr size, addr errPtr) @@ -138,35 +229,104 @@ proc get*(self: LevelDb, key: string): Option[string] = result = some(newString(s, size)) free(s) -proc delete*(self: LevelDb, key: string, sync = true) = +proc getOrDefault*(self: LevelDb, key: string, default = ""): string = + ## Get the value for the specified `key`, or `default` if no value was set. + ## + ## See also: + ## * `get proc <#get%2CLevelDb%2Cstring>`_ + runnableExamples: + let db = leveldb.open("/tmp/test") + doAssert db.getOrDefault("what?", "nothing") == "nothing" + db.close() + + let val = self.get(key) + if val.isNone(): + result = default + else: + result = val.get() + +proc delete*(self: LevelDb, key: string, sync = false) = + ## Delete the key/value pair for the specified key. + ## + ## See also: + ## * `delete proc <#delete%2CLevelDbWriteBatch%2Cstring>`_ var errPtr: cstring = nil let writeOptions = if sync: self.syncWriteOptions else: self.asyncWriteOptions leveldb_delete(self.db, writeOptions, key, key.len, addr errPtr) checkError(errPtr) -proc destroy*(self: LevelDbBatch) = +proc destroy*(self: LevelDbWriteBatch) = + ## Destroys this batch. + ## + ## See also: + ## * `newBatch proc <#newBatch>`_ if self.batch == nil: return leveldb_writebatch_destroy(self.batch) self.batch = nil -proc newBatch*(): LevelDbBatch = +proc newBatch*(): LevelDbWriteBatch = + ## Creates a new database write batch. + ## + ## See also: + ## * `write proc <#write%2CLevelDb%2CLevelDbWriteBatch>`_ + ## * `put proc <#put%2CLevelDbWriteBatch%2Cstring%2Cstring>`_ + ## * `delete proc <#delete%2CLevelDbWriteBatch%2Cstring>`_ + ## * `append proc <#append%2CLevelDbWriteBatch%2CLevelDbWriteBatch>`_ + ## * `clear proc <#clear%2CLevelDbWriteBatch>`_ + ## * `destroy proc <#destroy%2CLevelDbWriteBatch>`_ + + runnableExamples: + let db = leveldb.open("/tmp/test") + let batch = newBatch() + for i in 1..10: + batch.put("key" & $i, $i) + batch.delete("another") + db.write(batch) + db.close() + new(result, destroy) result.batch = leveldb_writebatch_create() -proc put*(self: LevelDbBatch, key: string, value: string, sync = true) = +proc put*(self: LevelDbWriteBatch, key: string, value: string, sync = false) = + ## Set a `value` for the specified `key`. + ## Same as `put <#put%2CLevelDb%2Cstring%2Cstring>`_ but operates on the + ## write batch instead. + ## + ## See also: + ## * `put proc <#put%2CLevelDb%2Cstring%2Cstring>`_ + ## * `newBatch proc <#newBatch>`_ leveldb_writebatch_put(self.batch, key, key.len.csize, value, value.len.csize) -proc append*(self, source: LevelDbBatch) = +proc append*(self, source: LevelDbWriteBatch) = + ## Merges the `source` batch into this batch. + ## + ## See also: + ## * `newBatch proc <#newBatch>`_ leveldb_writebatch_append(self.batch, source.batch) -proc delete*(self: LevelDbBatch, key: string) = +proc delete*(self: LevelDbWriteBatch, key: string) = + ## Delete the key/value pair for the specified `key`. + ## Same as `delete <#delete%2CLevelDb%2Cstring>`_ but operates on the + ## write batch instead. + ## + ## See also: + ## * `delete proc <#delete%2CLevelDb%2Cstring>`_ + ## * `newBatch proc <#newBatch>`_ leveldb_writebatch_delete(self.batch, key, key.len.csize) -proc clear*(self: LevelDbBatch) = +proc clear*(self: LevelDbWriteBatch) = + ## Clear all updates buffered in this batch. + ## + ## See also: + ## * `newBatch proc <#newBatch>`_ leveldb_writebatch_clear(self.batch) -proc write*(self: LevelDb, batch: LevelDbBatch) = +proc write*(self: LevelDb, batch: LevelDbWriteBatch) = + ## Write apply the given `batch` to the database. + ## + ## See also: + ## * `newBatch proc <#newBatch>`_ var errPtr: cstring = nil leveldb_write(self.db, self.syncWriteOptions, batch.batch, addr errPtr) checkError(errPtr) @@ -183,6 +343,14 @@ proc getIterData(iterPtr: ptr leveldb_iterator_t): (string, string) = iterator iter*(self: LevelDb, seek: string = "", reverse: bool = false): ( string, string) = + ## Iterate all key/value pairs in the database from the first one or + ## the specified key `seek`. + ## By default, the ordering will be lexicographic byte-wise ordering + ## with leveldb builtin comparator, unless `reverse` set to `true`. + ## + ## See also: + ## * `iterPrefix iterator <#iterPrefix.i%2CLevelDb%2Cstring>`_ + ## * `iterRange iterator <#iterRange.i%2CLevelDb%2Cstring%2Cstring>`_ var iterPtr = leveldb_create_iterator(self.db, self.readOptions) defer: leveldb_iter_destroy(iterPtr) @@ -210,6 +378,11 @@ iterator iter*(self: LevelDb, seek: string = "", reverse: bool = false): ( leveldb_iter_next(iterPtr) iterator iterPrefix*(self: LevelDb, prefix: string): (string, string) = + ## Iterate subset key/value pairs in the database with a particular `prefix`. + ## + ## See also: + ## * `iter iterator <#iter.i%2CLevelDb%2Cstring%2Cbool>`_ + ## * `iterRange iterator <#iterRange.i%2CLevelDb%2Cstring%2Cstring>`_ for key, value in iter(self, prefix, reverse = false): if key.startsWith(prefix): yield (key, value) @@ -217,6 +390,12 @@ iterator iterPrefix*(self: LevelDb, prefix: string): (string, string) = break iterator iterRange*(self: LevelDb, start, limit: string): (string, string) = + ## Yields all key/value pairs between the `start` and `limit` keys + ## (inclusive) in the database. + ## + ## See also: + ## * `iter iterator <#iter.i%2CLevelDb%2Cstring%2Cbool>`_ + ## * `iterPrefix iterator <#iterPrefix.i%2CLevelDb%2Cstring>`_ let reverse: bool = limit < start for key, value in iter(self, start, reverse = reverse): if reverse: @@ -228,12 +407,14 @@ iterator iterRange*(self: LevelDb, start, limit: string): (string, string) = yield (key, value) proc removeDb*(name: string) = + ## Remove the database `name`. var err: cstring = nil let options = leveldb_options_create() leveldb_destroy_db(options, name, addr err) checkError(err) proc repairDb*(name: string) = + ## Repairs the corrupted database `name`. let options = leveldb_options_create() leveldb_options_set_create_if_missing(options, levelDbFalse) leveldb_options_set_error_if_exists(options, levelDbFalse) diff --git a/src/leveldb/raw.nim b/src/leveldb/raw.nim index 7b2ffa8..cae7c85 100644 --- a/src/leveldb/raw.nim +++ b/src/leveldb/raw.nim @@ -27,10 +27,10 @@ ## be true on entry: ## errptr == NULL ## errptr points to a malloc()ed null-terminated error message -## (On Windows, *errptr must have been malloc()-ed by this library.) -## On success, a leveldb routine leaves *errptr unchanged. -## On failure, leveldb frees the old value of *errptr and -## set *errptr to a malloc()ed error message. +## (On Windows, \*errptr must have been malloc()-ed by this library.) +## On success, a leveldb routine leaves \*errptr unchanged. +## On failure, leveldb frees the old value of \*errptr and +## set \*errptr to a malloc()ed error message. ## ## (4) Bools have the type uint8_t (0 == false; rest == true) ## @@ -66,7 +66,7 @@ proc leveldb_delete*(db: ptr leveldb_t; options: ptr leveldb_writeoptions_t; proc leveldb_write*(db: ptr leveldb_t; options: ptr leveldb_writeoptions_t; batch: ptr leveldb_writebatch_t; errptr: ptr cstring) {.importc.} ## Returns NULL if not found. A malloc()ed array otherwise. -## Stores the length of the array in *vallen. +## Stores the length of the array in \*vallen. proc leveldb_get*(db: ptr leveldb_t; options: ptr leveldb_readoptions_t; key: cstring; keylen: csize; vallen: ptr csize; errptr: ptr cstring): cstring {.importc.} diff --git a/tests/test.nim b/tests/test.nim index dad2a3a..4b5b7ab 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -65,6 +65,9 @@ suite "leveldb": db.put("hello", "world") check(db.get("hello") == some("world")) + test "get or default": + check(db.getOrDefault("nothing", "yes") == "yes") + test "delete": db.put("hello", "world") db.delete("hello")