From 8c7165559355b31872ace8755f502ac4e28ac8fe Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Sep 2023 23:07:52 -0700 Subject: [PATCH] add index --- datastore/backend.nim | 13 +++++++++++-- datastore/sql/sqliteds.nim | 2 +- datastore/sql/sqlitedsdb.nim | 4 +++- datastore/sql/sqliteutils.nim | 6 +++--- datastore/threads/databuffer.nim | 8 ++++++++ tests/datastore/sql/testsqliteds.nim | 20 ++++++++++++++++---- tests/datastore/testdatabuffer.nim | 6 ++++++ 7 files changed, 48 insertions(+), 11 deletions(-) diff --git a/datastore/backend.nim b/datastore/backend.nim index b5135f7..3d757bc 100644 --- a/datastore/backend.nim +++ b/datastore/backend.nim @@ -34,7 +34,16 @@ type proc `$`*(id: KeyId): string = $(id.data) proc new*(tp: typedesc[KeyId], id: cstring): KeyId = - KeyId(data: DataBuffer.new(id.pointer, 0, id.len()-1)) + ## copy cstring including null terminator + KeyId(data: DataBuffer.new(id.pointer, 0, id.len())) + +proc new*(tp: typedesc[KeyId], id: string): KeyId = + ## copy cstring including null terminator + KeyId(data: DataBuffer.new(id, {dbNullTerminate})) + +proc toCString*(key: KeyId): cstring = + ## copy cstring including null terminator + cast[cstring](baseAddr key.data) proc toDb*(key: Key): DbKey {.inline, raises: [].} = let id: string = key.id() @@ -42,7 +51,7 @@ proc toDb*(key: Key): DbKey {.inline, raises: [].} = db.setData(id) DbKey(data: db) -proc toKey*(key: DbKey): Key {.inline, raises: [].} = +proc toKey*(key: KeyId): Key {.inline, raises: [].} = Key.init(key.data).expect("expected valid key here for but got `" & $key.data & "`") template toOpenArray*(x: DbKey): openArray[char] = diff --git a/datastore/sql/sqliteds.nim b/datastore/sql/sqliteds.nim index 2126fbe..66beed1 100644 --- a/datastore/sql/sqliteds.nim +++ b/datastore/sql/sqliteds.nim @@ -82,7 +82,7 @@ proc put*(self: SQLiteDatastore, key: DbKey, data: DbVal): ?!void = when DbVal is seq[byte]: return self.db.putStmt.exec((key, data, timestamp())) elif DbVal is DataBuffer: - return self.db.putBufferStmt.exec((key.id, data, timestamp())) + return self.db.putBufferStmt.exec((key, data, timestamp())) else: {.error: "unknown type".} diff --git a/datastore/sql/sqlitedsdb.nim b/datastore/sql/sqlitedsdb.nim index 572bc44..ea06e50 100644 --- a/datastore/sql/sqlitedsdb.nim +++ b/datastore/sql/sqlitedsdb.nim @@ -21,7 +21,7 @@ type DeleteStmt* = SQLiteStmt[(string), void] GetStmt* = SQLiteStmt[(string), void] PutStmt* = SQLiteStmt[(string, seq[byte], int64), void] - PutBufferStmt* = SQLiteStmt[(string, DataBuffer, int64), void] + PutBufferStmt* = SQLiteStmt[(KeyId, DataBuffer, int64), void] QueryStmt* = SQLiteStmt[(string), void] BeginStmt* = NoParamsStmt EndStmt* = NoParamsStmt @@ -36,6 +36,7 @@ type getDataCol*: (RawStmtPtr, int) getStmt*: GetStmt putStmt*: PutStmt + putBufferStmt*: PutBufferStmt beginStmt*: BeginStmt endStmt*: EndStmt rollbackStmt*: RollbackStmt @@ -317,6 +318,7 @@ proc open*( getStmt: getStmt, getDataCol: getDataCol, putStmt: putStmt, + putBufferStmt: putBufferStmt, beginStmt: beginStmt, endStmt: endStmt, rollbackStmt: rollbackStmt) diff --git a/datastore/sql/sqliteutils.nim b/datastore/sql/sqliteutils.nim index d070ce2..213438f 100644 --- a/datastore/sql/sqliteutils.nim +++ b/datastore/sql/sqliteutils.nim @@ -46,7 +46,7 @@ proc bindParam( n: int, val: auto): cint = - when val is openArray[byte]|seq[byte]: + when val is openArray[byte]|seq[byte]|DataBuffer: if val.len > 0: # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior # to the return from sqlite3_bind_*(). The object and pointer to it @@ -70,13 +70,13 @@ proc bindParam( # remain valid until then. SQLite will then manage the lifetime of its # private copy." sqlite3_bind_text(s, n.cint, val.cstring, -1.cint, SQLITE_TRANSIENT) - elif val is DbKey: + elif val is KeyId: # `-1` implies string length is num bytes up to first null-terminator; # `SQLITE_TRANSIENT` "indicate[s] that the object is to be copied prior # to the return from sqlite3_bind_*(). The object and pointer to it must # remain valid until then. SQLite will then manage the lifetime of its # private copy." - sqlite3_bind_text(s, n.cint, val.cstring, -1.cint, SQLITE_TRANSIENT) + sqlite3_bind_text(s, n.cint, val.toCString(), -1.cint, SQLITE_TRANSIENT) else: {.fatal: "Please add support for the '" & $typeof(val) & "' type".} diff --git a/datastore/threads/databuffer.nim b/datastore/threads/databuffer.nim index 561d41a..cae4b65 100644 --- a/datastore/threads/databuffer.nim +++ b/datastore/threads/databuffer.nim @@ -35,6 +35,11 @@ proc isNil*(a: DataBuffer): bool = smartptrs.isNil(a) proc hash*(a: DataBuffer): Hash = a[].buf.toOpenArray(0, a[].size-1).hash() +proc `[]`*(db: DataBuffer, idx: int): byte = + if idx >= db.len(): + raise newException(IndexDefect, "index out of bounds") + db[].buf[idx] + proc `==`*(a, b: DataBuffer): bool = if a.isNil and b.isNil: return true elif a.isNil or b.isNil: return false @@ -71,6 +76,9 @@ proc new*[T: byte | char](tp: type DataBuffer, data: openArray[T], opts: set[Dat proc new*(tp: type DataBuffer, data: pointer, first, last: int): DataBuffer = DataBuffer.new(toOpenArray(cast[ptr UncheckedArray[byte]](data), first, last)) +proc baseAddr*(db: DataBuffer): pointer = + db[].buf + proc clear*(db: DataBuffer) = zeroMem(db[].buf, db[].cap) db[].size = 0 diff --git a/tests/datastore/sql/testsqliteds.nim b/tests/datastore/sql/testsqliteds.nim index 38b83fc..240892e 100644 --- a/tests/datastore/sql/testsqliteds.nim +++ b/tests/datastore/sql/testsqliteds.nim @@ -14,11 +14,11 @@ import pkg/datastore/key import ../dscommontests import ../querycommontests -proc testBasic( +proc testBasic[K, V]( ds: SQLiteDatastore, - key: string, - bytes: seq[byte], - otherBytes: seq[byte], + key: K, + bytes: V, + otherBytes: V, ) = test "put": @@ -84,3 +84,15 @@ suite "Test Basic SQLiteDatastore": ds.close().tryGet() testBasic(ds, key, bytes, otherBytes) + +suite "Test DataBuffer SQLiteDatastore": + let + ds = SQLiteDatastore.new(Memory).tryGet() + key = KeyId.new Key.init("a:b/c/d:e").tryGet().id() + bytes = DataBuffer.new "some bytes" + otherBytes = DataBuffer.new "some other bytes" + + suiteTeardown: + ds.close().tryGet() + + testBasic(ds, key, bytes, otherBytes) diff --git a/tests/datastore/testdatabuffer.nim b/tests/datastore/testdatabuffer.nim index 0c6fa46..5ec8127 100644 --- a/tests/datastore/testdatabuffer.nim +++ b/tests/datastore/testdatabuffer.nim @@ -88,6 +88,12 @@ suite "Share buffer test": test "toString": check $a == "/a/b" + test "index": + check c[0] == '/'.byte + check c[1] == 'a'.byte + expect IndexDefect: + check c[2] == 'c'.byte + test "hash": check a.hash() == b.hash()