From d503bcd176121e58d5960bd8de4d0d73cd928222 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 20 Sep 2022 12:33:24 -0600 Subject: [PATCH] Revert "Refactor datastore to prepare for proper query support" --- datastore/datastore.nim | 48 +- datastore/filesystem_datastore.nim | 162 +++++ datastore/fsds.nim | 158 ----- datastore/key.nim | 134 ++-- datastore/null_datastore.nim | 47 ++ datastore/query.nim | 40 +- datastore/sql.nim | 3 - datastore/sql/sqliteds.nim | 155 ----- datastore/sql/sqlitedsdb.nim | 264 -------- datastore/{sql/sqliteutils.nim => sqlite.nim} | 6 +- datastore/sqlite_datastore.nim | 413 ++++++++++++ .../{tieredds.nim => tiered_datastore.nim} | 30 +- tests/datastore/basictests.nim | 34 - tests/datastore/sql/testsqliteds.nim | 350 ----------- tests/datastore/sql/testsqlitedsdb.nim | 161 ----- tests/datastore/templates.nim | 1 + .../{testdatastore.nim => test_datastore.nim} | 14 +- tests/datastore/test_filesystem_datastore.nim | 196 ++++++ tests/datastore/{testkey.nim => test_key.nim} | 16 +- tests/datastore/test_null_datastore.nim | 44 ++ tests/datastore/test_sqlite_datastore.nim | 586 ++++++++++++++++++ tests/datastore/test_tiered_datastore.nim | 154 +++++ tests/datastore/testfsds.nim | 84 --- tests/datastore/testsql.nim | 4 - tests/datastore/testtieredds.nim | 159 ----- tests/test_all.nim | 9 + tests/testall.nim | 8 - 27 files changed, 1741 insertions(+), 1539 deletions(-) create mode 100644 datastore/filesystem_datastore.nim delete mode 100644 datastore/fsds.nim create mode 100644 datastore/null_datastore.nim delete mode 100644 datastore/sql.nim delete mode 100644 datastore/sql/sqliteds.nim delete mode 100644 datastore/sql/sqlitedsdb.nim rename datastore/{sql/sqliteutils.nim => sqlite.nim} (99%) create mode 100644 datastore/sqlite_datastore.nim rename datastore/{tieredds.nim => tiered_datastore.nim} (79%) delete mode 100644 tests/datastore/basictests.nim delete mode 100644 tests/datastore/sql/testsqliteds.nim delete mode 100644 tests/datastore/sql/testsqlitedsdb.nim create mode 100644 tests/datastore/templates.nim rename tests/datastore/{testdatastore.nim => test_datastore.nim} (69%) create mode 100644 tests/datastore/test_filesystem_datastore.nim rename tests/datastore/{testkey.nim => test_key.nim} (95%) create mode 100644 tests/datastore/test_null_datastore.nim create mode 100644 tests/datastore/test_sqlite_datastore.nim create mode 100644 tests/datastore/test_tiered_datastore.nim delete mode 100644 tests/datastore/testfsds.nim delete mode 100644 tests/datastore/testsql.nim delete mode 100644 tests/datastore/testtieredds.nim create mode 100644 tests/test_all.nim delete mode 100644 tests/testall.nim diff --git a/datastore/datastore.nim b/datastore/datastore.nim index 139ca29..dd04725 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -11,29 +11,35 @@ export key, query push: {.upraises: [].} type - DatastoreError* = object of CatchableError - DatastoreKeyNotFound* = object of DatastoreError - - CodexResult*[T] = Result[T, ref DatastoreError] Datastore* = ref object of RootObj -method contains*(self: Datastore, key: Key): Future[?!bool] {.base, locks: "unknown".} = - raiseAssert("Not implemented!") - -method delete*(self: Datastore, key: Key): Future[?!void] {.base, locks: "unknown".} = - raiseAssert("Not implemented!") - -method get*(self: Datastore, key: Key): Future[?!seq[byte]] {.base, locks: "unknown".} = - raiseAssert("Not implemented!") - -method put*(self: Datastore, key: Key, data: seq[byte]): Future[?!void] {.base, locks: "unknown".} = - raiseAssert("Not implemented!") - -method close*(self: Datastore): Future[?!void] {.base, async, locks: "unknown".} = - return success() - -method query*( +method contains*( self: Datastore, - query: Query): Future[QueryIter] {.gcsafe.} = + key: Key): Future[?!bool] {.async, base, locks: "unknown".} = + + raiseAssert("Not implemented!") + +method delete*( + self: Datastore, + key: Key): Future[?!void] {.async, base, locks: "unknown".} = + + raiseAssert("Not implemented!") + +method get*( + self: Datastore, + key: Key): Future[?!(?seq[byte])] {.async, base, locks: "unknown".} = + + raiseAssert("Not implemented!") + +method put*( + self: Datastore, + key: Key, + data: seq[byte]): Future[?!void] {.async, base, locks: "unknown".} = + + raiseAssert("Not implemented!") + +iterator query*( + self: Datastore, + query: Query): Future[QueryResponse] = raiseAssert("Not implemented!") diff --git a/datastore/filesystem_datastore.nim b/datastore/filesystem_datastore.nim new file mode 100644 index 0000000..51c50a0 --- /dev/null +++ b/datastore/filesystem_datastore.nim @@ -0,0 +1,162 @@ +import std/os + +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +from pkg/stew/results as stewResults import get, isErr +import pkg/upraises + +import ./datastore + +export datastore + +push: {.upraises: [].} + +type + FileSystemDatastore* = ref object of Datastore + root: string + +const + objExt* = ".dsobject" + +proc new*( + T: type FileSystemDatastore, + root: string): ?!T = + + try: + let + root = if root.isAbsolute: root + else: getCurrentDir() / root + + if not dirExists(root): + failure "directory does not exist: " & root + else: + success T(root: root) + + except OSError as e: + failure e + +proc root*(self: FileSystemDatastore): string = + self.root + +proc path*( + self: FileSystemDatastore, + key: Key): string = + + var + segments: seq[string] + + for ns in key: + without field =? ns.field: + segments.add ns.value + continue + + segments.add(field / ns.value) + + # is it problematic that per this logic Key(/a:b) evaluates to the same path + # as Key(/a/b)? may need to check if/how other Datastore implementations + # distinguish them + + self.root / joinPath(segments) & objExt + +method contains*( + self: FileSystemDatastore, + key: Key): Future[?!bool] {.async, locks: "unknown".} = + + return success fileExists(self.path(key)) + +method delete*( + self: FileSystemDatastore, + key: Key): Future[?!void] {.async, locks: "unknown".} = + + let + path = self.path(key) + + try: + removeFile(path) + return success() + + # removing an empty directory might lead to surprising behavior depending + # on what the user specified as the `root` of the FileSystemDatastore, so + # until further consideration, empty directories will be left in place + + except OSError as e: + return failure e + +method get*( + self: FileSystemDatastore, + key: Key): Future[?!(?seq[byte])] {.async, locks: "unknown".} = + + # to support finer control of memory allocation, maybe could/should change + # the signature of `get` so that it has a 3rd parameter + # `bytes: var openArray[byte]` and return type `?!bool`; this variant with + # return type `?!(?seq[byte])` would be a special case (convenience method) + # calling the former after allocating a seq with size automatically + # determined via `getFileSize` + + let + path = self.path(key) + containsRes = await self.contains(key) + + if containsRes.isErr: return failure containsRes.error.msg + + if containsRes.get: + var + file: File + + if not file.open(path): + return failure "unable to open file: " & path + else: + try: + let + size = file.getFileSize + + var + bytes: seq[byte] + + if size > 0: + newSeq(bytes, size) + + let + bytesRead = file.readBytes(bytes, 0, size) + + if bytesRead < size: + return failure $bytesRead & " bytes were read from " & path & + " but " & $size & " bytes were expected" + + return success bytes.some + + except IOError as e: + return failure e + + finally: + file.close + + else: + return success seq[byte].none + +method put*( + self: FileSystemDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async, locks: "unknown".} = + + let + path = self.path(key) + + try: + createDir(parentDir(path)) + if data.len > 0: writeFile(path, data) + else: writeFile(path, "") + return success() + + except IOError as e: + return failure e + + except OSError as e: + return failure e + +# method query*( +# self: FileSystemDatastore, +# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = +# +# return success ....some diff --git a/datastore/fsds.nim b/datastore/fsds.nim deleted file mode 100644 index 9a5b794..0000000 --- a/datastore/fsds.nim +++ /dev/null @@ -1,158 +0,0 @@ -import std/os -import std/options - -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -from pkg/stew/results as stewResults import get, isErr -import pkg/upraises - -import ./datastore - -export datastore - -push: {.upraises: [].} - -type - FSDatastore* = ref object of Datastore - root*: string - ignoreProtected: bool - depth: int - -template path*(self: FSDatastore, key: Key): string = - var - segments: seq[string] - - for ns in key: - if ns.field == "": - segments.add ns.value - continue - - # `:` are replaced with `/` - segments.add(ns.field / ns.value) - - self.root / segments.joinPath() - -template validDepth*(self: FSDatastore, key: Key): bool = - key.len <= self.depth - -method contains*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = - - if not self.validDepth(key): - return failure "Path has invalid depth!" - - let - path = self.path(key) - - return success fileExists(path) - -method delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = - - if not self.validDepth(key): - return failure "Path has invalid depth!" - - let - path = self.path(key) - - try: - removeFile(path) - return success() - - # removing an empty directory might lead to surprising behavior depending - # on what the user specified as the `root` of the FSDatastore, so - # until further consideration, empty directories will be left in place - - except OSError as e: - return failure e - -method get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = - - # to support finer control of memory allocation, maybe could/should change - # the signature of `get` so that it has a 3rd parameter - # `bytes: var openArray[byte]` and return type `?!bool`; this variant with - # return type `?!(?seq[byte])` would be a special case (convenience method) - # calling the former after allocating a seq with size automatically - # determined via `getFileSize` - - if not self.validDepth(key): - return failure "Path has invalid depth!" - - let - path = self.path(key) - - if not fileExists(path): - return failure(newException(DatastoreKeyNotFound, "Key doesn't exist")) - - var - file: File - - defer: - file.close - - if not file.open(path): - return failure "unable to open file: " & path - - try: - let - size = file.getFileSize - - var - bytes = newSeq[byte](size) - read = 0 - - while read < size: - read += file.readBytes(bytes, read, size) - - if read < size: - return failure $read & " bytes were read from " & path & - " but " & $size & " bytes were expected" - - return success bytes - - except CatchableError as e: - return failure e - -method put*( - self: FSDatastore, - key: Key, - data: seq[byte]): Future[?!void] {.async, locks: "unknown".} = - - if not self.validDepth(key): - return failure "Path has invalid depth!" - - let - path = self.path(key) - - try: - createDir(parentDir(path)) - writeFile(path, data) - except CatchableError as e: - return failure e - - return success() - -# method query*( -# self: FSDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....some - -proc new*( - T: type FSDatastore, - root: string, - depth = 2, - caseSensitive = true, - ignoreProtected = false): ?!T = - - let root = ? ( - block: - if root.isAbsolute: root - else: getCurrentDir() / root).catch - - if not dirExists(root): - return failure "directory does not exist: " & root - - success T( - root: root, - ignoreProtected: ignoreProtected, - depth: depth) diff --git a/datastore/key.nim b/datastore/key.nim index ee0ab1d..44d6480 100644 --- a/datastore/key.nim +++ b/datastore/key.nim @@ -22,59 +22,59 @@ type namespaces*: seq[Namespace] const - Delimiter* = ":" - Separator* = "/" + delimiter = ":" + separator = "/" # TODO: operator/s for combining string|Namespace,string|Namespace # TODO: lifting from ?![Namespace|Key] for various ops -func init*( +proc init*( T: type Namespace, field, value: string): ?!T = if value.strip == "": return failure "value string must not be all whitespace or empty" - if value.contains(Delimiter): - return failure "value string must not contain Delimiter \"" & - Delimiter & "\"" + if value.contains(delimiter): + return failure "value string must not contain delimiter \"" & + delimiter & "\"" - if value.contains(Separator): - return failure "value string must not contain Separator \"" & - Separator & "\"" + if value.contains(separator): + return failure "value string must not contain separator \"" & + separator & "\"" if field != "": if field.strip == "": return failure "field string must not be all whitespace" - if field.contains(Delimiter): - return failure "field string must not contain Delimiter \"" & - Delimiter & "\"" + if field.contains(delimiter): + return failure "field string must not contain delimiter \"" & + delimiter & "\"" - if field.contains(Separator): - return failure "field string must not contain Separator \"" & - Separator & "\"" + if field.contains(separator): + return failure "field string must not contain separator \"" & + separator & "\"" success T(field: field, value: value) -func init*(T: type Namespace, id: string): ?!T = +proc init*(T: type Namespace, id: string): ?!T = if id.strip == "": return failure "id string must not be all whitespace or empty" - if id.contains(Separator): - return failure "id string must not contain Separator \"" & Separator & "\"" + if id.contains(separator): + return failure "id string must not contain separator \"" & separator & "\"" - if id == Delimiter: - return failure "value in id string \"[field]" & Delimiter & + if id == delimiter: + return failure "value in id string \"[field]" & delimiter & "[value]\" must not be empty" - if id.count(Delimiter) > 1: - return failure "id string must not contain more than one Delimiter \"" & - Delimiter & "\"" + if id.count(delimiter) > 1: + return failure "id string must not contain more than one delimiter \"" & + delimiter & "\"" let (field, value) = block: - let parts = id.split(Delimiter) + let parts = id.split(delimiter) if parts.len > 1: (parts[0], parts[^1]) else: @@ -82,25 +82,22 @@ func init*(T: type Namespace, id: string): ?!T = T.init(field, value) -func id*(self: Namespace): string = +proc id*(self: Namespace): string = if self.field.len > 0: - self.field & Delimiter & self.value + self.field & delimiter & self.value else: self.value -func hash*(namespace: Namespace): Hash = - hash(namespace.id) - -func `$`*(namespace: Namespace): string = +proc `$`*(namespace: Namespace): string = "Namespace(" & namespace.id & ")" -func init*(T: type Key, namespaces: varargs[Namespace]): ?!T = +proc init*(T: type Key, namespaces: varargs[Namespace]): ?!T = if namespaces.len == 0: failure "namespaces must contain at least one Namespace" else: success T(namespaces: @namespaces) -func init*(T: type Key, namespaces: varargs[string]): ?!T = +proc init*(T: type Key, namespaces: varargs[string]): ?!T = if namespaces.len == 0: failure "namespaces must contain at least one Namespace id string" else: @@ -109,7 +106,7 @@ func init*(T: type Key, namespaces: varargs[string]): ?!T = ?Namespace.init(it) )) -func init*(T: type Key, id: string): ?!T = +proc init*(T: type Key, id: string): ?!T = if id == "": return failure "id string must contain at least one Namespace" @@ -117,15 +114,15 @@ func init*(T: type Key, id: string): ?!T = return failure "id string must not be all whitespace" let - nsStrs = id.split(Separator).filterIt(it != "") + nsStrs = id.split(separator).filterIt(it != "") if nsStrs.len == 0: - return failure "id string must not contain more than one Separator " & - "\"" & Separator & "\"" + return failure "id string must not contain only one or more separator " & + "\"" & separator & "\"" Key.init(nsStrs) -func list*(self: Key): seq[Namespace] = +proc list*(self: Key): seq[Namespace] = self.namespaces proc random*(T: type Key): string = @@ -134,91 +131,78 @@ proc random*(T: type Key): string = template `[]`*(key: Key, x: auto): auto = key.namespaces[x] -func len*(self: Key): int = +proc len*(self: Key): int = self.namespaces.len iterator items*(key: Key): Namespace = for k in key.namespaces: yield k -func reversed*(self: Key): Key = +proc reversed*(self: Key): Key = Key(namespaces: self.namespaces.reversed) -func reverse*(self: Key): Key = +proc reverse*(self: Key): Key = self.reversed -func name*(self: Key): string = +proc name*(self: Key): string = if self.len > 0: return self[^1].value -func `type`*(self: Key): string = +proc `type`*(self: Key): string = if self.len > 0: return self[^1].field -func id*(self: Key): string = - Separator & self.namespaces.mapIt(it.id).join(Separator) +proc id*(self: Key): string = + separator & self.namespaces.mapIt(it.id).join(separator) -func root*(self: Key): bool = +proc isTopLevel*(self: Key): bool = self.len == 1 -func parent*(self: Key): ?!Key = - if self.root: +proc parent*(self: Key): ?!Key = + if self.isTopLevel: failure "key has no parent" else: success Key(namespaces: self.namespaces[0..^2]) -func path*(self: Key): ?!Key = +proc path*(self: Key): ?!Key = let - parent = ?self.parent + parent = ? self.parent if self[^1].field == "": return success parent - let ns = parent.namespaces & @[Namespace(value: self[^1].field)] - success Key(namespaces: ns) + success Key(namespaces: parent.namespaces & @[Namespace(value: self[^1].field)]) -func child*(self: Key, ns: Namespace): Key = +proc child*(self: Key, ns: Namespace): Key = Key(namespaces: self.namespaces & @[ns]) -func `/`*(self: Key, ns: Namespace): Key = +proc `/`*(self: Key, ns: Namespace): Key = self.child(ns) -func child*(self: Key, namespaces: varargs[Namespace]): Key = +proc child*(self: Key, namespaces: varargs[Namespace]): Key = Key(namespaces: self.namespaces & @namespaces) -func child*(self, key: Key): Key = +proc child*(self, key: Key): Key = Key(namespaces: self.namespaces & key.namespaces) -func `/`*(self, key: Key): Key = +proc `/`*(self, key: Key): Key = self.child(key) -func child*(self: Key, keys: varargs[Key]): Key = +proc child*(self: Key, keys: varargs[Key]): Key = Key(namespaces: self.namespaces & concat(keys.mapIt(it.namespaces))) -func child*(self: Key, ids: varargs[string]): ?!Key = +proc child*(self: Key, ids: varargs[string]): ?!Key = success self.child(ids.filterIt(it != "").mapIt( ?Key.init(it) )) -func relative*(self: Key, parent: Key): ?!Key = - ## Get a key relative to parent from current key - ## - - if self.len < parent.len: - return failure "Not a parent of this key!" - - Key.init(self.namespaces[parent.namespaces.high..self.namespaces.high]) - -func `/`*(self: Key, id: string): ?!Key = +proc `/`*(self: Key, id: string): ?!Key = self.child(id) -func ancestor*(self, other: Key): bool = +proc isAncestorOf*(self, other: Key): bool = if other.len <= self.len: false else: other.namespaces[0..