diff --git a/datastore/datastore.nim b/datastore/datastore.nim index dd04725..139ca29 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -11,35 +11,29 @@ export key, query push: {.upraises: [].} type + DatastoreError* = object of CatchableError + DatastoreKeyNotFound* = object of DatastoreError + + CodexResult*[T] = Result[T, ref DatastoreError] Datastore* = ref object of RootObj -method contains*( - self: Datastore, - key: Key): Future[?!bool] {.async, base, locks: "unknown".} = - +method contains*(self: Datastore, key: Key): Future[?!bool] {.base, locks: "unknown".} = raiseAssert("Not implemented!") -method delete*( - self: Datastore, - key: Key): Future[?!void] {.async, base, locks: "unknown".} = - +method delete*(self: Datastore, key: Key): Future[?!void] {.base, locks: "unknown".} = raiseAssert("Not implemented!") -method get*( - self: Datastore, - key: Key): Future[?!(?seq[byte])] {.async, base, locks: "unknown".} = - +method get*(self: Datastore, key: Key): Future[?!seq[byte]] {.base, locks: "unknown".} = raiseAssert("Not implemented!") -method put*( - self: Datastore, - key: Key, - data: seq[byte]): Future[?!void] {.async, base, locks: "unknown".} = - +method put*(self: Datastore, key: Key, data: seq[byte]): Future[?!void] {.base, locks: "unknown".} = raiseAssert("Not implemented!") -iterator query*( +method close*(self: Datastore): Future[?!void] {.base, async, locks: "unknown".} = + return success() + +method query*( self: Datastore, - query: Query): Future[QueryResponse] = + query: Query): Future[QueryIter] {.gcsafe.} = raiseAssert("Not implemented!") diff --git a/datastore/filesystem_datastore.nim b/datastore/filesystem_datastore.nim deleted file mode 100644 index 51c50a0..0000000 --- a/datastore/filesystem_datastore.nim +++ /dev/null @@ -1,162 +0,0 @@ -import std/os - -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -from pkg/stew/results as stewResults import get, isErr -import pkg/upraises - -import ./datastore - -export datastore - -push: {.upraises: [].} - -type - FileSystemDatastore* = ref object of Datastore - root: string - -const - objExt* = ".dsobject" - -proc new*( - T: type FileSystemDatastore, - root: string): ?!T = - - try: - let - root = if root.isAbsolute: root - else: getCurrentDir() / root - - if not dirExists(root): - failure "directory does not exist: " & root - else: - success T(root: root) - - except OSError as e: - failure e - -proc root*(self: FileSystemDatastore): string = - self.root - -proc path*( - self: FileSystemDatastore, - key: Key): string = - - var - segments: seq[string] - - for ns in key: - without field =? ns.field: - segments.add ns.value - continue - - segments.add(field / ns.value) - - # is it problematic that per this logic Key(/a:b) evaluates to the same path - # as Key(/a/b)? may need to check if/how other Datastore implementations - # distinguish them - - self.root / joinPath(segments) & objExt - -method contains*( - self: FileSystemDatastore, - key: Key): Future[?!bool] {.async, locks: "unknown".} = - - return success fileExists(self.path(key)) - -method delete*( - self: FileSystemDatastore, - key: Key): Future[?!void] {.async, locks: "unknown".} = - - let - path = self.path(key) - - try: - removeFile(path) - return success() - - # removing an empty directory might lead to surprising behavior depending - # on what the user specified as the `root` of the FileSystemDatastore, so - # until further consideration, empty directories will be left in place - - except OSError as e: - return failure e - -method get*( - self: FileSystemDatastore, - key: Key): Future[?!(?seq[byte])] {.async, locks: "unknown".} = - - # to support finer control of memory allocation, maybe could/should change - # the signature of `get` so that it has a 3rd parameter - # `bytes: var openArray[byte]` and return type `?!bool`; this variant with - # return type `?!(?seq[byte])` would be a special case (convenience method) - # calling the former after allocating a seq with size automatically - # determined via `getFileSize` - - let - path = self.path(key) - containsRes = await self.contains(key) - - if containsRes.isErr: return failure containsRes.error.msg - - if containsRes.get: - var - file: File - - if not file.open(path): - return failure "unable to open file: " & path - else: - try: - let - size = file.getFileSize - - var - bytes: seq[byte] - - if size > 0: - newSeq(bytes, size) - - let - bytesRead = file.readBytes(bytes, 0, size) - - if bytesRead < size: - return failure $bytesRead & " bytes were read from " & path & - " but " & $size & " bytes were expected" - - return success bytes.some - - except IOError as e: - return failure e - - finally: - file.close - - else: - return success seq[byte].none - -method put*( - self: FileSystemDatastore, - key: Key, - data: seq[byte]): Future[?!void] {.async, locks: "unknown".} = - - let - path = self.path(key) - - try: - createDir(parentDir(path)) - if data.len > 0: writeFile(path, data) - else: writeFile(path, "") - return success() - - except IOError as e: - return failure e - - except OSError as e: - return failure e - -# method query*( -# self: FileSystemDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....some diff --git a/datastore/fsds.nim b/datastore/fsds.nim new file mode 100644 index 0000000..9a5b794 --- /dev/null +++ b/datastore/fsds.nim @@ -0,0 +1,158 @@ +import std/os +import std/options + +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +from pkg/stew/results as stewResults import get, isErr +import pkg/upraises + +import ./datastore + +export datastore + +push: {.upraises: [].} + +type + FSDatastore* = ref object of Datastore + root*: string + ignoreProtected: bool + depth: int + +template path*(self: FSDatastore, key: Key): string = + var + segments: seq[string] + + for ns in key: + if ns.field == "": + segments.add ns.value + continue + + # `:` are replaced with `/` + segments.add(ns.field / ns.value) + + self.root / segments.joinPath() + +template validDepth*(self: FSDatastore, key: Key): bool = + key.len <= self.depth + +method contains*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = + + if not self.validDepth(key): + return failure "Path has invalid depth!" + + let + path = self.path(key) + + return success fileExists(path) + +method delete*(self: FSDatastore, key: Key): Future[?!void] {.async.} = + + if not self.validDepth(key): + return failure "Path has invalid depth!" + + let + path = self.path(key) + + try: + removeFile(path) + return success() + + # removing an empty directory might lead to surprising behavior depending + # on what the user specified as the `root` of the FSDatastore, so + # until further consideration, empty directories will be left in place + + except OSError as e: + return failure e + +method get*(self: FSDatastore, key: Key): Future[?!seq[byte]] {.async.} = + + # to support finer control of memory allocation, maybe could/should change + # the signature of `get` so that it has a 3rd parameter + # `bytes: var openArray[byte]` and return type `?!bool`; this variant with + # return type `?!(?seq[byte])` would be a special case (convenience method) + # calling the former after allocating a seq with size automatically + # determined via `getFileSize` + + if not self.validDepth(key): + return failure "Path has invalid depth!" + + let + path = self.path(key) + + if not fileExists(path): + return failure(newException(DatastoreKeyNotFound, "Key doesn't exist")) + + var + file: File + + defer: + file.close + + if not file.open(path): + return failure "unable to open file: " & path + + try: + let + size = file.getFileSize + + var + bytes = newSeq[byte](size) + read = 0 + + while read < size: + read += file.readBytes(bytes, read, size) + + if read < size: + return failure $read & " bytes were read from " & path & + " but " & $size & " bytes were expected" + + return success bytes + + except CatchableError as e: + return failure e + +method put*( + self: FSDatastore, + key: Key, + data: seq[byte]): Future[?!void] {.async, locks: "unknown".} = + + if not self.validDepth(key): + return failure "Path has invalid depth!" + + let + path = self.path(key) + + try: + createDir(parentDir(path)) + writeFile(path, data) + except CatchableError as e: + return failure e + + return success() + +# method query*( +# self: FSDatastore, +# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = +# +# return success ....some + +proc new*( + T: type FSDatastore, + root: string, + depth = 2, + caseSensitive = true, + ignoreProtected = false): ?!T = + + let root = ? ( + block: + if root.isAbsolute: root + else: getCurrentDir() / root).catch + + if not dirExists(root): + return failure "directory does not exist: " & root + + success T( + root: root, + ignoreProtected: ignoreProtected, + depth: depth) diff --git a/datastore/key.nim b/datastore/key.nim index 44d6480..ee0ab1d 100644 --- a/datastore/key.nim +++ b/datastore/key.nim @@ -22,59 +22,59 @@ type namespaces*: seq[Namespace] const - delimiter = ":" - separator = "/" + Delimiter* = ":" + Separator* = "/" # TODO: operator/s for combining string|Namespace,string|Namespace # TODO: lifting from ?![Namespace|Key] for various ops -proc init*( +func init*( T: type Namespace, field, value: string): ?!T = if value.strip == "": return failure "value string must not be all whitespace or empty" - if value.contains(delimiter): - return failure "value string must not contain delimiter \"" & - delimiter & "\"" + if value.contains(Delimiter): + return failure "value string must not contain Delimiter \"" & + Delimiter & "\"" - if value.contains(separator): - return failure "value string must not contain separator \"" & - separator & "\"" + if value.contains(Separator): + return failure "value string must not contain Separator \"" & + Separator & "\"" if field != "": if field.strip == "": return failure "field string must not be all whitespace" - if field.contains(delimiter): - return failure "field string must not contain delimiter \"" & - delimiter & "\"" + if field.contains(Delimiter): + return failure "field string must not contain Delimiter \"" & + Delimiter & "\"" - if field.contains(separator): - return failure "field string must not contain separator \"" & - separator & "\"" + if field.contains(Separator): + return failure "field string must not contain Separator \"" & + Separator & "\"" success T(field: field, value: value) -proc init*(T: type Namespace, id: string): ?!T = +func init*(T: type Namespace, id: string): ?!T = if id.strip == "": return failure "id string must not be all whitespace or empty" - if id.contains(separator): - return failure "id string must not contain separator \"" & separator & "\"" + if id.contains(Separator): + return failure "id string must not contain Separator \"" & Separator & "\"" - if id == delimiter: - return failure "value in id string \"[field]" & delimiter & + if id == Delimiter: + return failure "value in id string \"[field]" & Delimiter & "[value]\" must not be empty" - if id.count(delimiter) > 1: - return failure "id string must not contain more than one delimiter \"" & - delimiter & "\"" + if id.count(Delimiter) > 1: + return failure "id string must not contain more than one Delimiter \"" & + Delimiter & "\"" let (field, value) = block: - let parts = id.split(delimiter) + let parts = id.split(Delimiter) if parts.len > 1: (parts[0], parts[^1]) else: @@ -82,22 +82,25 @@ proc init*(T: type Namespace, id: string): ?!T = T.init(field, value) -proc id*(self: Namespace): string = +func id*(self: Namespace): string = if self.field.len > 0: - self.field & delimiter & self.value + self.field & Delimiter & self.value else: self.value -proc `$`*(namespace: Namespace): string = +func hash*(namespace: Namespace): Hash = + hash(namespace.id) + +func `$`*(namespace: Namespace): string = "Namespace(" & namespace.id & ")" -proc init*(T: type Key, namespaces: varargs[Namespace]): ?!T = +func init*(T: type Key, namespaces: varargs[Namespace]): ?!T = if namespaces.len == 0: failure "namespaces must contain at least one Namespace" else: success T(namespaces: @namespaces) -proc init*(T: type Key, namespaces: varargs[string]): ?!T = +func init*(T: type Key, namespaces: varargs[string]): ?!T = if namespaces.len == 0: failure "namespaces must contain at least one Namespace id string" else: @@ -106,7 +109,7 @@ proc init*(T: type Key, namespaces: varargs[string]): ?!T = ?Namespace.init(it) )) -proc init*(T: type Key, id: string): ?!T = +func init*(T: type Key, id: string): ?!T = if id == "": return failure "id string must contain at least one Namespace" @@ -114,15 +117,15 @@ proc init*(T: type Key, id: string): ?!T = return failure "id string must not be all whitespace" let - nsStrs = id.split(separator).filterIt(it != "") + nsStrs = id.split(Separator).filterIt(it != "") if nsStrs.len == 0: - return failure "id string must not contain only one or more separator " & - "\"" & separator & "\"" + return failure "id string must not contain more than one Separator " & + "\"" & Separator & "\"" Key.init(nsStrs) -proc list*(self: Key): seq[Namespace] = +func list*(self: Key): seq[Namespace] = self.namespaces proc random*(T: type Key): string = @@ -131,78 +134,91 @@ proc random*(T: type Key): string = template `[]`*(key: Key, x: auto): auto = key.namespaces[x] -proc len*(self: Key): int = +func len*(self: Key): int = self.namespaces.len iterator items*(key: Key): Namespace = for k in key.namespaces: yield k -proc reversed*(self: Key): Key = +func reversed*(self: Key): Key = Key(namespaces: self.namespaces.reversed) -proc reverse*(self: Key): Key = +func reverse*(self: Key): Key = self.reversed -proc name*(self: Key): string = +func name*(self: Key): string = if self.len > 0: return self[^1].value -proc `type`*(self: Key): string = +func `type`*(self: Key): string = if self.len > 0: return self[^1].field -proc id*(self: Key): string = - separator & self.namespaces.mapIt(it.id).join(separator) +func id*(self: Key): string = + Separator & self.namespaces.mapIt(it.id).join(Separator) -proc isTopLevel*(self: Key): bool = +func root*(self: Key): bool = self.len == 1 -proc parent*(self: Key): ?!Key = - if self.isTopLevel: +func parent*(self: Key): ?!Key = + if self.root: failure "key has no parent" else: success Key(namespaces: self.namespaces[0..^2]) -proc path*(self: Key): ?!Key = +func path*(self: Key): ?!Key = let - parent = ? self.parent + parent = ?self.parent if self[^1].field == "": return success parent - success Key(namespaces: parent.namespaces & @[Namespace(value: self[^1].field)]) + let ns = parent.namespaces & @[Namespace(value: self[^1].field)] + success Key(namespaces: ns) -proc child*(self: Key, ns: Namespace): Key = +func child*(self: Key, ns: Namespace): Key = Key(namespaces: self.namespaces & @[ns]) -proc `/`*(self: Key, ns: Namespace): Key = +func `/`*(self: Key, ns: Namespace): Key = self.child(ns) -proc child*(self: Key, namespaces: varargs[Namespace]): Key = +func child*(self: Key, namespaces: varargs[Namespace]): Key = Key(namespaces: self.namespaces & @namespaces) -proc child*(self, key: Key): Key = +func child*(self, key: Key): Key = Key(namespaces: self.namespaces & key.namespaces) -proc `/`*(self, key: Key): Key = +func `/`*(self, key: Key): Key = self.child(key) -proc child*(self: Key, keys: varargs[Key]): Key = +func child*(self: Key, keys: varargs[Key]): Key = Key(namespaces: self.namespaces & concat(keys.mapIt(it.namespaces))) -proc child*(self: Key, ids: varargs[string]): ?!Key = +func child*(self: Key, ids: varargs[string]): ?!Key = success self.child(ids.filterIt(it != "").mapIt( ?Key.init(it) )) -proc `/`*(self: Key, id: string): ?!Key = +func relative*(self: Key, parent: Key): ?!Key = + ## Get a key relative to parent from current key + ## + + if self.len < parent.len: + return failure "Not a parent of this key!" + + Key.init(self.namespaces[parent.namespaces.high..self.namespaces.high]) + +func `/`*(self: Key, id: string): ?!Key = self.child(id) -proc isAncestorOf*(self, other: Key): bool = +func ancestor*(self, other: Key): bool = if other.len <= self.len: false else: other.namespaces[0..