nim-datastore/datastore/sqlite_datastore.nim

import std/os
import std/times

import pkg/chronos
import pkg/questionable
import pkg/questionable/results
import pkg/sqlite3_abi
import pkg/stew/byteutils
from pkg/stew/results as stewResults import get, isErr
import pkg/upraises

import ./datastore
import ./sqlite

export datastore, sqlite

push: {.upraises: [].}

type
  BoundIdCol = proc (): string {.closure, gcsafe, upraises: [].}

  BoundDataCol = proc (): seq[byte] {.closure, gcsafe, upraises: [].}

  BoundTimestampCol = proc (): int64 {.closure, gcsafe, upraises: [].}

  # feels odd to use `void` for prepared statements corresponding to SELECT
  # queries but it fits with the rest of the SQLite wrapper adapted from
  # status-im/nwaku, at least in its current form in ./sqlite
  ContainsStmt = SQLiteStmt[(string), void]

  DeleteStmt = SQLiteStmt[(string), void]

  GetStmt = SQLiteStmt[(string), void]

  PutStmt = SQLiteStmt[(string, seq[byte], int64), void]

  SQLiteDatastore* = ref object of Datastore
    dbPath: string
    containsStmt: ContainsStmt
    deleteStmt: DeleteStmt
    env: SQLite
    getDataCol: BoundDataCol
    getStmt: GetStmt
    putStmt: PutStmt
    readOnly: bool

const
  dbExt* = ".sqlite3"
  tableName* = "Store"

  idColName* = "id"
  dataColName* = "data"
  timestampColName* = "timestamp"

  idColType = "TEXT"
  dataColType = "BLOB"
  timestampColType = "INTEGER"

  # https://stackoverflow.com/a/9756276
  # EXISTS returns a boolean value represented by an integer:
  # https://sqlite.org/datatype3.html#boolean_datatype
  # https://sqlite.org/lang_expr.html#the_exists_operator
  containsStmtStr = """
    SELECT EXISTS(
      SELECT 1 FROM """ & tableName & """
      WHERE """ & idColName & """ = ?
    );
  """

  containsStmtExistsCol = 0

  createStmtStr = """
    CREATE TABLE IF NOT EXISTS """ & tableName & """ (
      """ & idColName & """ """ & idColType & """ NOT NULL PRIMARY KEY,
      """ & dataColName & """ """ & dataColType & """,
      """ & timestampColName & """ """ & timestampColType & """ NOT NULL
    ) WITHOUT ROWID;
  """

  deleteStmtStr = """
    DELETE FROM """ & tableName & """
    WHERE """ & idColName & """ = ?;
  """

  getStmtStr = """
    SELECT """ & dataColName & """ FROM """ & tableName & """
    WHERE """ & idColName & """ = ?;
  """

  getStmtDataCol = 0

  putStmtStr = """
    REPLACE INTO """ & tableName & """ (
      """ & idColName & """,
      """ & dataColName & """,
      """ & timestampColName & """
    ) VALUES (?, ?, ?);
  """

proc checkColMetadata(s: RawStmtPtr, i: int, expectedName: string) =
  let
    colName = sqlite3_column_origin_name(s, i.cint)

  if colName.isNil:
    raise (ref Defect)(msg: "no column exists for index " & $i & " in `" &
      $sqlite3_sql(s) & "`")

  if $colName != expectedName:
    raise (ref Defect)(msg: "original column name for index " & $i & " was \"" &
      $colName & "\" in `" & $sqlite3_sql(s) & "` but callee expected \"" &
      expectedName & "\"")

proc idCol*(
  s: RawStmtPtr,
  index: int): BoundIdCol =

  checkColMetadata(s, index, idColName)

  return proc (): string =
    $sqlite3_column_text_not_null(s, index.cint)

proc dataCol*(
  s: RawStmtPtr,
  index: int): BoundDataCol =

  checkColMetadata(s, index, dataColName)

  return proc (): seq[byte] =
    let
      i = index.cint
      blob = sqlite3_column_blob(s, i)

    # detect out-of-memory error
    # see the conversion table and final paragraph of:
    # https://www.sqlite.org/c3ref/column_blob.html
    # see also https://www.sqlite.org/rescode.html

    # the "data" column can be NULL so in order to detect an out-of-memory error
    # it is necessary to check that the result is a null pointer and that the
    # result code is an error code
    if blob.isNil:
      let
        code = sqlite3_errcode(sqlite3_db_handle(s))

      if not (code in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]):
        raise (ref Defect)(msg: $sqlite3_errstr(code))

    let
      dataLen = sqlite3_column_bytes(s, i)
      dataBytes = cast[ptr UncheckedArray[byte]](blob)

    @(toOpenArray(dataBytes, 0, dataLen - 1))

proc timestampCol*(
  s: RawStmtPtr,
  index: int): BoundTimestampCol =

  checkColMetadata(s, index, timestampColName)

  return proc (): int64 =
    sqlite3_column_int64(s, index.cint)

proc new*(
  T: type SQLiteDatastore,
  basePath = "data",
  filename = "store" & dbExt,
  readOnly = false,
  inMemory = false): ?!T =

  # make it optional to enable WAL with it enabled being the default?

  # make it possible to specify a custom page size?
  # https://www.sqlite.org/pragma.html#pragma_page_size
  # https://www.sqlite.org/intern-v-extern-blob.html

  var
    env: AutoDisposed[SQLite]

  defer: disposeIfUnreleased(env)

  var
    basep, fname, dbPath: string

  if inMemory:
    if readOnly:
      return failure "SQLiteDatastore cannot be read-only and in-memory"
    else:
      dbPath = ":memory:"
  else:
    try:
      basep = normalizePathEnd(
        if basePath.isAbsolute: basePath
        else: getCurrentDir() / basePath)

      fname = filename.normalizePathEnd
      dbPath = basep / fname

      if readOnly and not fileExists(dbPath):
        return failure "read-only database does not exist: " & dbPath
      else:
        createDir(basep)

    except IOError as e:
      return failure e

    except OSError as e:
      return failure e

  let
    flags =
      if readOnly: SQLITE_OPEN_READONLY
      else: SQLITE_OPEN_READWRITE or SQLITE_OPEN_CREATE

  open(dbPath, env.val, flags)

  let
    pragmaStmt = journalModePragmaStmt(env.val)

  checkExec(pragmaStmt)

  var
    containsStmt: ContainsStmt
    deleteStmt: DeleteStmt
    getStmt: GetStmt
    putStmt: PutStmt

  if not readOnly:
    checkExec(env.val, createStmtStr)

    deleteStmt = ? DeleteStmt.prepare(
      env.val, deleteStmtStr, SQLITE_PREPARE_PERSISTENT)

    putStmt = ? PutStmt.prepare(
      env.val, putStmtStr, SQLITE_PREPARE_PERSISTENT)

  containsStmt = ? ContainsStmt.prepare(
    env.val, containsStmtStr, SQLITE_PREPARE_PERSISTENT)

  getStmt = ? GetStmt.prepare(
    env.val, getStmtStr, SQLITE_PREPARE_PERSISTENT)

  # if a readOnly/existing database does not satisfy the expected schema
  # `pepare()` will fail and `new` will return an error with message
  # "SQL logic error"

  let
    getDataCol = dataCol(RawStmtPtr(getStmt), getStmtDataCol)

  success T(dbPath: dbPath, containsStmt: containsStmt, deleteStmt: deleteStmt,
            env: env.release, getStmt: getStmt, getDataCol: getDataCol,
            putStmt: putStmt, readOnly: readOnly)

proc dbPath*(self: SQLiteDatastore): string =
  self.dbPath

proc env*(self: SQLiteDatastore): SQLite =
  self.env

proc close*(self: SQLiteDatastore) =
  self.containsStmt.dispose
  self.getStmt.dispose

  if not self.readOnly:
    self.deleteStmt.dispose
    self.putStmt.dispose

  self.env.dispose
  self[] = SQLiteDatastore()[]

proc timestamp*(t = epochTime()): int64 =
  (t * 1_000_000).int64

method contains*(
  self: SQLiteDatastore,
  key: Key): Future[?!bool] {.async, locks: "unknown".} =

  var
    exists = false

  proc onData(s: RawStmtPtr) =
    exists = sqlite3_column_int64(s, containsStmtExistsCol.cint).bool

  let
    queryRes = self.containsStmt.query((key.id), onData)

  if queryRes.isErr: return queryRes

  return success exists

method delete*(
  self: SQLiteDatastore,
  key: Key): Future[?!void] {.async, locks: "unknown".} =

  if self.readOnly:
    return failure "database is read-only":
  else:
    return self.deleteStmt.exec((key.id))

method get*(
  self: SQLiteDatastore,
  key: Key): Future[?!(?seq[byte])] {.async, locks: "unknown".} =

  # see comment in ./filesystem_datastore re: finer control of memory
  # allocation in `method get`, could apply here as well if bytes were read
  # incrementally with `sqlite3_blob_read`

  var
    bytes: ?seq[byte]

  let
    dataCol = self.getDataCol

  proc onData(s: RawStmtPtr) =
    bytes = dataCol().some

  let
    queryRes = self.getStmt.query((key.id), onData)

  if queryRes.isErr:
    return failure queryRes.error.msg
  else:
    return success bytes

proc put*(
  self: SQLiteDatastore,
  key: Key,
  data: seq[byte],
  timestamp: int64): Future[?!void] {.async.} =

  if self.readOnly:
    return failure "database is read-only"
  else:
    return self.putStmt.exec((key.id, @data, timestamp))

method put*(
  self: SQLiteDatastore,
  key: Key,
  data: seq[byte]): Future[?!void] {.async, locks: "unknown".} =

  return await self.put(key, data, timestamp())

# method query*(
#   self: SQLiteDatastore,
#   query: ...): Future[?!(?...)] {.async, locks: "unknown".} =
#
#   return success ....some