From b0241a7c573cfe77ea0e9eb5061bf7af0a51e7e1 Mon Sep 17 00:00:00 2001
From: "Michael Bradley, Jr" <michaelsbradleyjr@gmail.com>
Date: Fri, 2 Sep 2022 23:33:04 -0500
Subject: [PATCH] make page size configurable in SQLiteDatastore.new

also make cache size and journal mode configurable

closes #6
related to status-im/nim-codex#234
---
 datastore/sqlite.nim                      | 73 +++++++++++++++++++++--
 datastore/sqlite_datastore.nim            | 31 +++++++---
 tests/datastore/test_sqlite_datastore.nim |  3 +-
 3 files changed, 95 insertions(+), 12 deletions(-)

diff --git a/datastore/sqlite.nim b/datastore/sqlite.nim
index a8b136e..6e548a4 100644
--- a/datastore/sqlite.nim
+++ b/datastore/sqlite.nim
@@ -1,3 +1,6 @@
+import std/math
+import std/unicode
+
 import pkg/questionable
 import pkg/questionable/results
 import pkg/sqlite3_abi
@@ -20,6 +23,10 @@ type
 
   DataProc* = proc(s: RawStmtPtr) {.closure, gcsafe.}
 
+  # https://www.sqlite.org/pragma.html#pragma_journal_mode
+  JournalMode* = enum
+    DELETE, TRUNCATE, PERSIST, MEMORY, WAL, OFF
+
   NoParams* = tuple # empty tuple
 
   NoParamsStmt* = SQLiteStmt[NoParams, void]
@@ -172,9 +179,67 @@ proc sqlite3_column_text_not_null*(
 
   text
 
-template journalModePragmaStmt*(env: SQLite): RawStmtPtr =
+template pageSizePragmaStmt*(env: SQLite, pageSize: Positive): RawStmtPtr =
+  # https://www.sqlite.org/pragma.html#pragma_page_size
+  if not (isPowerOfTwo(pageSize) and pageSize >= 512 and pageSize <= 65536):
+    return failure "pageSize must be a power of two between 512 and 65536 inclusive"
+
   let
-    s = prepare(env, "PRAGMA journal_mode = WAL;")
+    s1 = prepare(env, "PRAGMA page_size = " & $pageSize & ";")
+
+  s1.dispose
+
+  let
+    s2 = prepare(env, "PRAGMA page_size;")
+
+  if (let x = sqlite3_step(s2); x != SQLITE_ROW):
+    s2.dispose
+    return failure $sqlite3_errstr(x)
+
+  if (let x = sqlite3_column_type(s2, 0); x != SQLITE_INTEGER):
+    s2.dispose
+    return failure $sqlite3_errstr(x)
+
+  let
+    x = sqlite3_column_int(s2, 0)
+
+  if x != pageSize:
+    s2.dispose
+    return failure "Unexpected page_size pragma result: " & $x
+
+  s2
+
+template cacheSizePragmaStmt*(env: SQLite, cacheSize: int): RawStmtPtr =
+  # https://www.sqlite.org/pragma.html#pragma_cache_size
+  let
+    s1 = prepare(env, "PRAGMA cache_size = " & $cacheSize & ";")
+
+  s1.dispose
+
+  let
+    s2 = prepare(env, "PRAGMA cache_size;")
+
+  if (let x = sqlite3_step(s2); x != SQLITE_ROW):
+    s2.dispose
+    return failure $sqlite3_errstr(x)
+
+  if (let x = sqlite3_column_type(s2, 0); x != SQLITE_INTEGER):
+    s2.dispose
+    return failure $sqlite3_errstr(x)
+
+  let
+    x = sqlite3_column_int(s2, 0)
+
+  if x != cacheSize:
+    s2.dispose
+    return failure "Unexpected cache_size pragma result: " & $x
+
+  s2
+
+template journalModePragmaStmt*(env: SQLite, mode: JournalMode): RawStmtPtr =
+  # https://www.sqlite.org/pragma.html#pragma_journal_mode
+  let
+    s = prepare(env, "PRAGMA journal_mode = " & $mode & ";")
 
   if (let x = sqlite3_step(s); x != SQLITE_ROW):
     s.dispose
@@ -187,9 +252,9 @@ template journalModePragmaStmt*(env: SQLite): RawStmtPtr =
   let
     x = $sqlite3_column_text_not_null(s, 0)
 
-  if not (x in ["memory", "wal"]):
+  if not (x in ["memory", ($mode).toLower]):
     s.dispose
-    return failure "Invalid pragma result: \"" & x & "\""
+    return failure "Unexpected journal_mode pragma result: \"" & x & "\""
 
   s
 
diff --git a/datastore/sqlite_datastore.nim b/datastore/sqlite_datastore.nim
index 0a4d321..a22eced 100644
--- a/datastore/sqlite_datastore.nim
+++ b/datastore/sqlite_datastore.nim
@@ -176,13 +176,26 @@ proc new*(
   T: type SQLiteDatastore,
   basePath: string,
   filename = "store" & dbExt,
-  readOnly = false): ?!T =
+  readOnly = false,
 
-  # make it optional to enable WAL with it enabled being the default?
-
-  # make it possible to specify a custom page size?
+  # SQLite's default page_size is 4096 bytes since v3.12.0 (2016-03-29)
   # https://www.sqlite.org/pragma.html#pragma_page_size
-  # https://www.sqlite.org/intern-v-extern-blob.html
+  # see also: https://www.sqlite.org/intern-v-extern-blob.html
+  pageSize: Positive = 4096,
+
+  # SQLite's default cache_size is -2000 since v3.12.0 (2016-03-29)
+  # a negative value translates to approximately "abs(cache_size*1024) bytes of memory"
+  # a positive value translates to "cache_size*page_size bytes of memory"
+  # docs for `PRAGMA cache_size` may need some clarification
+  # https://www.sqlite.org/pragma.html#pragma_cache_size
+  # https://www.sqlite.org/pgszchng2016.html
+  # https://sqlite.org/forum/forumpost/096a95c0f9
+  # NOTE: a system build may have used nonstandard compile-time options,
+  # e.g. in recent versions of macOS the default cache_size is (positive) 2000
+  cacheSize = -2000,
+
+  # https://www.sqlite.org/pragma.html#pragma_journal_mode
+  journalMode = WAL): ?!T =
 
   var
     env: AutoDisposed[SQLite]
@@ -225,9 +238,13 @@ proc new*(
   open(dbPath, env.val, flags)
 
   let
-    pragmaStmt = journalModePragmaStmt(env.val)
+    pageSizePragmaStmt = pageSizePragmaStmt(env.val, pageSize)
+    cacheSizePragmaStmt = cacheSizePragmaStmt(env.val, cacheSize)
+    journalModePragmaStmt = journalModePragmaStmt(env.val, journalMode)
 
-  checkExec(pragmaStmt)
+  checkExec(pageSizePragmaStmt)
+  checkExec(cacheSizePragmaStmt)
+  checkExec(journalModePragmaStmt)
 
   var
     containsStmt: ContainsStmt
diff --git a/tests/datastore/test_sqlite_datastore.nim b/tests/datastore/test_sqlite_datastore.nim
index e09767a..143d2fe 100644
--- a/tests/datastore/test_sqlite_datastore.nim
+++ b/tests/datastore/test_sqlite_datastore.nim
@@ -42,7 +42,8 @@ suite "SQLiteDatastore":
 
     check: dsRes.isErr
 
-    dsRes = SQLiteDatastore.new(basePathAbs, filename)
+    dsRes = SQLiteDatastore.new(basePathAbs, filename, pageSize = 65536,
+      cacheSize = 1600, journalMode = TRUNCATE)
 
     check:
       dsRes.isOk