feat(store): run sqlite database vacuum at node start

waku/v2/node/config.nim

@@ -68,6 +68,11 @@ type
       desc: "The database path for peristent storage",
       defaultValue: ""
       name: "db-path" }: string
+
+    dbVacuum* {.
+      desc: "Enable database vacuuming at start: true|false",
+      defaultValue: false
+      name: "db-vacuum" }: bool
     
     persistPeers* {.
       desc: "Enable peer persistence: true|false",

waku/v2/node/storage/sqlite.nim

@@ -15,8 +15,6 @@ logScope:
   topics = "sqlite"
 
 type
-  DatabaseResult*[T] = Result[T, string]
-
   Sqlite = ptr sqlite3
 
   NoParams* = tuple
@@ -26,8 +24,6 @@ type
   AutoDisposed[T: ptr|ref] = object
     val: T
 
-  SqliteDatabase* = ref object of RootObj
-    env*: Sqlite
 
 template dispose(db: Sqlite) =
   discard sqlite3_close(db)
@@ -55,6 +51,19 @@ template checkErr*(op, cleanup: untyped) =
 template checkErr*(op) =
   checkErr(op): discard
 
+
+type
+  DatabaseResult*[T] = Result[T, string]
+  
+  SqliteDatabase* = ref object of RootObj
+    env*: Sqlite
+
+
+type DataProc* = proc(s: RawStmtPtr) {.closure.} # the nim-eth definition is different; one more indirection
+
+const NoopRowHandler* = proc(s: RawStmtPtr) {.closure.} = discard
+
+
 proc init*(
     T: type SqliteDatabase,
     basePath: string,
@@ -192,9 +201,6 @@ template readResult(s: RawStmtPtr, T: type): auto =
   else:
     readResult(s, 0.cint, T)
 
-type 
-  DataProc* = proc(s: ptr sqlite3_stmt) {.closure.} # the nim-eth definition is different; one more indirection
-
 proc exec*[Params, Res](s: SqliteStmt[Params, Res],
                         params: Params,
                         onData: DataProc): DatabaseResult[bool] =
@@ -257,6 +263,62 @@ proc close*(db: SqliteDatabase) =
 
   db[] = SqliteDatabase()[]
 
+
+##  Maintenance procedures
+
+# TODO: Cache this value in the SqliteDatabase object.
+#       Page size should not change during the node execution time
+proc getPageSize*(db: SqliteDatabase): DatabaseResult[int64] =
+  ## Query or set the page size of the database. The page size must be a power of 
+  ## two between 512 and 65536 inclusive.
+  var count: int64
+  proc handler(s: RawStmtPtr) = 
+    count = sqlite3_column_int64(s, 0)
+
+  let res = db.query("PRAGMA page_size;", handler)
+  if res.isErr():
+      return err("failed to get page_size")
+
+  ok(count)
+
+
+proc getFreelistCount*(db: SqliteDatabase): DatabaseResult[int64] = 
+  ## Return the number of unused pages in the database file.
+  var count: int64
+  proc handler(s: RawStmtPtr) = 
+    count = sqlite3_column_int64(s, 0)
+
+  let res = db.query("PRAGMA freelist_count;", handler)
+  if res.isErr():
+      return err("failed to get freelist_count")
+
+  ok(count)
+
+
+proc getPageCount*(db: SqliteDatabase): DatabaseResult[int64] =
+  ## Return the total number of pages in the database file.
+  var count: int64
+  proc handler(s: RawStmtPtr) = 
+    count = sqlite3_column_int64(s, 0)
+
+  let res = db.query("PRAGMA page_count;", handler)
+  if res.isErr():
+      return err("failed to get page_count")
+
+  ok(count)
+
+
+proc vacuum*(db: SqliteDatabase): DatabaseResult[void] =
+  ## The VACUUM command rebuilds the database file, repacking it into a minimal amount of disk space.
+  let res = db.query("VACUUM;", NoopRowHandler)
+  if res.isErr():
+      return err("vacuum failed")
+
+  ok()
+
+
+## Migration procedures
+
 proc getUserVersion*(database: SqliteDatabase): DatabaseResult[int64] = 
   var version: int64
   proc handler(s: ptr sqlite3_stmt) = 
@@ -341,4 +403,4 @@ proc migrate*(db: SqliteDatabase, path: string, targetVersion: int64 = migration
       return err("failed to set the new user_version")
 
     debug "user_version is set to", targetVersion=targetVersion
-    ok(true)
+    ok(true)

waku/v2/node/wakunode2.nim

@@ -818,6 +818,7 @@ when isMainModule:
     ./wakunode2_setup_metrics,
     ./wakunode2_setup_rpc,
     ./wakunode2_setup_sql_migrations,
+    ./storage/sqlite,
     ./storage/message/sqlite_store,
     ./storage/peer/waku_peer_storage
   
@@ -853,6 +854,26 @@ when isMainModule:
         sqliteDatabase = dbRes.value
 
     if not sqliteDatabase.isNil and (conf.persistPeers or conf.persistMessages):
+      
+      ## Database vacuuming
+      # TODO: Wrap and move this logic to the appropriate module
+      let 
+        pageSize = ?sqliteDatabase.getPageSize()
+        pageCount = ?sqliteDatabase.getPageCount()
+        freelistCount = ?sqliteDatabase.getFreelistCount()
+
+      debug "sqlite database page stats", pageSize=pageSize, pages=pageCount, freePages=freelistCount
+
+      # TODO: Run vacuuming conditionally based on database page stats
+      if conf.dbVacuum and (pageCount > 0 and freelistCount > 0):
+        debug "starting sqlite database vacuuming"
+
+        let resVacuum = sqliteDatabase.vacuum()
+        if resVacuum.isErr():
+          return err("failed to execute vacuum: " & resVacuum.error())
+
+        debug "finished sqlite database vacuuming"
+
       # Database initialized. Let's set it up
       sqliteDatabase.runMigrations(conf) # First migrate what we have
 
@@ -876,7 +897,7 @@ when isMainModule:
           waku_node_errors.inc(labelValues = ["init_store_failure"])
         else:
           storeTuple.mStorage = res.value
-    
+ 
     ok(storeTuple)
 
   # 2/7 Retrieve dynamic bootstrap nodes