From 87479a603be4bb3d0eb63d8c5a31bf7c6613b58a Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Wed, 14 Sep 2022 18:09:08 +0200 Subject: [PATCH] feat(store): run sqlite database vacuum at node start --- waku/v2/node/config.nim | 5 +++ waku/v2/node/storage/sqlite.nim | 78 +++++++++++++++++++++++++++++---- waku/v2/node/wakunode2.nim | 23 +++++++++- 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/waku/v2/node/config.nim b/waku/v2/node/config.nim index 0c849ad99..cb47a80f0 100644 --- a/waku/v2/node/config.nim +++ b/waku/v2/node/config.nim @@ -68,6 +68,11 @@ type desc: "The database path for peristent storage", defaultValue: "" name: "db-path" }: string + + dbVacuum* {. + desc: "Enable database vacuuming at start: true|false", + defaultValue: false + name: "db-vacuum" }: bool persistPeers* {. desc: "Enable peer persistence: true|false", diff --git a/waku/v2/node/storage/sqlite.nim b/waku/v2/node/storage/sqlite.nim index c19bdd447..6a7127d93 100644 --- a/waku/v2/node/storage/sqlite.nim +++ b/waku/v2/node/storage/sqlite.nim @@ -15,8 +15,6 @@ logScope: topics = "sqlite" type - DatabaseResult*[T] = Result[T, string] - Sqlite = ptr sqlite3 NoParams* = tuple @@ -26,8 +24,6 @@ type AutoDisposed[T: ptr|ref] = object val: T - SqliteDatabase* = ref object of RootObj - env*: Sqlite template dispose(db: Sqlite) = discard sqlite3_close(db) @@ -55,6 +51,19 @@ template checkErr*(op, cleanup: untyped) = template checkErr*(op) = checkErr(op): discard + +type + DatabaseResult*[T] = Result[T, string] + + SqliteDatabase* = ref object of RootObj + env*: Sqlite + + +type DataProc* = proc(s: RawStmtPtr) {.closure.} # the nim-eth definition is different; one more indirection + +const NoopRowHandler* = proc(s: RawStmtPtr) {.closure.} = discard + + proc init*( T: type SqliteDatabase, basePath: string, @@ -192,9 +201,6 @@ template readResult(s: RawStmtPtr, T: type): auto = else: readResult(s, 0.cint, T) -type - DataProc* = proc(s: ptr sqlite3_stmt) {.closure.} # the nim-eth definition is different; one more indirection - proc exec*[Params, Res](s: SqliteStmt[Params, Res], params: Params, onData: DataProc): DatabaseResult[bool] = @@ -257,6 +263,62 @@ proc close*(db: SqliteDatabase) = db[] = SqliteDatabase()[] + +## Maintenance procedures + +# TODO: Cache this value in the SqliteDatabase object. +# Page size should not change during the node execution time +proc getPageSize*(db: SqliteDatabase): DatabaseResult[int64] = + ## Query or set the page size of the database. The page size must be a power of + ## two between 512 and 65536 inclusive. + var count: int64 + proc handler(s: RawStmtPtr) = + count = sqlite3_column_int64(s, 0) + + let res = db.query("PRAGMA page_size;", handler) + if res.isErr(): + return err("failed to get page_size") + + ok(count) + + +proc getFreelistCount*(db: SqliteDatabase): DatabaseResult[int64] = + ## Return the number of unused pages in the database file. + var count: int64 + proc handler(s: RawStmtPtr) = + count = sqlite3_column_int64(s, 0) + + let res = db.query("PRAGMA freelist_count;", handler) + if res.isErr(): + return err("failed to get freelist_count") + + ok(count) + + +proc getPageCount*(db: SqliteDatabase): DatabaseResult[int64] = + ## Return the total number of pages in the database file. + var count: int64 + proc handler(s: RawStmtPtr) = + count = sqlite3_column_int64(s, 0) + + let res = db.query("PRAGMA page_count;", handler) + if res.isErr(): + return err("failed to get page_count") + + ok(count) + + +proc vacuum*(db: SqliteDatabase): DatabaseResult[void] = + ## The VACUUM command rebuilds the database file, repacking it into a minimal amount of disk space. + let res = db.query("VACUUM;", NoopRowHandler) + if res.isErr(): + return err("vacuum failed") + + ok() + + +## Migration procedures + proc getUserVersion*(database: SqliteDatabase): DatabaseResult[int64] = var version: int64 proc handler(s: ptr sqlite3_stmt) = @@ -341,4 +403,4 @@ proc migrate*(db: SqliteDatabase, path: string, targetVersion: int64 = migration return err("failed to set the new user_version") debug "user_version is set to", targetVersion=targetVersion - ok(true) + ok(true) \ No newline at end of file diff --git a/waku/v2/node/wakunode2.nim b/waku/v2/node/wakunode2.nim index ca0037413..fcabdfdd0 100644 --- a/waku/v2/node/wakunode2.nim +++ b/waku/v2/node/wakunode2.nim @@ -818,6 +818,7 @@ when isMainModule: ./wakunode2_setup_metrics, ./wakunode2_setup_rpc, ./wakunode2_setup_sql_migrations, + ./storage/sqlite, ./storage/message/sqlite_store, ./storage/peer/waku_peer_storage @@ -853,6 +854,26 @@ when isMainModule: sqliteDatabase = dbRes.value if not sqliteDatabase.isNil and (conf.persistPeers or conf.persistMessages): + + ## Database vacuuming + # TODO: Wrap and move this logic to the appropriate module + let + pageSize = ?sqliteDatabase.getPageSize() + pageCount = ?sqliteDatabase.getPageCount() + freelistCount = ?sqliteDatabase.getFreelistCount() + + debug "sqlite database page stats", pageSize=pageSize, pages=pageCount, freePages=freelistCount + + # TODO: Run vacuuming conditionally based on database page stats + if conf.dbVacuum and (pageCount > 0 and freelistCount > 0): + debug "starting sqlite database vacuuming" + + let resVacuum = sqliteDatabase.vacuum() + if resVacuum.isErr(): + return err("failed to execute vacuum: " & resVacuum.error()) + + debug "finished sqlite database vacuuming" + # Database initialized. Let's set it up sqliteDatabase.runMigrations(conf) # First migrate what we have @@ -876,7 +897,7 @@ when isMainModule: waku_node_errors.inc(labelValues = ["init_store_failure"]) else: storeTuple.mStorage = res.value - + ok(storeTuple) # 2/7 Retrieve dynamic bootstrap nodes