mirror of https://github.com/waku-org/nwaku.git
feat(store): run sqlite database vacuum at node start
This commit is contained in:
parent
89069aa34f
commit
87479a603b
|
@ -68,6 +68,11 @@ type
|
||||||
desc: "The database path for peristent storage",
|
desc: "The database path for peristent storage",
|
||||||
defaultValue: ""
|
defaultValue: ""
|
||||||
name: "db-path" }: string
|
name: "db-path" }: string
|
||||||
|
|
||||||
|
dbVacuum* {.
|
||||||
|
desc: "Enable database vacuuming at start: true|false",
|
||||||
|
defaultValue: false
|
||||||
|
name: "db-vacuum" }: bool
|
||||||
|
|
||||||
persistPeers* {.
|
persistPeers* {.
|
||||||
desc: "Enable peer persistence: true|false",
|
desc: "Enable peer persistence: true|false",
|
||||||
|
|
|
@ -15,8 +15,6 @@ logScope:
|
||||||
topics = "sqlite"
|
topics = "sqlite"
|
||||||
|
|
||||||
type
|
type
|
||||||
DatabaseResult*[T] = Result[T, string]
|
|
||||||
|
|
||||||
Sqlite = ptr sqlite3
|
Sqlite = ptr sqlite3
|
||||||
|
|
||||||
NoParams* = tuple
|
NoParams* = tuple
|
||||||
|
@ -26,8 +24,6 @@ type
|
||||||
AutoDisposed[T: ptr|ref] = object
|
AutoDisposed[T: ptr|ref] = object
|
||||||
val: T
|
val: T
|
||||||
|
|
||||||
SqliteDatabase* = ref object of RootObj
|
|
||||||
env*: Sqlite
|
|
||||||
|
|
||||||
template dispose(db: Sqlite) =
|
template dispose(db: Sqlite) =
|
||||||
discard sqlite3_close(db)
|
discard sqlite3_close(db)
|
||||||
|
@ -55,6 +51,19 @@ template checkErr*(op, cleanup: untyped) =
|
||||||
template checkErr*(op) =
|
template checkErr*(op) =
|
||||||
checkErr(op): discard
|
checkErr(op): discard
|
||||||
|
|
||||||
|
|
||||||
|
type
|
||||||
|
DatabaseResult*[T] = Result[T, string]
|
||||||
|
|
||||||
|
SqliteDatabase* = ref object of RootObj
|
||||||
|
env*: Sqlite
|
||||||
|
|
||||||
|
|
||||||
|
type DataProc* = proc(s: RawStmtPtr) {.closure.} # the nim-eth definition is different; one more indirection
|
||||||
|
|
||||||
|
const NoopRowHandler* = proc(s: RawStmtPtr) {.closure.} = discard
|
||||||
|
|
||||||
|
|
||||||
proc init*(
|
proc init*(
|
||||||
T: type SqliteDatabase,
|
T: type SqliteDatabase,
|
||||||
basePath: string,
|
basePath: string,
|
||||||
|
@ -192,9 +201,6 @@ template readResult(s: RawStmtPtr, T: type): auto =
|
||||||
else:
|
else:
|
||||||
readResult(s, 0.cint, T)
|
readResult(s, 0.cint, T)
|
||||||
|
|
||||||
type
|
|
||||||
DataProc* = proc(s: ptr sqlite3_stmt) {.closure.} # the nim-eth definition is different; one more indirection
|
|
||||||
|
|
||||||
proc exec*[Params, Res](s: SqliteStmt[Params, Res],
|
proc exec*[Params, Res](s: SqliteStmt[Params, Res],
|
||||||
params: Params,
|
params: Params,
|
||||||
onData: DataProc): DatabaseResult[bool] =
|
onData: DataProc): DatabaseResult[bool] =
|
||||||
|
@ -257,6 +263,62 @@ proc close*(db: SqliteDatabase) =
|
||||||
|
|
||||||
db[] = SqliteDatabase()[]
|
db[] = SqliteDatabase()[]
|
||||||
|
|
||||||
|
|
||||||
|
## Maintenance procedures
|
||||||
|
|
||||||
|
# TODO: Cache this value in the SqliteDatabase object.
|
||||||
|
# Page size should not change during the node execution time
|
||||||
|
proc getPageSize*(db: SqliteDatabase): DatabaseResult[int64] =
|
||||||
|
## Query or set the page size of the database. The page size must be a power of
|
||||||
|
## two between 512 and 65536 inclusive.
|
||||||
|
var count: int64
|
||||||
|
proc handler(s: RawStmtPtr) =
|
||||||
|
count = sqlite3_column_int64(s, 0)
|
||||||
|
|
||||||
|
let res = db.query("PRAGMA page_size;", handler)
|
||||||
|
if res.isErr():
|
||||||
|
return err("failed to get page_size")
|
||||||
|
|
||||||
|
ok(count)
|
||||||
|
|
||||||
|
|
||||||
|
proc getFreelistCount*(db: SqliteDatabase): DatabaseResult[int64] =
|
||||||
|
## Return the number of unused pages in the database file.
|
||||||
|
var count: int64
|
||||||
|
proc handler(s: RawStmtPtr) =
|
||||||
|
count = sqlite3_column_int64(s, 0)
|
||||||
|
|
||||||
|
let res = db.query("PRAGMA freelist_count;", handler)
|
||||||
|
if res.isErr():
|
||||||
|
return err("failed to get freelist_count")
|
||||||
|
|
||||||
|
ok(count)
|
||||||
|
|
||||||
|
|
||||||
|
proc getPageCount*(db: SqliteDatabase): DatabaseResult[int64] =
|
||||||
|
## Return the total number of pages in the database file.
|
||||||
|
var count: int64
|
||||||
|
proc handler(s: RawStmtPtr) =
|
||||||
|
count = sqlite3_column_int64(s, 0)
|
||||||
|
|
||||||
|
let res = db.query("PRAGMA page_count;", handler)
|
||||||
|
if res.isErr():
|
||||||
|
return err("failed to get page_count")
|
||||||
|
|
||||||
|
ok(count)
|
||||||
|
|
||||||
|
|
||||||
|
proc vacuum*(db: SqliteDatabase): DatabaseResult[void] =
|
||||||
|
## The VACUUM command rebuilds the database file, repacking it into a minimal amount of disk space.
|
||||||
|
let res = db.query("VACUUM;", NoopRowHandler)
|
||||||
|
if res.isErr():
|
||||||
|
return err("vacuum failed")
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
## Migration procedures
|
||||||
|
|
||||||
proc getUserVersion*(database: SqliteDatabase): DatabaseResult[int64] =
|
proc getUserVersion*(database: SqliteDatabase): DatabaseResult[int64] =
|
||||||
var version: int64
|
var version: int64
|
||||||
proc handler(s: ptr sqlite3_stmt) =
|
proc handler(s: ptr sqlite3_stmt) =
|
||||||
|
@ -341,4 +403,4 @@ proc migrate*(db: SqliteDatabase, path: string, targetVersion: int64 = migration
|
||||||
return err("failed to set the new user_version")
|
return err("failed to set the new user_version")
|
||||||
|
|
||||||
debug "user_version is set to", targetVersion=targetVersion
|
debug "user_version is set to", targetVersion=targetVersion
|
||||||
ok(true)
|
ok(true)
|
|
@ -818,6 +818,7 @@ when isMainModule:
|
||||||
./wakunode2_setup_metrics,
|
./wakunode2_setup_metrics,
|
||||||
./wakunode2_setup_rpc,
|
./wakunode2_setup_rpc,
|
||||||
./wakunode2_setup_sql_migrations,
|
./wakunode2_setup_sql_migrations,
|
||||||
|
./storage/sqlite,
|
||||||
./storage/message/sqlite_store,
|
./storage/message/sqlite_store,
|
||||||
./storage/peer/waku_peer_storage
|
./storage/peer/waku_peer_storage
|
||||||
|
|
||||||
|
@ -853,6 +854,26 @@ when isMainModule:
|
||||||
sqliteDatabase = dbRes.value
|
sqliteDatabase = dbRes.value
|
||||||
|
|
||||||
if not sqliteDatabase.isNil and (conf.persistPeers or conf.persistMessages):
|
if not sqliteDatabase.isNil and (conf.persistPeers or conf.persistMessages):
|
||||||
|
|
||||||
|
## Database vacuuming
|
||||||
|
# TODO: Wrap and move this logic to the appropriate module
|
||||||
|
let
|
||||||
|
pageSize = ?sqliteDatabase.getPageSize()
|
||||||
|
pageCount = ?sqliteDatabase.getPageCount()
|
||||||
|
freelistCount = ?sqliteDatabase.getFreelistCount()
|
||||||
|
|
||||||
|
debug "sqlite database page stats", pageSize=pageSize, pages=pageCount, freePages=freelistCount
|
||||||
|
|
||||||
|
# TODO: Run vacuuming conditionally based on database page stats
|
||||||
|
if conf.dbVacuum and (pageCount > 0 and freelistCount > 0):
|
||||||
|
debug "starting sqlite database vacuuming"
|
||||||
|
|
||||||
|
let resVacuum = sqliteDatabase.vacuum()
|
||||||
|
if resVacuum.isErr():
|
||||||
|
return err("failed to execute vacuum: " & resVacuum.error())
|
||||||
|
|
||||||
|
debug "finished sqlite database vacuuming"
|
||||||
|
|
||||||
# Database initialized. Let's set it up
|
# Database initialized. Let's set it up
|
||||||
sqliteDatabase.runMigrations(conf) # First migrate what we have
|
sqliteDatabase.runMigrations(conf) # First migrate what we have
|
||||||
|
|
||||||
|
@ -876,7 +897,7 @@ when isMainModule:
|
||||||
waku_node_errors.inc(labelValues = ["init_store_failure"])
|
waku_node_errors.inc(labelValues = ["init_store_failure"])
|
||||||
else:
|
else:
|
||||||
storeTuple.mStorage = res.value
|
storeTuple.mStorage = res.value
|
||||||
|
|
||||||
ok(storeTuple)
|
ok(storeTuple)
|
||||||
|
|
||||||
# 2/7 Retrieve dynamic bootstrap nodes
|
# 2/7 Retrieve dynamic bootstrap nodes
|
||||||
|
|
Loading…
Reference in New Issue