waku-org/nwaku
1
0
Fork 0
mirror of https://github.com/waku-org/nwaku.git synced 2025-02-28 15:00:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: removing automatic vacuuming from retention policy code (#2228)

Browse Source 
* retention policy and testcase updated

* removing dead code

* review updated code
This commit is contained in:
Abhimanyu 2023-11-21 11:27:50 +01:00 committed by GitHub
parent 13fb7c2f74
commit 9ff441ab47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 35 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tests/waku_archive/test_retention_policy.nim
View File

@ -70,11 +70,6 @@ suite "Waku Archive - Retention policy":
let retentionPolicy: RetentionPolicy = SizeRetentionPolicy.init(size=sizeLimit) let retentionPolicy: RetentionPolicy = SizeRetentionPolicy.init(size=sizeLimit)
var putFutures = newSeq[Future[ArchiveDriverResult[void]]]() var putFutures = newSeq[Future[ArchiveDriverResult[void]]]()
# variables to check the db size
var pageSize = (waitFor driver.getPagesSize()).tryGet()
var pageCount = (waitFor driver.getPagesCount()).tryGet()
var sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
# make sure that the db is empty to before test begins # make sure that the db is empty to before test begins
let storedMsg = (waitFor driver.getAllMessages()).tryGet() let storedMsg = (waitFor driver.getAllMessages()).tryGet()
# if there are messages in db, empty them # if there are messages in db, empty them
@ -95,24 +90,27 @@ suite "Waku Archive - Retention policy":
## Then ## Then
# calculate the current database size # calculate the current database size
pageSize = (waitFor driver.getPagesSize()).tryGet() let pageSize = (waitFor driver.getPagesSize()).tryGet()
pageCount = (waitFor driver.getPagesCount()).tryGet() let pageCount = (waitFor driver.getPagesCount()).tryGet()
sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0) let sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
# execute policy provided the current db size oveflows # NOTE: since vacuumin is done manually, this needs to be revisited if vacuuming done automatically
# get the rows count pre-deletion
let rowsCountBeforeDeletion = (waitFor driver.getMessagesCount()).tryGet()
# execute policy provided the current db size oveflows, results in rows deletion
require (sizeDB >= sizeLimit) require (sizeDB >= sizeLimit)
require (waitFor retentionPolicy.execute(driver)).isOk() require (waitFor retentionPolicy.execute(driver)).isOk()
# update the current db size # get the number or rows from DB
pageSize = (waitFor driver.getPagesSize()).tryGet() let rowCountAfterDeletion = (waitFor driver.getMessagesCount()).tryGet()
pageCount = (waitFor driver.getPagesCount()).tryGet()
sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
check: check:
# size of the database is used to check if the storage limit has been preserved # size of the database is used to check if the storage limit has been preserved
# check the current database size with the limitSize provided by the user # check the current database size with the limitSize provided by the user
# it should be lower # it should be lower
sizeDB <= sizeLimit rowCountAfterDeletion <= rowsCountBeforeDeletion
## Cleanup ## Cleanup
(waitFor driver.close()).expect("driver to close") (waitFor driver.close()).expect("driver to close")

18
waku/waku_archive/retention_policy/retention_policy_capacity.nim
View File

@ -62,23 +62,13 @@ method execute*(p: CapacityRetentionPolicy,
driver: ArchiveDriver): driver: ArchiveDriver):
Future[RetentionPolicyResult[void]] {.async.} = Future[RetentionPolicyResult[void]] {.async.} =
let numMessagesRes = await driver.getMessagesCount() let numMessages = (await driver.getMessagesCount()).valueOr:
if numMessagesRes.isErr(): return err("failed to get messages count: " & error)
return err("failed to get messages count: " & numMessagesRes.error)
let numMessages = numMessagesRes.value
if numMessages < p.totalCapacity: if numMessages < p.totalCapacity:
return ok() return ok()
let res = await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow) (await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow)).isOkOr:
if res.isErr(): return err("deleting oldest messages failed: " & error)
return err("deleting oldest messages failed: " & res.error)
# vacuum to get the deleted pages defragments to save storage space
# this will resize the database size
let resVaccum = await driver.performVacuum()
if resVaccum.isErr():
return err("vacuumming failed: " & resVaccum.error)
return ok() return ok()

45
waku/waku_archive/retention_policy/retention_policy_size.nim
View File

@ -50,36 +50,29 @@ method execute*(p: SizeRetentionPolicy,
if pageSize == 0: if pageSize == 0:
return err("failed to get Page size: " & pageSizeRes.error) return err("failed to get Page size: " & pageSizeRes.error)
# keep deleting until the current db size falls within size limit # to get the size of the database, pageCount and PageSize is required
while true: # get page count in "messages" database
# to get the size of the database, pageCount and PageSize is required let pageCount = (await driver.getPagesCount()).valueOr:
# get page count in "messages" database return err("failed to get Pages count: " & $error)
let pageCount = (await driver.getPagesCount()).valueOr:
return err("failed to get Pages count: " & $error)
# database size in megabytes (Mb) # database size in megabytes (Mb)
let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0 let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0
if totalSizeOfDB < p.sizeLimit: if totalSizeOfDB < p.sizeLimit:
break return ok()
# to shread/delete messsges, get the total row/message count # to shread/delete messsges, get the total row/message count
let numMessagesRes = await driver.getMessagesCount() let numMessages = (await driver.getMessagesCount()).valueOr:
if numMessagesRes.isErr(): return err("failed to get messages count: " & error)
return err("failed to get messages count: " & numMessagesRes.error)
let numMessages = numMessagesRes.value
# 80% of the total messages are to be kept, delete others # NOTE: Using SQLite vacuuming is done manually, we delete a percentage of rows
let pageDeleteWindow = int(float(numMessages) * DeleteLimit) # if vacumming is done automatically then we aim to check DB size periodially for efficient
# retention policy implementation.
let res = await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow) # 80% of the total messages are to be kept, delete others
if res.isErr(): let pageDeleteWindow = int(float(numMessages) * DeleteLimit)
return err("deleting oldest messages failed: " & res.error)
(await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow)).isOkOr:
# vacuum to get the deleted pages defragments to save storage space return err("deleting oldest messages failed: " & error)
# this will resize the database size
let resVaccum = await driver.performVacuum()
if resVaccum.isErr():
return err("vacuumming failed: " & resVaccum.error)
return ok() return ok()