waku-org/nwaku
1
0
Fork 0
mirror of https://github.com/waku-org/nwaku.git synced 2025-02-28 15:00:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: removing automatic vacuuming from retention policy code (#2228)

Browse Source 
* retention policy and testcase updated

* removing dead code

* review updated code
This commit is contained in:
Abhimanyu 2023-11-21 11:27:50 +01:00 committed by GitHub
parent 13fb7c2f74
commit 9ff441ab47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 35 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tests/waku_archive/test_retention_policy.nim
View File

@ -70,11 +70,6 @@ suite "Waku Archive - Retention policy":
let retentionPolicy: RetentionPolicy = SizeRetentionPolicy.init(size=sizeLimit)
var putFutures = newSeq[Future[ArchiveDriverResult[void]]]()
# variables to check the db size
var pageSize = (waitFor driver.getPagesSize()).tryGet()
var pageCount = (waitFor driver.getPagesCount()).tryGet()
var sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
# make sure that the db is empty to before test begins
let storedMsg = (waitFor driver.getAllMessages()).tryGet()
# if there are messages in db, empty them
@ -95,24 +90,27 @@ suite "Waku Archive - Retention policy":
## Then
# calculate the current database size
pageSize = (waitFor driver.getPagesSize()).tryGet()
pageCount = (waitFor driver.getPagesCount()).tryGet()
sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
let pageSize = (waitFor driver.getPagesSize()).tryGet()
let pageCount = (waitFor driver.getPagesCount()).tryGet()
let sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
# execute policy provided the current db size oveflows
# NOTE: since vacuumin is done manually, this needs to be revisited if vacuuming done automatically
# get the rows count pre-deletion
let rowsCountBeforeDeletion = (waitFor driver.getMessagesCount()).tryGet()
# execute policy provided the current db size oveflows, results in rows deletion
require (sizeDB >= sizeLimit)
require (waitFor retentionPolicy.execute(driver)).isOk()
# update the current db size
pageSize = (waitFor driver.getPagesSize()).tryGet()
pageCount = (waitFor driver.getPagesCount()).tryGet()
sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
# get the number or rows from DB
let rowCountAfterDeletion = (waitFor driver.getMessagesCount()).tryGet()
check:
# size of the database is used to check if the storage limit has been preserved
# check the current database size with the limitSize provided by the user
# it should be lower
sizeDB <= sizeLimit
rowCountAfterDeletion <= rowsCountBeforeDeletion
## Cleanup
(waitFor driver.close()).expect("driver to close")

18
waku/waku_archive/retention_policy/retention_policy_capacity.nim
View File

@ -62,23 +62,13 @@ method execute*(p: CapacityRetentionPolicy,
driver: ArchiveDriver):
Future[RetentionPolicyResult[void]] {.async.} =
let numMessagesRes = await driver.getMessagesCount()
if numMessagesRes.isErr():
return err("failed to get messages count: " & numMessagesRes.error)
let numMessages = numMessagesRes.value
let numMessages = (await driver.getMessagesCount()).valueOr:
return err("failed to get messages count: " & error)
if numMessages < p.totalCapacity:
return ok()
let res = await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow)
if res.isErr():
return err("deleting oldest messages failed: " & res.error)
# vacuum to get the deleted pages defragments to save storage space
# this will resize the database size
let resVaccum = await driver.performVacuum()
if resVaccum.isErr():
return err("vacuumming failed: " & resVaccum.error)
(await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow)).isOkOr:
return err("deleting oldest messages failed: " & error)
return ok()

45
waku/waku_archive/retention_policy/retention_policy_size.nim
View File

@ -50,36 +50,29 @@ method execute*(p: SizeRetentionPolicy,
if pageSize == 0:
return err("failed to get Page size: " & pageSizeRes.error)
# keep deleting until the current db size falls within size limit
while true:
# to get the size of the database, pageCount and PageSize is required
# get page count in "messages" database
let pageCount = (await driver.getPagesCount()).valueOr:
return err("failed to get Pages count: " & $error)
# to get the size of the database, pageCount and PageSize is required
# get page count in "messages" database
let pageCount = (await driver.getPagesCount()).valueOr:
return err("failed to get Pages count: " & $error)
# database size in megabytes (Mb)
let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0
# database size in megabytes (Mb)
let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0
if totalSizeOfDB < p.sizeLimit:
break
if totalSizeOfDB < p.sizeLimit:
return ok()
# to shread/delete messsges, get the total row/message count
let numMessagesRes = await driver.getMessagesCount()
if numMessagesRes.isErr():
return err("failed to get messages count: " & numMessagesRes.error)
let numMessages = numMessagesRes.value
# to shread/delete messsges, get the total row/message count
let numMessages = (await driver.getMessagesCount()).valueOr:
return err("failed to get messages count: " & error)
# 80% of the total messages are to be kept, delete others
let pageDeleteWindow = int(float(numMessages) * DeleteLimit)
# NOTE: Using SQLite vacuuming is done manually, we delete a percentage of rows
# if vacumming is done automatically then we aim to check DB size periodially for efficient
# retention policy implementation.
let res = await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow)
if res.isErr():
return err("deleting oldest messages failed: " & res.error)
# vacuum to get the deleted pages defragments to save storage space
# this will resize the database size
let resVaccum = await driver.performVacuum()
if resVaccum.isErr():
return err("vacuumming failed: " & resVaccum.error)
# 80% of the total messages are to be kept, delete others
let pageDeleteWindow = int(float(numMessages) * DeleteLimit)
(await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow)).isOkOr:
return err("deleting oldest messages failed: " & error)
return ok()