diff --git a/tests/waku_archive/test_retention_policy.nim b/tests/waku_archive/test_retention_policy.nim index 0927a15e8..2aa386632 100644 --- a/tests/waku_archive/test_retention_policy.nim +++ b/tests/waku_archive/test_retention_policy.nim @@ -70,11 +70,6 @@ suite "Waku Archive - Retention policy": let retentionPolicy: RetentionPolicy = SizeRetentionPolicy.init(size=sizeLimit) var putFutures = newSeq[Future[ArchiveDriverResult[void]]]() - # variables to check the db size - var pageSize = (waitFor driver.getPagesSize()).tryGet() - var pageCount = (waitFor driver.getPagesCount()).tryGet() - var sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0) - # make sure that the db is empty to before test begins let storedMsg = (waitFor driver.getAllMessages()).tryGet() # if there are messages in db, empty them @@ -95,24 +90,27 @@ suite "Waku Archive - Retention policy": ## Then # calculate the current database size - pageSize = (waitFor driver.getPagesSize()).tryGet() - pageCount = (waitFor driver.getPagesCount()).tryGet() - sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0) + let pageSize = (waitFor driver.getPagesSize()).tryGet() + let pageCount = (waitFor driver.getPagesCount()).tryGet() + let sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0) - # execute policy provided the current db size oveflows + # NOTE: since vacuumin is done manually, this needs to be revisited if vacuuming done automatically + + # get the rows count pre-deletion + let rowsCountBeforeDeletion = (waitFor driver.getMessagesCount()).tryGet() + + # execute policy provided the current db size oveflows, results in rows deletion require (sizeDB >= sizeLimit) require (waitFor retentionPolicy.execute(driver)).isOk() - # update the current db size - pageSize = (waitFor driver.getPagesSize()).tryGet() - pageCount = (waitFor driver.getPagesCount()).tryGet() - sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0) + # get the number or rows from DB + let rowCountAfterDeletion = (waitFor driver.getMessagesCount()).tryGet() check: # size of the database is used to check if the storage limit has been preserved # check the current database size with the limitSize provided by the user # it should be lower - sizeDB <= sizeLimit + rowCountAfterDeletion <= rowsCountBeforeDeletion ## Cleanup (waitFor driver.close()).expect("driver to close") diff --git a/waku/waku_archive/retention_policy/retention_policy_capacity.nim b/waku/waku_archive/retention_policy/retention_policy_capacity.nim index 73430c597..bb1d146cf 100644 --- a/waku/waku_archive/retention_policy/retention_policy_capacity.nim +++ b/waku/waku_archive/retention_policy/retention_policy_capacity.nim @@ -62,23 +62,13 @@ method execute*(p: CapacityRetentionPolicy, driver: ArchiveDriver): Future[RetentionPolicyResult[void]] {.async.} = - let numMessagesRes = await driver.getMessagesCount() - if numMessagesRes.isErr(): - return err("failed to get messages count: " & numMessagesRes.error) - - let numMessages = numMessagesRes.value + let numMessages = (await driver.getMessagesCount()).valueOr: + return err("failed to get messages count: " & error) if numMessages < p.totalCapacity: return ok() - let res = await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow) - if res.isErr(): - return err("deleting oldest messages failed: " & res.error) - - # vacuum to get the deleted pages defragments to save storage space - # this will resize the database size - let resVaccum = await driver.performVacuum() - if resVaccum.isErr(): - return err("vacuumming failed: " & resVaccum.error) + (await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow)).isOkOr: + return err("deleting oldest messages failed: " & error) return ok() diff --git a/waku/waku_archive/retention_policy/retention_policy_size.nim b/waku/waku_archive/retention_policy/retention_policy_size.nim index 30c5858c1..8d645083b 100644 --- a/waku/waku_archive/retention_policy/retention_policy_size.nim +++ b/waku/waku_archive/retention_policy/retention_policy_size.nim @@ -50,36 +50,29 @@ method execute*(p: SizeRetentionPolicy, if pageSize == 0: return err("failed to get Page size: " & pageSizeRes.error) - # keep deleting until the current db size falls within size limit - while true: - # to get the size of the database, pageCount and PageSize is required - # get page count in "messages" database - let pageCount = (await driver.getPagesCount()).valueOr: - return err("failed to get Pages count: " & $error) + # to get the size of the database, pageCount and PageSize is required + # get page count in "messages" database + let pageCount = (await driver.getPagesCount()).valueOr: + return err("failed to get Pages count: " & $error) - # database size in megabytes (Mb) - let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0 + # database size in megabytes (Mb) + let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0 - if totalSizeOfDB < p.sizeLimit: - break + if totalSizeOfDB < p.sizeLimit: + return ok() - # to shread/delete messsges, get the total row/message count - let numMessagesRes = await driver.getMessagesCount() - if numMessagesRes.isErr(): - return err("failed to get messages count: " & numMessagesRes.error) - let numMessages = numMessagesRes.value + # to shread/delete messsges, get the total row/message count + let numMessages = (await driver.getMessagesCount()).valueOr: + return err("failed to get messages count: " & error) - # 80% of the total messages are to be kept, delete others - let pageDeleteWindow = int(float(numMessages) * DeleteLimit) + # NOTE: Using SQLite vacuuming is done manually, we delete a percentage of rows + # if vacumming is done automatically then we aim to check DB size periodially for efficient + # retention policy implementation. - let res = await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow) - if res.isErr(): - return err("deleting oldest messages failed: " & res.error) - - # vacuum to get the deleted pages defragments to save storage space - # this will resize the database size - let resVaccum = await driver.performVacuum() - if resVaccum.isErr(): - return err("vacuumming failed: " & resVaccum.error) + # 80% of the total messages are to be kept, delete others + let pageDeleteWindow = int(float(numMessages) * DeleteLimit) + + (await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow)).isOkOr: + return err("deleting oldest messages failed: " & error) return ok()