nwaku/tests/v2/test_message_store.nim
Sanaz Taheri Boshrooyeh ddf93814fe
Persisting Waku message timestamp & implementing DB migration & convert receiver timestamp data type to float64 (#607)
* adds timestamp to waku message store impl

* stores timestamp as int64

* adds untitest

* stores timestamp as seq of bytes

* minor

* re-orders unittest

* changes receiver timestamp to float64

* unit test for receiver timestamps

* adds comments

* reorder a few lines

* updates changelog

* more updates on changelog

* WIP

* WIP

* adds migration

* more debug messages

* passes the path to the migration scripts from message store module

* adds migration result type

* replaces migrationScripts with migrationScriptsResult

* adds path calculation to the message store

* removes some tests binary file

* removes redundant imports

* comments out user_version assignment in sqlite  init

* more descriptive err messages

* clean up test file

* more info logs

* minor code format

* removes a todo

* minor updates

* remove a binary file

* unit tests for migration utils

* adds split script

* integrates split query to handle scripts with multiple commands

* updates migration script for v1

* updates the v1 migration script

* update user version

* updates script

* fixes a few bugs on the splitScript

* more debug logs

* adds float64 parameter support to sqlite3

* change in timestamp type in the script

* deletes float64 toBytes utils

* enables storage of timestamp as a real number in the sqlite db

* bump up script index

* comment edits

* removes migrate unit test

* adds todo and docstring

* updates changelog

* removes an unused item in .gitignore

* minor

* updates changelog

* organizes imports

* cleans up imports

* WIP

* updates script


fixes a few bugs on the splitScript


more debug logs


adds float64 parameter support to sqlite3


change in timestamp type in the script


deletes float64 toBytes utils

* edits migration util test

* remove an empty test file

* includes migration utils tests in

* deletes unused codes

* tides up imports

* adds range based filter to the filterMigrationScripts

* renames procs: removes Migration

* tides up imports

* edits docstring

* edits docstring

* edits docstring

* removes unused imports

* more clean up

* groups std imports

* updates changelog

* adds docstring for setUserVersion

* adds unittest for the migrate

* Update waku/v2/node/storage/message/waku_message_store.nim

Co-authored-by: RichΛrd <info@richardramos.me>

* Update waku/v2/node/storage/sqlite.nim

Co-authored-by: RichΛrd <info@richardramos.me>

* Update waku/v2/node/storage/sqlite.nim

Co-authored-by: RichΛrd <info@richardramos.me>

* removes split scripts

* fixes a naming issue

* fixes a bug

* fixes a typo

* adds a log re updated user_version

* fixes a proc naming mismatch

* fixes naming mismatch

* more descriptive var names

* adds migration script of the first user version

* moves migration to after persistMessages flag is checked

* deletes unused comment

* fixes a bug

* brings back split script

* adds unit tests for split scripts

* runs scripts one command at a time

* deletes a commented line

* relocates the migrate proc to sqlite.nim

* adds unit test for filter scripts

* adds filterScripts unittest testing varying zero-prefixed user versions

* minor

Co-authored-by: RichΛrd <info@richardramos.me>
2021-06-16 13:23:55 -07:00

115 lines
3.4 KiB
Nim

{.used.}
import
std/[unittest, options, tables, sets, times, os, strutils],
chronos,
../../waku/v2/node/storage/message/waku_message_store,
../../waku/v2/node/storage/sqlite,
../../waku/v2/protocol/waku_store/waku_store,
./utils
suite "Message Store":
test "set and get works":
let
database = SqliteDatabase.init("", inMemory = true)[]
store = WakuMessageStore.init(database)[]
topic = ContentTopic("/waku/2/default-content/proto")
pubsubTopic = "/waku/2/default-waku/proto"
t1 = epochTime()
t2 = epochTime()
t3 = high(float64)
var msgs = @[
WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic, version: uint32(0), timestamp: t1),
WakuMessage(payload: @[byte 1, 2, 3, 4], contentTopic: topic, version: uint32(1), timestamp: t2),
WakuMessage(payload: @[byte 1, 2, 3, 4, 5], contentTopic: topic, version: high(uint32), timestamp: t3),
]
defer: store.close()
var indexes: seq[Index] = @[]
for msg in msgs:
var index = computeIndex(msg)
let output = store.put(index, msg, pubsubTopic)
check output.isOk
indexes.add(index)
# flags for version
var v0Flag, v1Flag, vMaxFlag: bool = false
# flags for sender timestamp
var t1Flag, t2Flag, t3Flag: bool = false
# flags for receiver timestamp
var rt1Flag, rt2Flag, rt3Flag: bool = false
var responseCount = 0
proc data(receiverTimestamp: float64, msg: WakuMessage, psTopic: string) =
responseCount += 1
check msg in msgs
check psTopic == pubsubTopic
# check the correct retrieval of versions
if msg.version == uint32(0): v0Flag = true
if msg.version == uint32(1): v1Flag = true
# high(uint32) is the largest value that fits in uint32, this is to make sure there is no overflow in the storage
if msg.version == high(uint32): vMaxFlag = true
# check correct retrieval of sender timestamps
if msg.timestamp == t1: t1Flag = true
if msg.timestamp == t2: t2Flag = true
if msg.timestamp == t3: t3Flag = true
# check correct retrieval of receiver timestamps
if receiverTimestamp == indexes[0].receivedTime: rt1Flag = true
if receiverTimestamp == indexes[1].receivedTime: rt2Flag = true
if receiverTimestamp == indexes[2].receivedTime: rt3Flag = true
let res = store.getAll(data)
check:
res.isErr == false
responseCount == 3
# check version
v0Flag == true
v1Flag == true
vMaxFlag == true
# check sender timestamp
t1Flag == true
t2Flag == true
t3Flag == true
# check receiver timestamp
rt1Flag == true
rt2Flag == true
rt3Flag == true
test "set and get user version":
let
database = SqliteDatabase.init("", inMemory = true)[]
store = WakuMessageStore.init(database)[]
defer: store.close()
let res = database.setUserVersion(5)
check res.isErr == false
let ver = database.getUserVersion()
check:
ver.isErr == false
ver.value == 5
test "migration":
let
database = SqliteDatabase.init("", inMemory = true)[]
store = WakuMessageStore.init(database)[]
defer: store.close()
template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
let migrationPath = sourceDir
let res = database.migrate(migrationPath, 10)
check:
res.isErr == false
let ver = database.getUserVersion()
check:
ver.isErr == false
ver.value == 10