2022-11-22 18:40:24 +00:00
|
|
|
{.used.}
|
|
|
|
|
|
|
|
import
|
2024-03-15 23:08:47 +00:00
|
|
|
std/[options, sequtils, algorithm], testutils/unittests, libp2p/protobuf/minprotobuf
|
2022-11-22 18:40:24 +00:00
|
|
|
import
|
2024-07-05 22:03:38 +00:00
|
|
|
waku/[
|
|
|
|
waku_archive,
|
|
|
|
waku_archive/driver/queue_driver/queue_driver {.all.},
|
|
|
|
waku_archive/driver/queue_driver/index,
|
|
|
|
waku_core,
|
|
|
|
],
|
2023-02-13 10:43:49 +00:00
|
|
|
../testlib/common,
|
2023-04-05 14:01:51 +00:00
|
|
|
../testlib/wakucore
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
proc getTestQueueDriver(numMessages: int): QueueDriver =
|
|
|
|
let testQueueDriver = QueueDriver.new(numMessages)
|
|
|
|
|
|
|
|
var data {.noinit.}: array[32, byte]
|
2024-03-15 23:08:47 +00:00
|
|
|
for x in data.mitems:
|
|
|
|
x = 1
|
2022-11-22 18:40:24 +00:00
|
|
|
|
2024-03-15 23:08:47 +00:00
|
|
|
for i in 0 ..< numMessages:
|
2024-03-12 11:51:03 +00:00
|
|
|
let msg = WakuMessage(payload: @[byte i], timestamp: Timestamp(i))
|
|
|
|
|
|
|
|
let index = Index(
|
2024-07-12 16:19:12 +00:00
|
|
|
time: Timestamp(i),
|
2024-04-25 13:09:52 +00:00
|
|
|
hash: computeMessageHash(DefaultPubsubTopic, msg),
|
2024-07-12 16:19:12 +00:00
|
|
|
pubsubTopic: DefaultPubsubTopic,
|
2022-11-22 18:40:24 +00:00
|
|
|
)
|
2024-03-15 23:08:47 +00:00
|
|
|
|
2024-03-12 11:51:03 +00:00
|
|
|
discard testQueueDriver.add(index, msg)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
return testQueueDriver
|
|
|
|
|
|
|
|
procSuite "Queue driver - pagination":
|
|
|
|
let driver = getTestQueueDriver(10)
|
|
|
|
let
|
|
|
|
indexList: seq[Index] = toSeq(driver.fwdIterator()).mapIt(it[0])
|
2024-03-12 11:51:03 +00:00
|
|
|
msgList: seq[WakuMessage] = toSeq(driver.fwdIterator()).mapIt(it[1])
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Forward pagination - normal pagination":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 2
|
|
|
|
cursor: Option[Index] = some(indexList[3])
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 2
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[4 .. 5]
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Forward pagination - initial pagination request with an empty cursor":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 2
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 2
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[0 .. 1]
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Forward pagination - initial pagination request with an empty cursor to fetch the entire history":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 13
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 10
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[0 .. 9]
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Forward pagination - empty msgList":
|
|
|
|
## Given
|
|
|
|
let driver = getTestQueueDriver(0)
|
|
|
|
let
|
|
|
|
pageSize: uint = 2
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 0
|
|
|
|
|
|
|
|
test "Forward pagination - page size larger than the remaining messages":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 10
|
|
|
|
cursor: Option[Index] = some(indexList[3])
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 6
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[4 .. 9]
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Forward pagination - page size larger than the maximum allowed page size":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = MaxPageSize + 1
|
|
|
|
cursor: Option[Index] = some(indexList[3])
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
uint(data.len) <= MaxPageSize
|
|
|
|
|
|
|
|
test "Forward pagination - cursor pointing to the end of the message list":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 10
|
|
|
|
cursor: Option[Index] = some(indexList[9])
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 0
|
|
|
|
|
|
|
|
test "Forward pagination - invalid cursor":
|
|
|
|
## Given
|
2024-03-15 23:08:47 +00:00
|
|
|
let msg = fakeWakuMessage(payload = @[byte 10])
|
2024-07-12 16:19:12 +00:00
|
|
|
let index = Index(hash: computeMessageHash(DefaultPubsubTopic, msg))
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
let
|
|
|
|
pageSize: uint = 10
|
|
|
|
cursor: Option[Index] = some(index)
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
|
|
|
let error = page.tryError()
|
|
|
|
check:
|
|
|
|
error == QueueDriverErrorKind.INVALID_CURSOR
|
|
|
|
|
|
|
|
test "Forward pagination - initial paging query over a message list with one message":
|
|
|
|
## Given
|
|
|
|
let driver = getTestQueueDriver(1)
|
|
|
|
let
|
|
|
|
pageSize: uint = 10
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 1
|
|
|
|
|
|
|
|
test "Forward pagination - pagination over a message list with one message":
|
|
|
|
## Given
|
|
|
|
let driver = getTestQueueDriver(1)
|
|
|
|
let
|
|
|
|
pageSize: uint = 10
|
|
|
|
cursor: Option[Index] = some(indexList[0])
|
|
|
|
forward: bool = true
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 0
|
|
|
|
|
|
|
|
test "Forward pagination - with pradicate":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 3
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward = true
|
|
|
|
|
2024-03-15 23:08:47 +00:00
|
|
|
proc onlyEvenTimes(index: Index, msg: WakuMessage): bool =
|
|
|
|
msg.timestamp.int64 mod 2 == 0
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(
|
|
|
|
pageSize = pageSize, forward = forward, cursor = cursor, predicate = onlyEvenTimes
|
|
|
|
)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.mapIt(it.timestamp.int) == @[0, 2, 4]
|
|
|
|
|
|
|
|
test "Backward pagination - normal pagination":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 2
|
|
|
|
cursor: Option[Index] = some(indexList[3])
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[1 .. 2].reversed
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Backward pagination - empty msgList":
|
|
|
|
## Given
|
|
|
|
let driver = getTestQueueDriver(0)
|
|
|
|
let
|
|
|
|
pageSize: uint = 2
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 0
|
|
|
|
|
|
|
|
test "Backward pagination - initial pagination request with an empty cursor":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 2
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 2
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[8 .. 9].reversed
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Backward pagination - initial pagination request with an empty cursor to fetch the entire history":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 13
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 10
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[0 .. 9].reversed
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Backward pagination - page size larger than the remaining messages":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 5
|
|
|
|
cursor: Option[Index] = some(indexList[3])
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
2024-03-15 23:08:47 +00:00
|
|
|
data == msgList[0 .. 2].reversed
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
test "Backward pagination - page size larger than the Maximum allowed page size":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = MaxPageSize + 1
|
|
|
|
cursor: Option[Index] = some(indexList[3])
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
uint(data.len) <= MaxPageSize
|
|
|
|
|
|
|
|
test "Backward pagination - cursor pointing to the begining of the message list":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 5
|
|
|
|
cursor: Option[Index] = some(indexList[0])
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 0
|
|
|
|
|
|
|
|
test "Backward pagination - invalid cursor":
|
|
|
|
## Given
|
2024-03-15 23:08:47 +00:00
|
|
|
let msg = fakeWakuMessage(payload = @[byte 10])
|
2024-07-12 16:19:12 +00:00
|
|
|
let index = Index(hash: computeMessageHash(DefaultPubsubTopic, msg))
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
let
|
|
|
|
pageSize: uint = 2
|
|
|
|
cursor: Option[Index] = some(index)
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
|
|
|
let error = page.tryError()
|
|
|
|
check:
|
|
|
|
error == QueueDriverErrorKind.INVALID_CURSOR
|
|
|
|
|
|
|
|
test "Backward pagination - initial paging query over a message list with one message":
|
|
|
|
## Given
|
|
|
|
let driver = getTestQueueDriver(1)
|
|
|
|
let
|
|
|
|
pageSize: uint = 10
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 1
|
|
|
|
|
|
|
|
test "Backward pagination - paging query over a message list with one message":
|
|
|
|
## Given
|
|
|
|
let driver = getTestQueueDriver(1)
|
|
|
|
let
|
|
|
|
pageSize: uint = 10
|
|
|
|
cursor: Option[Index] = some(indexList[0])
|
|
|
|
forward: bool = false
|
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
|
|
|
data.len == 0
|
|
|
|
|
|
|
|
test "Backward pagination - with predicate":
|
|
|
|
## Given
|
|
|
|
let
|
|
|
|
pageSize: uint = 3
|
|
|
|
cursor: Option[Index] = none(Index)
|
|
|
|
forward = false
|
|
|
|
|
2024-03-15 23:08:47 +00:00
|
|
|
proc onlyOddTimes(index: Index, msg: WakuMessage): bool =
|
|
|
|
msg.timestamp.int64 mod 2 != 0
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## When
|
2024-03-15 23:08:47 +00:00
|
|
|
let page = driver.getPage(
|
|
|
|
pageSize = pageSize, forward = forward, cursor = cursor, predicate = onlyOddTimes
|
|
|
|
)
|
2022-11-22 18:40:24 +00:00
|
|
|
|
|
|
|
## Then
|
2024-07-12 16:19:12 +00:00
|
|
|
let data = page.tryGet().mapIt(it[2])
|
2022-11-22 18:40:24 +00:00
|
|
|
check:
|
2024-03-15 23:08:47 +00:00
|
|
|
data.mapIt(it.timestamp.int) == @[5, 7, 9].reversed
|