mirror of
https://github.com/logos-messaging/logos-delivery.git
synced 2026-07-25 22:13:12 +00:00
Store performance improvements (#849)
This commit is contained in:
Hanno Cornelius
@@ -19,7 +19,9 @@ import
|
||||
./v2/test_namespacing_utils,
|
||||
./v2/test_waku_dnsdisc,
|
||||
./v2/test_waku_discv5,
|
||||
./v2/test_enr_utils
|
||||
./v2/test_enr_utils,
|
||||
./v2/test_waku_store_queue,
|
||||
./v2/test_pagination_utils
|
||||
|
||||
when defined(rln):
|
||||
import ./v2/test_waku_rln_relay
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
{.used.}
|
||||
|
||||
import
|
||||
testutils/unittests,
|
||||
chronos,
|
||||
stew/byteutils,
|
||||
libp2p/crypto/crypto,
|
||||
../../waku/v2/utils/pagination
|
||||
|
||||
procSuite "Pagination utils":
|
||||
|
||||
## Helpers
|
||||
proc hashFromStr(input: string): MDigest[256] =
|
||||
var ctx: sha256
|
||||
|
||||
ctx.init()
|
||||
ctx.update(input.toBytes()) # converts the input to bytes
|
||||
|
||||
let hashed = ctx.finish() # computes the hash
|
||||
ctx.clear()
|
||||
|
||||
return hashed
|
||||
|
||||
## Test vars
|
||||
let
|
||||
smallIndex1 = Index(digest: hashFromStr("1234"),
|
||||
receiverTime: 0.00,
|
||||
senderTime: 1000.00)
|
||||
smallIndex2 = Index(digest: hashFromStr("1234567"), # digest is less significant than senderTime
|
||||
receiverTime: 0.00,
|
||||
senderTime: 1000.00)
|
||||
largeIndex1 = Index(digest: hashFromStr("1234"),
|
||||
receiverTime: 0.00,
|
||||
senderTime: 9000.00) # only senderTime differ from smallIndex1
|
||||
largeIndex2 = Index(digest: hashFromStr("12345"), # only digest differs from smallIndex1
|
||||
receiverTime: 0.00,
|
||||
senderTime: 1000.00)
|
||||
eqIndex1 = Index(digest: hashFromStr("0003"),
|
||||
receiverTime: 0.00,
|
||||
senderTime: 54321.00)
|
||||
eqIndex2 = Index(digest: hashFromStr("0003"),
|
||||
receiverTime: 0.00,
|
||||
senderTime: 54321.00)
|
||||
eqIndex3 = Index(digest: hashFromStr("0003"),
|
||||
receiverTime: 9999.00, # receiverTime difference should have no effect on comparisons
|
||||
senderTime: 54321.00)
|
||||
|
||||
|
||||
## Test suite
|
||||
asyncTest "Index comparison":
|
||||
check:
|
||||
# Index comparison with senderTime diff
|
||||
cmp(smallIndex1, largeIndex1) < 0
|
||||
cmp(smallIndex2, largeIndex1) < 0
|
||||
|
||||
# Index comparison with digest diff
|
||||
cmp(smallIndex1, smallIndex2) < 0
|
||||
cmp(smallIndex1, largeIndex2) < 0
|
||||
cmp(smallIndex2, largeIndex2) > 0
|
||||
cmp(largeIndex1, largeIndex2) > 0
|
||||
|
||||
# Index comparison when equal
|
||||
cmp(eqIndex1, eqIndex2) == 0
|
||||
|
||||
# receiverTime difference play no role
|
||||
cmp(eqIndex1, eqIndex3) == 0
|
||||
|
||||
asyncTest "Index equality":
|
||||
check:
|
||||
# Exactly equal
|
||||
eqIndex1 == eqIndex2
|
||||
|
||||
# Receiver time plays no role
|
||||
eqIndex1 == eqIndex3
|
||||
|
||||
# Unequal sender time
|
||||
smallIndex1 != largeIndex1
|
||||
|
||||
# Unequal digest
|
||||
smallIndex1 != smallIndex2
|
||||
|
||||
# Unequal hash and digest
|
||||
smallIndex1 != eqIndex1
|
||||
@@ -1,18 +1,27 @@
|
||||
{.used.}
|
||||
import
|
||||
std/[algorithm, options],
|
||||
std/[algorithm, options, sequtils],
|
||||
testutils/unittests, nimcrypto/sha2,
|
||||
libp2p/protobuf/minprotobuf,
|
||||
../../waku/v2/protocol/waku_store/waku_store,
|
||||
../test_helpers
|
||||
|
||||
|
||||
proc createSampleList(s: int): seq[IndexedWakuMessage] =
|
||||
## takes s as input and outputs a sequence with s amount of IndexedWakuMessage
|
||||
proc createSampleStoreQueue(s: int): StoreQueueRef =
|
||||
## takes s as input and outputs a StoreQueue with s amount of IndexedWakuMessage
|
||||
|
||||
let testStoreQueue = StoreQueueRef.new(s)
|
||||
|
||||
var data {.noinit.}: array[32, byte]
|
||||
for x in data.mitems: x = 1
|
||||
|
||||
for i in 0..<s:
|
||||
result.add(IndexedWakuMessage(msg: WakuMessage(payload: @[byte i]), index: Index(receiverTime: float64(i), senderTime: float64(i), digest: MDigest[256](data: data)) ))
|
||||
discard testStoreQueue.add(IndexedWakuMessage(msg: WakuMessage(payload: @[byte i]),
|
||||
index: Index(receiverTime: float64(i),
|
||||
senderTime: float64(i),
|
||||
digest: MDigest[256](data: data)) ))
|
||||
|
||||
return testStoreQueue
|
||||
|
||||
procSuite "pagination":
|
||||
test "Index computation test":
|
||||
@@ -36,90 +45,48 @@ procSuite "pagination":
|
||||
# the digests of two identical WakuMessages must be the same
|
||||
index1.digest == index2.digest
|
||||
|
||||
test "Index comparison, IndexedWakuMessage comparison, and Sorting tests":
|
||||
var data1 {.noinit.}: array[32, byte]
|
||||
for x in data1.mitems: x = 1
|
||||
var data2 {.noinit.}: array[32, byte]
|
||||
for x in data2.mitems: x = 2
|
||||
var data3 {.noinit.}: array[32, byte]
|
||||
for x in data3.mitems: x = 3
|
||||
|
||||
let
|
||||
index1 = Index(receiverTime: 2, senderTime: 1, digest: MDigest[256](data: data1))
|
||||
index2 = Index(receiverTime: 2, senderTime: 1, digest: MDigest[256](data: data2))
|
||||
index3 = Index(receiverTime: 1, senderTime: 2, digest: MDigest[256](data: data3))
|
||||
iwm1 = IndexedWakuMessage(index: index1)
|
||||
iwm2 = IndexedWakuMessage(index: index2)
|
||||
iwm3 = IndexedWakuMessage(index: index3)
|
||||
|
||||
check:
|
||||
indexComparison(index1, index1) == 0
|
||||
indexComparison(index1, index2) == -1
|
||||
indexComparison(index2, index1) == 1
|
||||
indexComparison(index1, index3) == -1
|
||||
indexComparison(index3, index1) == 1
|
||||
|
||||
check:
|
||||
indexedWakuMessageComparison(iwm1, iwm1) == 0
|
||||
indexedWakuMessageComparison(iwm1, iwm2) == -1
|
||||
indexedWakuMessageComparison(iwm2, iwm1) == 1
|
||||
indexedWakuMessageComparison(iwm1, iwm3) == -1
|
||||
indexedWakuMessageComparison(iwm3, iwm1) == 1
|
||||
|
||||
var sortingList = @[iwm3, iwm1, iwm2]
|
||||
sortingList.sort(indexedWakuMessageComparison)
|
||||
check:
|
||||
sortingList[0] == iwm1
|
||||
sortingList[1] == iwm2
|
||||
sortingList[2] == iwm3
|
||||
|
||||
|
||||
test "Find Index test":
|
||||
let msgList = createSampleList(10)
|
||||
check:
|
||||
msgList.findIndex(msgList[3].index).get() == 3
|
||||
msgList.findIndex(Index()).isNone == true
|
||||
|
||||
test "Forward pagination test":
|
||||
var
|
||||
msgList = createSampleList(10)
|
||||
pagingInfo = PagingInfo(pageSize: 2, cursor: msgList[3].index, direction: PagingDirection.FORWARD)
|
||||
var
|
||||
stQ = createSampleStoreQueue(10)
|
||||
indexList = toSeq(stQ.fwdIterator()).mapIt(it[0]) # Seq copy of the store queue indices for verification
|
||||
msgList = toSeq(stQ.fwdIterator()).mapIt(it[1].msg) # Seq copy of the store queue messages for verification
|
||||
pagingInfo = PagingInfo(pageSize: 2, cursor: indexList[3], direction: PagingDirection.FORWARD)
|
||||
|
||||
# test for a normal pagination
|
||||
var (data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
var (data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 2
|
||||
data == msgList[4..5]
|
||||
newPagingInfo.cursor == msgList[5].index
|
||||
newPagingInfo.cursor == indexList[5]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == pagingInfo.pageSize
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for an initial pagination request with an empty cursor
|
||||
pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 2
|
||||
data == msgList[0..1]
|
||||
newPagingInfo.cursor == msgList[1].index
|
||||
newPagingInfo.cursor == indexList[1]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 2
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for an initial pagination request with an empty cursor to fetch the entire history
|
||||
pagingInfo = PagingInfo(pageSize: 13, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 10
|
||||
data == msgList[0..9]
|
||||
newPagingInfo.cursor == msgList[9].index
|
||||
newPagingInfo.cursor == indexList[9]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 10
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for an empty msgList
|
||||
pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(@[], pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(createSampleStoreQueue(0), pagingInfo)
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.pageSize == 0
|
||||
@@ -128,19 +95,19 @@ procSuite "pagination":
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for a page size larger than the remaining messages
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[3].index, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: indexList[3], direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 6
|
||||
data == msgList[4..9]
|
||||
newPagingInfo.cursor == msgList[9].index
|
||||
newPagingInfo.cursor == indexList[9]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 6
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for a page size larger than the maximum allowed page size
|
||||
pagingInfo = PagingInfo(pageSize: MaxPageSize+1, cursor: msgList[3].index, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
pagingInfo = PagingInfo(pageSize: MaxPageSize+1, cursor: indexList[3], direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
uint64(data.len) <= MaxPageSize
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
@@ -148,18 +115,18 @@ procSuite "pagination":
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for a cursor pointing to the end of the message list
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[9].index, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: indexList[9], direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.cursor == msgList[9].index
|
||||
newPagingInfo.cursor == indexList[9]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 0
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for an invalid cursor
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: computeIndex(WakuMessage(payload: @[byte 10])), direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.cursor == pagingInfo.cursor
|
||||
@@ -168,44 +135,46 @@ procSuite "pagination":
|
||||
error == HistoryResponseError.INVALID_CURSOR
|
||||
|
||||
# test initial paging query over a message list with one message
|
||||
var singleItemMsgList = msgList[0..0]
|
||||
var singleItemMsgList = createSampleStoreQueue(1)
|
||||
pagingInfo = PagingInfo(pageSize: 10, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(singleItemMsgList, pagingInfo)
|
||||
check:
|
||||
data.len == 1
|
||||
newPagingInfo.cursor == msgList[0].index
|
||||
newPagingInfo.cursor == indexList[0]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 1
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test pagination over a message list with one message
|
||||
singleItemMsgList = msgList[0..0]
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[0].index, direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo)
|
||||
singleItemMsgList = createSampleStoreQueue(1)
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: indexList[0], direction: PagingDirection.FORWARD)
|
||||
(data, newPagingInfo, error) = getPage(singleItemMsgList, pagingInfo)
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.cursor == msgList[0].index
|
||||
newPagingInfo.cursor == indexList[0]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 0
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
test "Backward pagination test":
|
||||
var
|
||||
msgList = createSampleList(10)
|
||||
pagingInfo = PagingInfo(pageSize: 2, cursor: msgList[3].index, direction: PagingDirection.BACKWARD)
|
||||
stQ = createSampleStoreQueue(10)
|
||||
indexList = toSeq(stQ.fwdIterator()).mapIt(it[0]) # Seq copy of the store queue indices for verification
|
||||
msgList = toSeq(stQ.fwdIterator()).mapIt(it[1].msg) # Seq copy of the store queue messages for verification
|
||||
pagingInfo = PagingInfo(pageSize: 2, cursor: indexList[3], direction: PagingDirection.BACKWARD)
|
||||
|
||||
# test for a normal pagination
|
||||
var (data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
var (data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data == msgList[1..2]
|
||||
newPagingInfo.cursor == msgList[1].index
|
||||
newPagingInfo.cursor == indexList[1]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == pagingInfo.pageSize
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for an empty msgList
|
||||
pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(@[], pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(createSampleStoreQueue(0), pagingInfo)
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.pageSize == 0
|
||||
@@ -215,40 +184,39 @@ procSuite "pagination":
|
||||
|
||||
# test for an initial pagination request with an empty cursor
|
||||
pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 2
|
||||
data == msgList[8..9]
|
||||
newPagingInfo.cursor == msgList[8].index
|
||||
newPagingInfo.cursor == indexList[8]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 2
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for an initial pagination request with an empty cursor to fetch the entire history
|
||||
pagingInfo = PagingInfo(pageSize: 13, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 10
|
||||
data == msgList[0..9]
|
||||
newPagingInfo.cursor == msgList[0].index
|
||||
newPagingInfo.cursor == indexList[0]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 10
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
|
||||
# test for a page size larger than the remaining messages
|
||||
pagingInfo = PagingInfo(pageSize: 5, cursor: msgList[3].index, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
pagingInfo = PagingInfo(pageSize: 5, cursor: indexList[3], direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data == msgList[0..2]
|
||||
newPagingInfo.cursor == msgList[0].index
|
||||
newPagingInfo.cursor == indexList[0]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 3
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for a page size larger than the Maximum allowed page size
|
||||
pagingInfo = PagingInfo(pageSize: MaxPageSize+1, cursor: msgList[3].index, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
pagingInfo = PagingInfo(pageSize: MaxPageSize+1, cursor: indexList[3], direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
uint64(data.len) <= MaxPageSize
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
@@ -256,19 +224,19 @@ procSuite "pagination":
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for a cursor pointing to the begining of the message list
|
||||
pagingInfo = PagingInfo(pageSize: 5, cursor: msgList[0].index, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
pagingInfo = PagingInfo(pageSize: 5, cursor: indexList[0], direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.cursor == msgList[0].index
|
||||
newPagingInfo.cursor == indexList[0]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 0
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test for an invalid cursor
|
||||
pagingInfo = PagingInfo(pageSize: 5, cursor: computeIndex(WakuMessage(payload: @[byte 10])), direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(msgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(stQ, pagingInfo)
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.cursor == pagingInfo.cursor
|
||||
@@ -277,23 +245,23 @@ procSuite "pagination":
|
||||
error == HistoryResponseError.INVALID_CURSOR
|
||||
|
||||
# test initial paging query over a message list with one message
|
||||
var singleItemMsgList = msgList[0..0]
|
||||
var singleItemMsgList = createSampleStoreQueue(1)
|
||||
pagingInfo = PagingInfo(pageSize: 10, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo)
|
||||
(data, newPagingInfo, error) = getPage(singleItemMsgList, pagingInfo)
|
||||
check:
|
||||
data.len == 1
|
||||
newPagingInfo.cursor == msgList[0].index
|
||||
newPagingInfo.cursor == indexList[0]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 1
|
||||
error == HistoryResponseError.NONE
|
||||
|
||||
# test paging query over a message list with one message
|
||||
singleItemMsgList = msgList[0..0]
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[0].index, direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo)
|
||||
singleItemMsgList = createSampleStoreQueue(1)
|
||||
pagingInfo = PagingInfo(pageSize: 10, cursor: indexList[0], direction: PagingDirection.BACKWARD)
|
||||
(data, newPagingInfo, error) = getPage(singleItemMsgList, pagingInfo)
|
||||
check:
|
||||
data.len == 0
|
||||
newPagingInfo.cursor == msgList[0].index
|
||||
newPagingInfo.cursor == indexList[0]
|
||||
newPagingInfo.direction == pagingInfo.direction
|
||||
newPagingInfo.pageSize == 0
|
||||
error == HistoryResponseError.NONE
|
||||
@@ -319,6 +287,7 @@ suite "time-window history query":
|
||||
timestampDecoded = msgDecoded.value.timestamp
|
||||
check:
|
||||
timestampDecoded == timestamp
|
||||
|
||||
test "Encode/Decode waku message without timestamp":
|
||||
# test the encoding and decoding of a WakuMessage with an empty timestamp field
|
||||
|
||||
|
||||
@@ -634,8 +634,6 @@ procSuite "Waku Store":
|
||||
for wakuMsg in msgList2:
|
||||
# the pubsub topic should be DefaultTopic
|
||||
await proto2.handleMessage(DefaultTopic, wakuMsg)
|
||||
|
||||
|
||||
|
||||
asyncTest "handle temporal history query with a valid time window":
|
||||
var completionFut = newFuture[bool]()
|
||||
@@ -686,22 +684,6 @@ procSuite "Waku Store":
|
||||
check:
|
||||
(await completionFut.withTimeout(5.seconds)) == true
|
||||
|
||||
test "find last seen message":
|
||||
var
|
||||
msgList = @[IndexedWakuMessage(msg: WakuMessage(payload: @[byte 0], contentTopic: ContentTopic("2"))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 1],contentTopic: ContentTopic("1"), timestamp: float(1))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 2],contentTopic: ContentTopic("2"), timestamp: float(2))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 3],contentTopic: ContentTopic("1"), timestamp: float(3))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 4],contentTopic: ContentTopic("2"), timestamp: float(4))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 5],contentTopic: ContentTopic("1"), timestamp: float(9))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 6],contentTopic: ContentTopic("2"), timestamp: float(6))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 7],contentTopic: ContentTopic("1"), timestamp: float(7))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 8],contentTopic: ContentTopic("2"), timestamp: float(8))),
|
||||
IndexedWakuMessage(msg: WakuMessage(payload: @[byte 9],contentTopic: ContentTopic("1"),timestamp: float(5)))]
|
||||
|
||||
check:
|
||||
findLastSeen(msgList) == float(9)
|
||||
|
||||
asyncTest "resume message history":
|
||||
# starts a new node
|
||||
var dialSwitch3 = newStandardSwitch()
|
||||
@@ -808,15 +790,15 @@ procSuite "Waku Store":
|
||||
let store = WakuStore.init(PeerManager.new(newStandardSwitch()), crypto.newRng(), capacity = capacity)
|
||||
|
||||
for i in 1..capacity:
|
||||
await store.handleMessage(pubsubTopic, WakuMessage(payload: @[byte i], contentTopic: contentTopic))
|
||||
await store.handleMessage(pubsubTopic, WakuMessage(payload: @[byte i], contentTopic: contentTopic, timestamp: i.float64))
|
||||
await sleepAsync(1.millis) # Sleep a millisecond to ensure messages are stored chronologically
|
||||
|
||||
check:
|
||||
store.messages.len == capacity # Store is at capacity
|
||||
|
||||
# Test that capacity holds
|
||||
await store.handleMessage(pubsubTopic, WakuMessage(payload: @[byte (capacity + 1)], contentTopic: contentTopic))
|
||||
await store.handleMessage(pubsubTopic, WakuMessage(payload: @[byte (capacity + 1)], contentTopic: contentTopic, timestamp: (capacity + 1).float64))
|
||||
|
||||
check:
|
||||
store.messages.len == capacity # Store is still at capacity
|
||||
store.messages.filterIt(it.msg.payload == @[byte (capacity + 1)]).len == 1 # Simple check to verify last added item is stored
|
||||
store.messages.last().get().msg.payload == @[byte (capacity + 1)] # Simple check to verify last added item is stored
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
{.used.}
|
||||
|
||||
import
|
||||
std/sequtils,
|
||||
testutils/unittests,
|
||||
../../waku/v2/protocol/waku_store/waku_store_types
|
||||
|
||||
procSuite "Sorted store queue":
|
||||
|
||||
# Helper functions
|
||||
proc genIndexedWakuMessage(i: int8): IndexedWakuMessage =
|
||||
## Use i to generate an IndexedWakuMessage
|
||||
var data {.noinit.}: array[32, byte]
|
||||
for x in data.mitems: x = i.byte
|
||||
return IndexedWakuMessage(msg: WakuMessage(payload: @[byte i], timestamp: float64(i)),
|
||||
index: Index(receiverTime: float64(i), senderTime: float64(i), digest: MDigest[256](data: data)))
|
||||
|
||||
# Test variables
|
||||
let
|
||||
capacity = 5
|
||||
unsortedSet = [5,1,3,2,4]
|
||||
|
||||
var testStoreQueue = StoreQueueRef.new(capacity)
|
||||
for i in unsortedSet:
|
||||
discard testStoreQueue.add(genIndexedWakuMessage(i.int8))
|
||||
|
||||
test "Store queue can be created with limited capacity":
|
||||
var stQ = StoreQueueRef.new(capacity)
|
||||
check:
|
||||
stQ.len == 0 # Empty when initialised
|
||||
|
||||
for i in 1..capacity: # Fill up the queue
|
||||
check:
|
||||
stQ.add(genIndexedWakuMessage(i.int8)).isOk()
|
||||
|
||||
check:
|
||||
stQ.len == capacity
|
||||
|
||||
# Add one more. Capacity should not be exceeded.
|
||||
check:
|
||||
stQ.add(genIndexedWakuMessage(capacity.int8 + 1)).isOk()
|
||||
|
||||
check:
|
||||
stQ.len == capacity
|
||||
|
||||
test "Store queue sort-on-insert works":
|
||||
# Walk forward through the set and verify ascending order
|
||||
var prevSmaller = genIndexedWakuMessage(min(unsortedSet).int8 - 1).index
|
||||
for i in testStoreQueue.fwdIterator:
|
||||
let (index, indexedWakuMessage) = i
|
||||
check cmp(index, prevSmaller) > 0
|
||||
prevSmaller = index
|
||||
|
||||
# Walk backward through the set and verify descending order
|
||||
var prevLarger = genIndexedWakuMessage(max(unsortedSet).int8 + 1).index
|
||||
for i in testStoreQueue.bwdIterator:
|
||||
let (index, indexedWakuMessage) = i
|
||||
check cmp(index, prevLarger) < 0
|
||||
prevLarger = index
|
||||
|
||||
test "Can access first item from store queue":
|
||||
let first = testStoreQueue.first()
|
||||
check:
|
||||
first.isOk()
|
||||
first.get().msg.timestamp == 1.0
|
||||
|
||||
# Error condition
|
||||
let emptyQ = StoreQueueRef.new(capacity)
|
||||
check:
|
||||
emptyQ.first().isErr()
|
||||
|
||||
test "Can access last item from store queue":
|
||||
let last = testStoreQueue.last()
|
||||
check:
|
||||
last.isOk()
|
||||
last.get().msg.timestamp == 5.0
|
||||
|
||||
# Error condition
|
||||
let emptyQ = StoreQueueRef.new(capacity)
|
||||
check:
|
||||
emptyQ.last().isErr()
|
||||
|
||||
test "Store queue forward pagination works":
|
||||
proc predicate(i: IndexedWakuMessage): bool = true # no filtering
|
||||
|
||||
var (res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
PagingInfo(pageSize: 3,
|
||||
direction: PagingDirection.FORWARD))
|
||||
|
||||
check:
|
||||
# First page
|
||||
pInfo.pageSize == 3
|
||||
pInfo.direction == PagingDirection.FORWARD
|
||||
pInfo.cursor.senderTime == 3.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.mapIt(it.timestamp.int) == @[1,2,3]
|
||||
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
pInfo)
|
||||
|
||||
check:
|
||||
# Second page
|
||||
pInfo.pageSize == 2
|
||||
pInfo.direction == PagingDirection.FORWARD
|
||||
pInfo.cursor.senderTime == 5.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.mapIt(it.timestamp.int) == @[4,5]
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
pInfo)
|
||||
|
||||
check:
|
||||
# Empty last page
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.FORWARD
|
||||
pInfo.cursor.senderTime == 5.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.len == 0
|
||||
|
||||
test "Store queue backward pagination works":
|
||||
proc predicate(i: IndexedWakuMessage): bool = true # no filtering
|
||||
|
||||
var (res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
PagingInfo(pageSize: 3,
|
||||
direction: PagingDirection.BACKWARD))
|
||||
|
||||
check:
|
||||
# First page
|
||||
pInfo.pageSize == 3
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 3.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.mapIt(it.timestamp.int) == @[3,4,5]
|
||||
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
pInfo)
|
||||
|
||||
check:
|
||||
# Second page
|
||||
pInfo.pageSize == 2
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 1.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.mapIt(it.timestamp.int) == @[1,2]
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
pInfo)
|
||||
|
||||
check:
|
||||
# Empty last page
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 1.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.len == 0
|
||||
|
||||
test "Store queue pagination works with predicate":
|
||||
proc onlyEvenTimes(i: IndexedWakuMessage): bool = i.msg.timestamp.int64 mod 2 == 0
|
||||
proc onlyOddTimes(i: IndexedWakuMessage): bool = i.msg.timestamp.int64 mod 2 != 0
|
||||
|
||||
## Forward pagination: only even timestamped messages
|
||||
|
||||
var (res, pInfo, err) = testStoreQueue.getPage(onlyEvenTimes,
|
||||
PagingInfo(pageSize: 2,
|
||||
direction: PagingDirection.FORWARD))
|
||||
|
||||
check:
|
||||
# First page
|
||||
pInfo.pageSize == 2
|
||||
pInfo.direction == PagingDirection.FORWARD
|
||||
pInfo.cursor.senderTime == 4.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.mapIt(it.timestamp.int) == @[2,4]
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(onlyEvenTimes,
|
||||
pInfo)
|
||||
|
||||
check:
|
||||
# Empty next page
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.FORWARD
|
||||
pInfo.cursor.senderTime == 4.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.len == 0
|
||||
|
||||
## Backward pagination: only odd timestamped messages
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(onlyOddTimes,
|
||||
PagingInfo(pageSize: 2,
|
||||
direction: PagingDirection.BACKWARD))
|
||||
|
||||
check:
|
||||
# First page
|
||||
pInfo.pageSize == 2
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 3.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.mapIt(it.timestamp.int) == @[3,5]
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(onlyOddTimes,
|
||||
pInfo)
|
||||
|
||||
check:
|
||||
# Next page
|
||||
pInfo.pageSize == 1
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 1.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.mapIt(it.timestamp.int) == @[1]
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(onlyOddTimes,
|
||||
pInfo)
|
||||
|
||||
check:
|
||||
# Empty last page
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 1.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.len == 0
|
||||
|
||||
test "Store queue pagination handles invalid cursor":
|
||||
proc predicate(i: IndexedWakuMessage): bool = true # no filtering
|
||||
|
||||
# Invalid cursor in backwards direction
|
||||
|
||||
var (res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
PagingInfo(pageSize: 3,
|
||||
cursor: Index(receiverTime: float64(3), senderTime: float64(3), digest: MDigest[256]()),
|
||||
direction: PagingDirection.BACKWARD))
|
||||
|
||||
check:
|
||||
# Empty response with error
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 3.0
|
||||
err == HistoryResponseError.INVALID_CURSOR
|
||||
res.len == 0
|
||||
|
||||
# Same test, but forward direction
|
||||
|
||||
(res, pInfo, err) = testStoreQueue.getPage(predicate,
|
||||
PagingInfo(pageSize: 3,
|
||||
cursor: Index(receiverTime: float64(3), senderTime: float64(3), digest: MDigest[256]()),
|
||||
direction: PagingDirection.FORWARD))
|
||||
|
||||
check:
|
||||
# Empty response with error
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.FORWARD
|
||||
pInfo.cursor.senderTime == 3.0
|
||||
err == HistoryResponseError.INVALID_CURSOR
|
||||
res.len == 0
|
||||
|
||||
test "Store queue pagination works on empty list":
|
||||
var stQ = StoreQueueRef.new(capacity)
|
||||
check:
|
||||
stQ.len == 0 # Empty when initialised
|
||||
|
||||
proc predicate(i: IndexedWakuMessage): bool = true # no filtering
|
||||
|
||||
# Get page from empty queue in bwd dir
|
||||
|
||||
var (res, pInfo, err) = stQ.getPage(predicate,
|
||||
PagingInfo(pageSize: 3,
|
||||
direction: PagingDirection.BACKWARD))
|
||||
|
||||
check:
|
||||
# Empty response
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.BACKWARD
|
||||
pInfo.cursor.senderTime == 0.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.len == 0
|
||||
|
||||
# Get page from empty queue in fwd dir
|
||||
|
||||
(res, pInfo, err) = stQ.getPage(predicate,
|
||||
PagingInfo(pageSize: 3,
|
||||
direction: PagingDirection.FORWARD))
|
||||
|
||||
check:
|
||||
# Empty response
|
||||
pInfo.pageSize == 0
|
||||
pInfo.direction == PagingDirection.FORWARD
|
||||
pInfo.cursor.senderTime == 0.0
|
||||
err == HistoryResponseError.NONE
|
||||
res.len == 0
|
||||
|
||||
test "Can verify if store queue contains an index":
|
||||
let
|
||||
existingIndex = genIndexedWakuMessage(4).index
|
||||
nonExistingIndex = genIndexedWakuMessage(99).index
|
||||
check:
|
||||
testStoreQueue.contains(existingIndex) == true
|
||||
testStoreQueue.contains(nonExistingIndex) == false
|
||||
@@ -1149,7 +1149,7 @@ procSuite "WakuNode":
|
||||
let index1 = computeIndex(msg1)
|
||||
let output1 = store.put(index1, msg1, DefaultTopic)
|
||||
check output1.isOk
|
||||
node1.wakuStore.messages.add(IndexedWakuMessage(msg: msg1, index: index1, pubsubTopic: DefaultTopic))
|
||||
discard node1.wakuStore.messages.add(IndexedWakuMessage(msg: msg1, index: index1, pubsubTopic: DefaultTopic))
|
||||
|
||||
# now run the resume proc
|
||||
await node1.resume()
|
||||
|
||||
Reference in New Issue
Block a user