diff --git a/tests/v2/test_waku_pagination.nim b/tests/v2/test_waku_pagination.nim index b31aba753..16fef0695 100644 --- a/tests/v2/test_waku_pagination.nim +++ b/tests/v2/test_waku_pagination.nim @@ -86,98 +86,108 @@ procSuite "pagination": pagingInfo = PagingInfo(pageSize: 2, cursor: msgList[3].index, direction: PagingDirection.FORWARD) # test for a normal pagination - var (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + var (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 2 data == msgList[4..5] newPagingInfo.cursor == msgList[5].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == pagingInfo.pageSize + error == HistoryResponseError.NONE # test for an initial pagination request with an empty cursor pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 2 data == msgList[0..1] newPagingInfo.cursor == msgList[1].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 2 + error == HistoryResponseError.NONE # test for an initial pagination request with an empty cursor to fetch the entire history pagingInfo = PagingInfo(pageSize: 13, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 10 data == msgList[0..9] newPagingInfo.cursor == msgList[9].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 10 + error == HistoryResponseError.NONE # test for an empty msgList pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(@[], pagingInfo) + (data, newPagingInfo, error) = paginate(@[], pagingInfo) check: data.len == 0 newPagingInfo.pageSize == 0 newPagingInfo.direction == pagingInfo.direction newPagingInfo.cursor == pagingInfo.cursor + error == HistoryResponseError.NONE # test for a page size larger than the remaining messages pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[3].index, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 6 data == msgList[4..9] newPagingInfo.cursor == msgList[9].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 6 + error == HistoryResponseError.NONE # test for a page size larger than the maximum allowed page size pagingInfo = PagingInfo(pageSize: MaxPageSize+1, cursor: msgList[3].index, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: uint64(data.len) <= MaxPageSize newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize <= MaxPageSize + error == HistoryResponseError.NONE - # test for a cursor poiting to the end of the message list + # test for a cursor pointing to the end of the message list pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[9].index, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 0 newPagingInfo.cursor == msgList[9].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 0 + error == HistoryResponseError.NONE # test for an invalid cursor pagingInfo = PagingInfo(pageSize: 10, cursor: computeIndex(WakuMessage(payload: @[byte 10])), direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 0 newPagingInfo.cursor == pagingInfo.cursor newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 0 + error == HistoryResponseError.INVALID_CURSOR # test initial paging query over a message list with one message var singleItemMsgList = msgList[0..0] pagingInfo = PagingInfo(pageSize: 10, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(singleItemMsgList, pagingInfo) + (data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo) check: data.len == 1 newPagingInfo.cursor == msgList[0].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 1 + error == HistoryResponseError.NONE # test pagination over a message list with one message singleItemMsgList = msgList[0..0] pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[0].index, direction: PagingDirection.FORWARD) - (data, newPagingInfo) = paginateWithIndex(singleItemMsgList, pagingInfo) + (data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo) check: data.len == 0 newPagingInfo.cursor == msgList[0].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 0 + error == HistoryResponseError.NONE test "Backward pagination test": var @@ -185,98 +195,108 @@ procSuite "pagination": pagingInfo = PagingInfo(pageSize: 2, cursor: msgList[3].index, direction: PagingDirection.BACKWARD) # test for a normal pagination - var (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + var (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data == msgList[1..2] newPagingInfo.cursor == msgList[1].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == pagingInfo.pageSize + error == HistoryResponseError.NONE # test for an empty msgList pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(@[], pagingInfo) + (data, newPagingInfo, error) = paginate(@[], pagingInfo) check: data.len == 0 newPagingInfo.pageSize == 0 newPagingInfo.direction == pagingInfo.direction newPagingInfo.cursor == pagingInfo.cursor + error == HistoryResponseError.NONE # test for an initial pagination request with an empty cursor pagingInfo = PagingInfo(pageSize: 2, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 2 data == msgList[8..9] newPagingInfo.cursor == msgList[8].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 2 + error == HistoryResponseError.NONE # test for an initial pagination request with an empty cursor to fetch the entire history pagingInfo = PagingInfo(pageSize: 13, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 10 data == msgList[0..9] newPagingInfo.cursor == msgList[0].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 10 + error == HistoryResponseError.NONE # test for a page size larger than the remaining messages pagingInfo = PagingInfo(pageSize: 5, cursor: msgList[3].index, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data == msgList[0..2] newPagingInfo.cursor == msgList[0].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 3 + error == HistoryResponseError.NONE # test for a page size larger than the Maximum allowed page size pagingInfo = PagingInfo(pageSize: MaxPageSize+1, cursor: msgList[3].index, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: uint64(data.len) <= MaxPageSize newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize <= MaxPageSize + error == HistoryResponseError.NONE # test for a cursor pointing to the begining of the message list pagingInfo = PagingInfo(pageSize: 5, cursor: msgList[0].index, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 0 newPagingInfo.cursor == msgList[0].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 0 + error == HistoryResponseError.NONE # test for an invalid cursor pagingInfo = PagingInfo(pageSize: 5, cursor: computeIndex(WakuMessage(payload: @[byte 10])), direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(msgList, pagingInfo) + (data, newPagingInfo, error) = paginate(msgList, pagingInfo) check: data.len == 0 newPagingInfo.cursor == pagingInfo.cursor newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 0 + error == HistoryResponseError.INVALID_CURSOR # test initial paging query over a message list with one message var singleItemMsgList = msgList[0..0] pagingInfo = PagingInfo(pageSize: 10, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(singleItemMsgList, pagingInfo) + (data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo) check: data.len == 1 newPagingInfo.cursor == msgList[0].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 1 + error == HistoryResponseError.NONE # test paging query over a message list with one message singleItemMsgList = msgList[0..0] pagingInfo = PagingInfo(pageSize: 10, cursor: msgList[0].index, direction: PagingDirection.BACKWARD) - (data, newPagingInfo) = paginateWithIndex(singleItemMsgList, pagingInfo) + (data, newPagingInfo, error) = paginate(singleItemMsgList, pagingInfo) check: data.len == 0 newPagingInfo.cursor == msgList[0].index newPagingInfo.direction == pagingInfo.direction newPagingInfo.pageSize == 0 + error == HistoryResponseError.NONE suite "time-window history query": test "Encode/Decode waku message with timestamp": diff --git a/tests/v2/test_waku_store.nim b/tests/v2/test_waku_store.nim index c3c8d0119..5fb8428ae 100644 --- a/tests/v2/test_waku_store.nim +++ b/tests/v2/test_waku_store.nim @@ -507,7 +507,7 @@ procSuite "Waku Store": wm = WakuMessage(payload: @[byte 1], contentTopic: defaultContentTopic) index = computeIndex(wm) pagingInfo = PagingInfo(pageSize: 1, cursor: index, direction: PagingDirection.BACKWARD) - res = HistoryResponse(messages: @[wm], pagingInfo:pagingInfo) + res = HistoryResponse(messages: @[wm], pagingInfo:pagingInfo, error: HistoryResponseError.INVALID_CURSOR) pb = res.encode() decodedRes = HistoryResponse.init(pb.buffer) diff --git a/waku/v2/protocol/waku_store/waku_store.nim b/waku/v2/protocol/waku_store/waku_store.nim index 28e705fb7..8718747b2 100644 --- a/waku/v2/protocol/waku_store/waku_store.nim +++ b/waku/v2/protocol/waku_store/waku_store.nim @@ -164,6 +164,10 @@ proc init*(T: type HistoryResponse, buffer: seq[byte]): ProtoResult[T] = discard ? pb.getField(2,pagingInfoBuffer) msg.pagingInfo= ? PagingInfo.init(pagingInfoBuffer) + var error: uint32 + discard ? pb.getField(3, error) + msg.error = HistoryResponseError(error) + ok(msg) proc init*(T: type HistoryRPC, buffer: seq[byte]): ProtoResult[T] = @@ -210,6 +214,8 @@ proc encode*(response: HistoryResponse): ProtoBuffer = result.write(2, response.pagingInfo.encode()) + result.write(3, uint32(ord(response.error))) + proc encode*(rpc: HistoryRPC): ProtoBuffer = result = initProtoBuffer() @@ -244,7 +250,7 @@ proc findIndex*(msgList: seq[IndexedWakuMessage], index: Index): Option[int] = return some(i) return none(int) -proc paginateWithIndex*(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[IndexedWakuMessage], PagingInfo) = +proc paginate*(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[IndexedWakuMessage], PagingInfo, HistoryResponseError) = ## takes list, and performs paging based on pinfo ## returns the page i.e, a sequence of IndexedWakuMessage and the new paging info to be used for the next paging request var @@ -253,10 +259,10 @@ proc paginateWithIndex*(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[ dir = pinfo.direction if pageSize == uint64(0): # pageSize being zero indicates that no pagination is required - return (list, pinfo) + return (list, pinfo, HistoryResponseError.NONE) if list.len == 0: # no pagination is needed for an empty list - return (list, PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction)) + return (list, PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE) var msgList = list # makes a copy of the list # sorts msgList based on the custom comparison proc indexedWakuMessageComparison @@ -272,7 +278,7 @@ proc paginateWithIndex*(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[ cursor = msgList[list.len - 1].index # perform paging from the end of the list var foundIndexOption = msgList.findIndex(cursor) if foundIndexOption.isNone: # the cursor is not valid - return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction)) + return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.INVALID_CURSOR) var foundIndex = uint64(foundIndexOption.get()) var retrievedPageSize, s, e: uint64 var newCursor: Index # to be returned as part of the new paging info @@ -298,20 +304,13 @@ proc paginateWithIndex*(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[ newCursor = msgList[s].index # the new cursor points to the begining of the page if (retrievedPageSize == 0): - return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction)) + return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE) # retrieve the messages + var retMessages: seq[IndexedWakuMessage]= @[] for i in s..e: - result[0].add(msgList[i]) - result[1] = PagingInfo(pageSize : retrievedPageSize, cursor : newCursor, direction : pinfo.direction) - -proc paginateWithoutIndex(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[WakuMessage], PagingInfo) = - ## takes list, and perfomrs paging based on pinfo - ## returns the page i.e, a sequence of WakuMessage and the new paging info to be used for the next paging request - var (indexedData, updatedPagingInfo) = paginateWithIndex(list,pinfo) - for indexedMsg in indexedData: - result[0].add(indexedMsg.msg) - result[1] = updatedPagingInfo + retMessages.add(msgList[i]) + return (retMessages, PagingInfo(pageSize : retrievedPageSize, cursor : newCursor, direction : pinfo.direction), HistoryResponseError.NONE) proc findMessages(w: WakuStore, query: HistoryQuery): HistoryResponse = result = HistoryResponse(messages: newSeq[WakuMessage]()) @@ -342,7 +341,13 @@ proc findMessages(w: WakuStore, query: HistoryQuery): HistoryResponse = # perform pagination - (result.messages, result.pagingInfo)= paginateWithoutIndex(data, query.pagingInfo) + var (indexedWakuMsgList, updatedPagingInfo, error) = paginate(data, query.pagingInfo) + + # extract waku messages + var wakuMsgList = indexedWakuMsgList.mapIt(it.msg) + + var historyRes = HistoryResponse(messages: wakuMsgList, pagingInfo: updatedPagingInfo, error: error) + return historyRes proc init*(ws: WakuStore) {.raises: [Defect, Exception]} = diff --git a/waku/v2/protocol/waku_store/waku_store_types.nim b/waku/v2/protocol/waku_store/waku_store_types.nim index f73005a41..58869a3fe 100644 --- a/waku/v2/protocol/waku_store/waku_store_types.nim +++ b/waku/v2/protocol/waku_store/waku_store_types.nim @@ -52,9 +52,15 @@ type startTime*: float64 # used for time-window query endTime*: float64 # used for time-window query + HistoryResponseError* {.pure.} = enum + ## HistoryResponseError contains error message to inform the querying node about the state of its request + NONE = uint32(0) + INVALID_CURSOR = uint32(1) + HistoryResponse* = object messages*: seq[WakuMessage] pagingInfo*: PagingInfo # used for pagination + error*: HistoryResponseError HistoryRPC* = object requestId*: string