From 027cd7f95da5ca044ecacd0f19eb14cdcd5c3d97 Mon Sep 17 00:00:00 2001 From: Sanaz Taheri Boshrooyeh <35961250+staheri14@users.noreply.github.com> Date: Wed, 4 Aug 2021 15:24:36 -0700 Subject: [PATCH] Simplify pagination (#689) * initial simplification * simplify forward pagination * simplifies the backward pagination part * removes old paginate proc * revises comments, cleans up the code * comments on initial paging query * defines temporary variable for the paginate output --- waku/v2/protocol/waku_store/waku_store.nim | 118 +++++++++++++-------- 1 file changed, 76 insertions(+), 42 deletions(-) diff --git a/waku/v2/protocol/waku_store/waku_store.nim b/waku/v2/protocol/waku_store/waku_store.nim index 2d9c707a3..0c573d7ab 100644 --- a/waku/v2/protocol/waku_store/waku_store.nim +++ b/waku/v2/protocol/waku_store/waku_store.nim @@ -283,60 +283,94 @@ proc paginate*(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[IndexedWa cursor = pinfo.cursor pageSize = pinfo.pageSize dir = pinfo.direction + output: (seq[IndexedWakuMessage], PagingInfo, HistoryResponseError) if pageSize == uint64(0): # pageSize being zero indicates that no pagination is required - return (list, pinfo, HistoryResponseError.NONE) + let output = (list, pinfo, HistoryResponseError.NONE) + return output if list.len == 0: # no pagination is needed for an empty list - return (list, PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE) - - var msgList = list # makes a copy of the list - # sorts msgList based on the custom comparison proc indexedWakuMessageComparison - msgList.sort(indexedWakuMessageComparison) + output = (list, PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE) + return output - var initQuery = false - if cursor == Index(): - initQuery = true # an empty cursor means it is an initial query + # adjust pageSize + if pageSize > MaxPageSize: + pageSize = MaxPageSize + + # sort the existing messages + var + msgList = list # makes a copy of the list + total = uint64(msgList.len) + # sorts msgList based on the custom comparison proc indexedWakuMessageComparison + msgList.sort(indexedWakuMessageComparison) + + # set the cursor of the initial paging request + var isInitialQuery = false + if cursor == Index(): # an empty cursor means it is an initial query + isInitialQuery = true case dir of PagingDirection.FORWARD: - cursor = msgList[0].index # perform paging from the begining of the list + cursor = msgList[0].index # set the cursor to the beginning of the list of PagingDirection.BACKWARD: - cursor = msgList[list.len - 1].index # perform paging from the end of the list - var foundIndexOption = msgList.findIndex(cursor) - if foundIndexOption.isNone: # the cursor is not valid - return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.INVALID_CURSOR) - var foundIndex = uint64(foundIndexOption.get()) - var retrievedPageSize, s, e: uint64 - var newCursor: Index # to be returned as part of the new paging info + cursor = msgList[list.len - 1].index # set the cursor to the end of the list + + var cursorIndexOption = msgList.findIndex(cursor) + if cursorIndexOption.isNone: # the cursor is not valid + output = (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.INVALID_CURSOR) + return output + var cursorIndex = uint64(cursorIndexOption.get()) + case dir of PagingDirection.FORWARD: # forward pagination - # the message that is pointed by the cursor is excluded for the retrieved list, this is because this message has already been retrieved by the querier in its prior request - var remainingMessages= uint64(msgList.len) - uint64(foundIndex) - 1 - if initQuery: remainingMessages = remainingMessages + 1 - # the number of queried messages cannot exceed the MaxPageSize and the total remaining messages i.e., msgList.len-foundIndex - retrievedPageSize = min(uint64(pageSize), MaxPageSize).min(remainingMessages) - if initQuery : foundIndex = foundIndex - 1 - s = foundIndex + 1 # non inclusive - e = foundIndex + retrievedPageSize - newCursor = msgList[e].index # the new cursor points to the end of the page - of PagingDirection.BACKWARD: # backward pagination - var remainingMessages = foundIndex - if initQuery: remainingMessages = remainingMessages + 1 - # the number of queried messages cannot exceed the MaxPageSize and the total remaining messages i.e., foundIndex-0 - retrievedPageSize = min(uint64(pageSize), MaxPageSize).min(remainingMessages) - if initQuery : foundIndex = foundIndex + 1 - s = foundIndex - retrievedPageSize - e = foundIndex - 1 - newCursor = msgList[s].index # the new cursor points to the begining of the page + # set the index of the first message in the page + # exclude the message pointing by the cursor + var startIndex = cursorIndex + 1 + # for the initial query, include the message pointing by the cursor + if isInitialQuery: + startIndex = cursorIndex + + # adjust the pageSize based on the total remaining messages + pageSize = min(pageSize, total - startIndex) - if (retrievedPageSize == 0): - return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE) + if (pageSize == 0): + output = (@[], PagingInfo(pageSize: pageSize, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE) + return output + + # set the index of the last message in the page + var endIndex = startIndex + pageSize - 1 + + # retrieve the messages + var retMessages: seq[IndexedWakuMessage] + for i in startIndex..endIndex: + retMessages.add(msgList[i]) + output = (retMessages, PagingInfo(pageSize : pageSize, cursor : msgList[endIndex].index, direction : pinfo.direction), HistoryResponseError.NONE) + return output + + of PagingDirection.BACKWARD: + # set the index of the last message in the page + # exclude the message pointing by the cursor + var endIndex = cursorIndex - 1 + # for the initial query, include the message pointing by the cursor + if isInitialQuery: + endIndex = cursorIndex + + # adjust the pageSize based on the total remaining messages + pageSize = min(pageSize, endIndex + 1) + + if (pageSize == 0): + output = (@[], PagingInfo(pageSize: pageSize, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE) + return output + + # set the index of the first message in the page + var startIndex = endIndex - pageSize + 1 + + # retrieve the messages + var retMessages: seq[IndexedWakuMessage] + for i in startIndex..endIndex: + retMessages.add(msgList[i]) + output = (retMessages, PagingInfo(pageSize : pageSize, cursor : msgList[startIndex].index, direction : pinfo.direction), HistoryResponseError.NONE) + return output - # retrieve the messages - var retMessages: seq[IndexedWakuMessage]= @[] - for i in s..e: - retMessages.add(msgList[i]) - return (retMessages, PagingInfo(pageSize : retrievedPageSize, cursor : newCursor, direction : pinfo.direction), HistoryResponseError.NONE) proc findMessages(w: WakuStore, query: HistoryQuery): HistoryResponse = var data : seq[IndexedWakuMessage] = w.messages