Simplify pagination (#689)

* initial simplification * simplify forward pagination * simplifies the backward pagination part * removes old paginate proc * revises comments, cleans up the code * comments on initial paging query * defines temporary variable for the paginate output
2025-01-26 23:02:30 +00:00 · 2021-08-04 15:24:36 -07:00 · 2021-08-04 15:24:36 -07:00 · 027cd7f95d
commit 027cd7f95d
parent 6b1e0c3079
1 changed files with 76 additions and 42 deletions
--- a/waku/v2/protocol/waku_store/waku_store.nim
+++ b/waku/v2/protocol/waku_store/waku_store.nim
@ -283,60 +283,94 @@ proc paginate*(list: seq[IndexedWakuMessage], pinfo: PagingInfo): (seq[IndexedWa
    cursor = pinfo.cursor
    pageSize = pinfo.pageSize
    dir = pinfo.direction
+    output: (seq[IndexedWakuMessage], PagingInfo, HistoryResponseError) 

  if pageSize == uint64(0): # pageSize being zero indicates that no pagination is required
-    return (list, pinfo, HistoryResponseError.NONE)
+    let output = (list, pinfo, HistoryResponseError.NONE)
+    return output

  if list.len == 0: # no pagination is needed for an empty list
-    return (list, PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE)
-
-  var msgList = list # makes a copy of the list
-  # sorts msgList based on the custom comparison proc indexedWakuMessageComparison
-  msgList.sort(indexedWakuMessageComparison) 
+    output = (list, PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE)
+    return output
  
-  var initQuery = false
-  if cursor == Index(): 
-    initQuery = true # an empty cursor means it is an initial query
+  # adjust pageSize
+  if pageSize > MaxPageSize:
+    pageSize = MaxPageSize
+
+  # sort the existing messages
+  var 
+    msgList = list # makes a copy of the list
+    total = uint64(msgList.len)
+  # sorts msgList based on the custom comparison proc indexedWakuMessageComparison
+  msgList.sort(indexedWakuMessageComparison)  
+  
+  # set the cursor of the initial paging request
+  var isInitialQuery = false
+  if cursor == Index(): # an empty cursor means it is an initial query
+    isInitialQuery = true
    case dir
      of PagingDirection.FORWARD: 
-        cursor = msgList[0].index # perform paging from the begining of the list
+        cursor = msgList[0].index # set the cursor to the beginning of the list
      of PagingDirection.BACKWARD: 
-        cursor = msgList[list.len - 1].index # perform paging from the end of the list
-  var foundIndexOption = msgList.findIndex(cursor) 
-  if foundIndexOption.isNone: # the cursor is not valid
-    return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.INVALID_CURSOR)
-  var foundIndex = uint64(foundIndexOption.get())
-  var retrievedPageSize, s, e: uint64
-  var newCursor: Index # to be returned as part of the new paging info
+        cursor = msgList[list.len - 1].index # set the cursor to the end of the list
+    
+  var cursorIndexOption = msgList.findIndex(cursor) 
+  if cursorIndexOption.isNone: # the cursor is not valid
+    output = (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.INVALID_CURSOR)
+    return output
+  var cursorIndex = uint64(cursorIndexOption.get()) 
+    
  case dir
    of PagingDirection.FORWARD: # forward pagination
-      # the message that is pointed by the cursor is excluded for the retrieved list, this is because this message has already been retrieved by the querier in its prior request
-      var remainingMessages= uint64(msgList.len) - uint64(foundIndex) - 1 
-      if initQuery:  remainingMessages = remainingMessages + 1
-      # the number of queried messages cannot exceed the MaxPageSize and the total remaining messages i.e., msgList.len-foundIndex
-      retrievedPageSize = min(uint64(pageSize), MaxPageSize).min(remainingMessages)  
-      if initQuery : foundIndex = foundIndex - 1
-      s = foundIndex + 1  # non inclusive
-      e = foundIndex + retrievedPageSize 
-      newCursor = msgList[e].index # the new cursor points to the end of the page
-    of PagingDirection.BACKWARD: # backward pagination
-      var remainingMessages = foundIndex 
-      if initQuery:  remainingMessages = remainingMessages + 1
-      # the number of queried messages cannot exceed the MaxPageSize and the total remaining messages i.e., foundIndex-0
-      retrievedPageSize = min(uint64(pageSize), MaxPageSize).min(remainingMessages) 
-      if initQuery : foundIndex = foundIndex + 1
-      s = foundIndex - retrievedPageSize 
-      e = foundIndex - 1
-      newCursor = msgList[s].index # the new cursor points to the begining of the page
+      # set the index of the first message in the page
+      # exclude the message pointing by the cursor 
+      var startIndex = cursorIndex + 1
+      # for the initial query, include the message pointing by the cursor 
+      if isInitialQuery:  
+        startIndex = cursorIndex
+      
+      # adjust the pageSize based on the total remaining messages
+      pageSize = min(pageSize, total - startIndex)  

-  if (retrievedPageSize == 0):
-    return (@[], PagingInfo(pageSize: 0, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE)
+      if (pageSize == 0):
+        output = (@[], PagingInfo(pageSize: pageSize, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE)
+        return output
+      
+      # set the index of the last message in the page
+      var endIndex = startIndex + pageSize - 1 
+
+      # retrieve the messages
+      var retMessages: seq[IndexedWakuMessage]
+      for i in startIndex..endIndex:
+        retMessages.add(msgList[i])
+      output = (retMessages, PagingInfo(pageSize : pageSize, cursor : msgList[endIndex].index, direction : pinfo.direction), HistoryResponseError.NONE)
+      return output
+
+    of PagingDirection.BACKWARD: 
+      # set the index of the last message in the page
+      # exclude the message pointing by the cursor 
+      var endIndex = cursorIndex - 1
+      # for the initial query, include the message pointing by the cursor
+      if isInitialQuery:  
+        endIndex = cursorIndex
+      
+      # adjust the pageSize based on the total remaining messages
+      pageSize = min(pageSize, endIndex + 1) 
+
+      if (pageSize == 0):
+        output =  (@[], PagingInfo(pageSize: pageSize, cursor:pinfo.cursor, direction: pinfo.direction), HistoryResponseError.NONE)
+        return output
+
+      # set the index of the first message in the page
+      var startIndex = endIndex - pageSize + 1
+
+      # retrieve the messages
+      var retMessages: seq[IndexedWakuMessage]
+      for i in startIndex..endIndex:
+        retMessages.add(msgList[i])
+      output = (retMessages, PagingInfo(pageSize : pageSize, cursor : msgList[startIndex].index, direction : pinfo.direction), HistoryResponseError.NONE)
+      return output

-  # retrieve the messages
-  var retMessages: seq[IndexedWakuMessage]= @[]
-  for i in s..e:
-    retMessages.add(msgList[i])
-  return (retMessages, PagingInfo(pageSize : retrievedPageSize, cursor : newCursor, direction : pinfo.direction), HistoryResponseError.NONE)

 proc findMessages(w: WakuStore, query: HistoryQuery): HistoryResponse =
  var data : seq[IndexedWakuMessage] = w.messages