chore(chat): request mailserver messages since last request

Prior to this commit, the app would request messages from the last
24 hours for all joined channels.
The changes made here ensure that messages since the *last* request
are requested. The last request for each topic/chat is stored in the
nodes mailserver topics.

When no channels are joined, the default is still 24 hours, however,
as soon as a channel is joined, the last request time will be updated
and honored in the next request.

src/app/chat/event_handling.nim

@@ -1,3 +1,4 @@
+import sugar, sequtils, times
 
 proc handleChatEvents(self: ChatController) =
   # Display already saved messages
@@ -67,9 +68,18 @@ proc handleChatEvents(self: ChatController) =
 
 proc handleMailserverEvents(self: ChatController) =
   self.status.events.on("mailserverTopics") do(e: Args):
-    self.status.mailservers.addTopics(TopicArgs(e).topics)
+    var topics = TopicArgs(e).topics
+    for topic in topics:
+      topic.lastRequest = times.toUnix(times.getTime())
+      self.status.mailservers.addMailserverTopic(topic)
+
     if(self.status.mailservers.isSelectedMailserverAvailable):
-      self.status.mailservers.requestMessages()
+      self.status.mailservers.requestMessages(topics.map(t => t.topic))
 
   self.status.events.on("mailserverAvailable") do(e:Args):
-    self.status.mailservers.requestMessages()
+    let mailserverTopics = self.status.mailservers.getMailserverTopics()
+    var fromValue: int64 = times.toUnix(times.getTime()) - 86400
+
+    if mailserverTopics.len > 0:
+       fromValue = min(mailserverTopics.map(topic => topic.lastRequest))
+    self.status.mailservers.requestMessages(mailserverTopics.map(t => t.topic), fromValue)

src/status/chat.nim

@@ -1,8 +1,10 @@
 import eventemitter, json, strutils, sequtils, tables, chronicles, sugar
 import libstatus/settings as status_settings
 import libstatus/chat as status_chat
+import libstatus/mailservers as status_mailservers
 import libstatus/stickers as status_stickers
 import libstatus/types
+import mailservers
 import profile/profile
 import chat/[chat, message]
 import ../signals/messages
@@ -29,7 +31,7 @@ type
     chats*: seq[Chat]
 
   TopicArgs* = ref object of Args
-    topics*: seq[string]
+    topics*: seq[MailserverTopic]
 
   MsgsLoadedArgs* = ref object of Args
     messages*: seq[Message]
@@ -91,11 +93,17 @@ proc join*(self: ChatModel, chatId: string, chatType: ChatType) =
   status_chat.saveChat(chatId, chatType.isOneToOne, true, chat.color)
   let filterResult = status_chat.loadFilters(@[status_chat.buildFilter(chat)])
 
-  var topics:seq[string] = @[]
+  var topics:seq[MailserverTopic] = @[]
   let parsedResult = parseJson(filterResult)["result"]
   for topicObj in parsedResult:
     if ($topicObj["chatId"].getStr == chatId):
-      topics.add($topicObj["topic"].getStr)
+      topics.add(MailserverTopic(
+        topic: $topicObj["topic"].getStr,
+        discovery: topicObj["discovery"].getBool,
+        negotiated: topicObj["negotiated"].getBool,
+        chatIds: @[chatId],
+        lastRequest: 1
+      ))
 
   if (topics.len == 0): 
     warn "No topics found for chats. Cannot load past messages"
@@ -184,17 +192,25 @@ proc init*(self: ChatModel) =
 
   self.events.emit("chatsLoaded", ChatArgs(chats: chatList))
 
-  var topics:seq[string] = @[]
+  var topics:seq[MailserverTopic] = @[]
   let parsedResult = parseJson(filterResult)["result"]
   for topicObj in parsedResult:
     # Only add topics for chats the user has joined
     let topic_chat = topicObj["chatId"].getStr
     if self.channels.hasKey(topic_chat) and self.channels[topic_chat].isActive:
-      topics.add($topicObj["topic"].getStr)
+      
+      topics.add(MailserverTopic(
+        topic: $topicObj["topic"].getStr,
+        discovery: topicObj["discovery"].getBool,
+        negotiated: topicObj["negotiated"].getBool,
+        chatIds: @[self.channels[topic_chat].id],
+        lastRequest: 1
+      ))
 
   if (topics.len == 0): 
     warn "No topics found for chats. Cannot load past messages"
   else:
+    self.events.once("mailserverAvailable") do(a: Args):
       self.events.emit("mailserverTopics", TopicArgs(topics: topics));
 
 proc leave*(self: ChatModel, chatId: string) =

src/status/libstatus/mailservers.nim

@@ -1,6 +1,14 @@
 import json, times
 import core, utils
 
+type
+  MailserverTopic* = ref object
+    topic*: string
+    discovery*: bool
+    negotiated*: bool
+    chatIds*: seq[string]
+    lastRequest*: int64
+
 proc getMailservers*(): array[0..8, (string, string)] =
   result = [
     (
@@ -55,7 +63,11 @@ proc update*(peer: string) =
 proc delete*(peer: string) =
   discard callPrivateRPC("mailservers_deleteMailserver", %* [peer])
 
-proc requestMessages*(topics: seq[string], symKeyID: string, peer: string, numberOfMessages: int) =
+proc requestMessages*(topics: seq[string], symKeyID: string, peer: string, numberOfMessages: int, fromTimestamp: int64 = 0) =
+  var fromValue = (times.toUnix(times.getTime()) - 86400)
+  if fromTimestamp != 0:
+    fromValue = fromTimestamp
+
   echo callPrivateRPC("requestMessages".prefix, %* [
     {
         "topics": topics,
@@ -64,6 +76,18 @@ proc requestMessages*(topics: seq[string], symKeyID: string, peer: string, numbe
         "timeout": 30,
         "limit": numberOfMessages,
         "cursor": nil,
-        "from": (times.toUnix(times.getTime()) - 86400) # Unhardcode this. Need to keep the last fetch in a DB
+        "from": fromValue
     }
   ])
+
+proc getMailserverTopics*(): string = 
+  return callPrivateRPC("mailservers_getMailserverTopics", %*[])
+
+proc addMailserverTopic*(topic: MailserverTopic): string =
+  return callPrivateRPC("mailservers_addMailserverTopic", %*[{
+    "topic": topic.topic,
+    "discovery?": topic.discovery,
+    "negotiated?": topic.negotiated,
+    "chat-ids": topic.chatIds,
+    "last-request": topic.lastRequest
+  }])

src/status/mailservers.nim

@@ -50,12 +50,8 @@ proc newMailserverModel*(events: EventEmitter): MailserverModel =
   result.events = events
   result.nodes = initTable[string, MailserverStatus]()
   result.selectedMailserver = ""
-  result.topics = initHashSet[string]()
   result.lock.initLock()
 
-proc addTopics*(self: MailserverModel, topics: seq[string]) =
-  for t in topics: self.topics.incl(t)
-
 proc trustPeer*(self: MailserverModel, enode:string) = 
   markTrustedPeer(enode)
   self.nodes[enode] = MailserverStatus.Trusted
@@ -132,10 +128,27 @@ proc peerSummaryChange*(self: MailserverModel, peers: seq[string]) =
     self.nodes[peer] = MailserverStatus.Connected
     self.events.emit("peerConnected", MailserverArg(peer: peer))
 
-proc requestMessages*(self: MailserverModel) =
+proc requestMessages*(self: MailserverModel, topics: seq[string], fromValue: int64 = 0) =
   debug "Requesting messages from", mailserver=self.selectedMailserver
   let generatedSymKey = status_chat.generateSymKeyFromPassword()
-  status_mailservers.requestMessages(toSeq(self.topics), generatedSymKey, self.selectedMailserver, 1000)
+  status_mailservers.requestMessages(topics, generatedSymKey, self.selectedMailserver, 1000, fromValue)
+
+proc getMailserverTopics*(self: MailserverModel): seq[MailserverTopic] =
+  let response = status_mailservers.getMailserverTopics()
+  let topics = parseJson(response)["result"]
+  result = @[]
+  if topics.kind != JNull:
+    for topic in topics:
+      result.add(MailserverTopic(
+        topic: topic["topic"].getStr,
+        discovery: topic["discovery?"].getBool,
+        negotiated: topic["negotiated?"].getBool,
+        chatIds: topic["chat-ids"].to(seq[string]),
+        lastRequest: topic["last-request"].getInt
+      ))
+
+proc addMailserverTopic*(self: MailserverModel, topic: MailserverTopic) =
+  discard status_mailservers.addMailserverTopic(topic)
 
 proc autoConnect*(self: MailserverModel) =
   let mailserversReply = parseJson(status_mailservers.ping(500))["result"]