Fetch missed range of messages

src/status_im/chat/db.cljs

@@ -5,7 +5,8 @@
             [status-im.chat.commands.core :as commands]
             [status-im.chat.commands.input :as commands.input]
             [status-im.group-chats.db :as group-chats.db]
-            [status-im.utils.gfycat.core :as gfycat]))
+            [status-im.utils.gfycat.core :as gfycat]
+            [status-im.transport.partitioned-topic :as topic]))
 
 (defn group-chat-name
   [{:keys [public? name]}]
@@ -49,11 +50,28 @@
           {}
           chats))
 
+(defn topic-by-current-chat
+  [{:keys [current-chat-id chats] :as db}]
+  (let [{:keys [public?]} (get chats current-chat-id)
+        public-key (get-in db [:account/account :public-key])]
+    (if public?
+      (get-in db [:transport/chats current-chat-id :topic])
+      (topic/public-key->discovery-topic-hash public-key))))
+
+(defn messages-gap
+  [mailserver-topics topic]
+  (let [{:keys [gap-from gap-to]}
+        (get mailserver-topics topic)]
+    {:from    gap-from
+     :to      gap-to
+     :exists? (and gap-from gap-to
+                   (> gap-to gap-from))}))
+
 (defn sort-message-groups
   "Sorts message groups according to timestamp of first message in group"
   [message-groups messages]
   (sort-by
-   (comp unchecked-negate :timestamp (partial get messages) :message-id first second)
+   (comp :timestamp (partial get messages) :message-id first second)
    message-groups))
 
 (defn quoted-message-data
@@ -64,27 +82,93 @@
     {:from from
      :text (:text content)}))
 
+(defn add-datemark
+  [[datemark
+    message-references]]
+  (let [{:keys [whisper-timestamp timestamp]} (first message-references)]
+    (conj message-references
+          {:value             datemark
+           :type              :datemark
+           :whisper-timestamp whisper-timestamp
+           :timestamp         timestamp})))
+
+(defn datemark? [{:keys [type]}]
+  (= type :datemark))
+
+(defn transform-message
+  [messages message-statuses referenced-messages]
+  (fn [{:keys [message-id timestamp-str] :as reference}]
+    (if (datemark? reference)
+      reference
+      (let [{:keys [content] :as message} (get messages message-id)
+            {:keys [response-to response-to-v2]} content
+            quote (some-> (or response-to-v2 response-to)
+                          (quoted-message-data messages referenced-messages))]
+        (cond-> (-> message
+                    (update :content dissoc :response-to :response-to-v2)
+                    (assoc :timestamp-str timestamp-str
+                           :user-statuses (get message-statuses message-id)))
+          ;; quoted message reference
+          quote
+          (assoc-in [:content :response-to] quote))))))
+
+(defn check-gap
+  [{:keys [exists? from]} previous-message next-message gap-added?]
+  (let [previous-timestamp (:whisper-timestamp previous-message)
+        next-whisper-timestamp     (:whisper-timestamp next-message)
+        next-timestamp (quot (:timestamp next-message) 1000)
+        ignore-next-message? (> (js/Math.abs
+                                 (- next-whisper-timestamp next-timestamp))
+                                120)]
+    (and (not gap-added?)
+         (or (and exists? previous-timestamp next-whisper-timestamp
+                  (not ignore-next-message?)
+                  (< previous-timestamp from next-whisper-timestamp))
+             (and exists? (nil? next-message))))))
+
+(defn add-gap [messages gap]
+  (conj messages
+        {:type  :gap
+         :value (str (:from gap))}))
+
 (defn messages-with-datemarks-and-statuses
   "Converts message groups into sequence of messages interspersed with datemarks,
   with correct user statuses associated into message"
-  [message-groups messages message-statuses referenced-messages]
+  [message-groups messages message-statuses referenced-messages messages-gap]
-  (mapcat (fn [[datemark message-references]]
+  (transduce
-            (into (list {:value datemark
+   (comp
-                         :type  :datemark})
+    (mapcat add-datemark)
-                  (map (fn [{:keys [message-id timestamp-str]}]
+    (map (transform-message messages message-statuses referenced-messages)))
-                         (let [{:keys [content] :as message} (get messages message-id)
+   (completing
-                               {:keys [response-to response-to-v2]} content
+    (fn [{:keys [messages datemark-reference previous-message gap-added?]}
-                               quote (some-> (or response-to-v2 response-to)
+         message]
-                                             (quoted-message-data messages referenced-messages))]
+      (let [new-datemark? (datemark? message)
-                           (cond-> (-> message
+            add-gap?      (check-gap messages-gap previous-message message gap-added?)]
-                                       (update :content dissoc :response-to :response-to-v2)
+        {:messages           (cond-> messages
-                                       (assoc :datemark      datemark
+
-                                              :timestamp-str timestamp-str
+                               add-gap?
-                                              :user-statuses (get message-statuses message-id)))
+                               (add-gap messages-gap)
-                             quote ;; quoted message reference
+
-                             (assoc-in [:content :response-to] quote)))))
+                               :always
-                  message-references))
+                               (conj
-          message-groups))
+                                (cond-> message
+                                  (not new-datemark?)
+                                  (assoc
+                                   :datemark
+                                   (:value datemark-reference)))))
+         :previous-message   message
+         :datemark-reference (if new-datemark?
+                               message
+                               datemark-reference)
+         :gap-added?         (or gap-added? add-gap?)}))
+    (fn [{:keys [messages previous-message gap-added?]}]
+      (let [add-gap? (check-gap messages-gap previous-message nil gap-added?)]
+        (cond-> messages
+          add-gap?
+          (add-gap messages-gap)))))
+   {:messages         (list)
+    :previous-message nil}
+   message-groups))
 
 (defn- set-previous-message-info [stream]
   (let [{:keys [display-photo? message-type] :as previous-message} (peek stream)]

src/status_im/chat/models/loading.cljs

@@ -27,9 +27,11 @@
                  (update-in db [:chats chat-id :message-groups datemark]
                             (fn [message-references]
                               (->> grouped-messages
-                                   (map (fn [{:keys [message-id timestamp]}]
+                                   (map (fn [{:keys [message-id timestamp whisper-timestamp]}]
-                                          {:message-id    message-id
+                                          {:message-id        message-id
-                                           :timestamp-str (time/timestamp->time timestamp)}))
+                                           :timestamp-str     (time/timestamp->time timestamp)
+                                           :timestamp         timestamp
+                                           :whisper-timestamp whisper-timestamp}))
                                    (into (or message-references '()))
                                    (sort-references (get-in db [:chats chat-id :messages]))))))
                db

src/status_im/chat/models/message.cljs

@@ -273,7 +273,7 @@
         (get all-chats chat-id)
         {:keys [content content-type clock-value]}
         (->> (chat.db/sort-message-groups message-groups messages)
-             first
+             last
              second
              last
              :message-id

src/status_im/chat/subs.cljs

@@ -166,17 +166,37 @@
  (fn [{:keys [referenced-messages]}]
    (or referenced-messages {})))
 
+(re-frame/reg-sub
+ :chats/current-chat-topic
+ (fn [db]
+   (chat.db/topic-by-current-chat db)))
+
+(re-frame/reg-sub
+ :chats/messages-gap
+ :<- [:get-in [:mailserver/topics]]
+ :<- [:chats/current-chat-topic]
+ (fn [[mailserver-topics topic]]
+   (chat.db/messages-gap mailserver-topics topic)))
+
 (re-frame/reg-sub
  :chats/current-chat-messages-stream
  :<- [:chats/current-chat-messages]
  :<- [:chats/current-chat-message-groups]
  :<- [:chats/current-chat-message-statuses]
  :<- [:chats/current-chat-referenced-messages]
- (fn [[messages message-groups message-statuses referenced-messages]]
+ :<- [:chats/messages-gap]
+ (fn [[messages message-groups message-statuses referenced-messages messages-gap]]
    (-> (chat.db/sort-message-groups message-groups messages)
-       (chat.db/messages-with-datemarks-and-statuses messages message-statuses referenced-messages)
+       (chat.db/messages-with-datemarks-and-statuses messages message-statuses referenced-messages messages-gap)
        chat.db/messages-stream)))
 
+(re-frame/reg-sub
+ :chats/fetching-gap-in-progress?
+ (fn [db]
+   (let [chat-id (:current-chat-id db)
+         gaps (:mailserver/fetching-gaps-in-progress db)]
+     (contains? gaps chat-id))))
+
 (re-frame/reg-sub
  :chats/current-chat-intro-status
  :<- [:chats/current-chat]

src/status_im/data_store/realm/schemas/account/core.cljs

@@ -461,6 +461,21 @@
           contact-device-info/v1
           contact-recovery/v1])
 
+(def v41 [chat/v14
+          transport/v8
+          contact/v7
+          message/v10
+          mailserver/v11
+          mailserver-topic/v2
+          user-status/v2
+          membership-update/v1
+          installation/v3
+          local-storage/v1
+          browser/v8
+          dapp-permissions/v9
+          contact-device-info/v1
+          contact-recovery/v1])
+
 ;; put schemas ordered by version
 (def schemas [{:schema        v1
                :schemaVersion 1
@@ -581,4 +596,7 @@
                :migration     (constantly nil)}
               {:schema        v40
                :schemaVersion 40
-               :migration     migrations/v40}])
+               :migration     migrations/v40}
+              {:schema        v41
+               :schemaVersion 41
+               :migration     (constantly nil)}])

src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs

@@ -5,3 +5,14 @@
          :properties {:topic        :string
                       :chat-ids     :string
                       :last-request {:type :int :default 1}}})
+
+(def v2
+  (-> v1
+      (assoc-in
+       [:properties :gap-from]
+       {:type     :int
+        :optional true})
+      (assoc-in
+       [:properties :gap-to]
+       {:type     :int
+        :optional true})))

src/status_im/data_store/realm/schemas/account/message.cljs

@@ -62,3 +62,9 @@
   (-> v8
       (assoc-in [:properties :raw-payload-hash]
                 {:type :string})))
+
+(def v10
+  (-> v9
+      (assoc-in [:properties :whisper-timestamp]
+                {:type :int
+                 :optional true})))

src/status_im/data_store/realm/schemas/account/transport_inbox_topic.cljs

@@ -1,3 +1,4 @@
+; This entity is not used in the newer version of schema
 (ns status-im.data-store.realm.schemas.account.transport-inbox-topic)
 
 (def v1 {:name       :transport-inbox-topic

src/status_im/events.cljs

@@ -54,7 +54,8 @@
             [status-im.utils.config :as config]
             [status-im.ui.components.bottom-sheet.core :as bottom-sheet]
             [status-im.ui.components.react :as react]
-            [status-im.utils.build :as build]))
+            [status-im.utils.build :as build]
+            [status-im.chat.db :as chat.db]))
 
 ;; init module
 
@@ -445,7 +446,7 @@
 (handlers/register-handler-fx
  :mailserver/fetch-history
  (fn [cofx [_ chat-id from-timestamp]]
-   (mailserver/fetch-history cofx chat-id from-timestamp)))
+   (mailserver/fetch-history cofx chat-id {:from from-timestamp})))
 
 (handlers/register-handler-fx
  :mailserver.callback/generate-mailserver-symkey-success
@@ -721,8 +722,22 @@
 
 (handlers/register-handler-fx
  :chat.ui/fetch-history-pressed
- (fn [cofx [_ chat-id]]
+ (fn [{:keys [now] :as cofx} [_ chat-id]]
-   (mailserver/fetch-history cofx chat-id 1)))
+   (mailserver/fetch-history cofx chat-id
+                             {:from (- (quot now 1000) (* 24 3600))})))
+
+(handlers/register-handler-fx
+ :chat.ui/fill-the-gap
+ (fn [{:keys [db] :as cofx}]
+   (let [mailserver-topics (:mailserver/topics db)
+         chat-id           (:current-chat-id db)
+         topic             (chat.db/topic-by-current-chat db)
+         gap               (chat.db/messages-gap mailserver-topics topic)]
+     (mailserver/fill-the-gap
+      cofx
+      (assoc gap
+             :topic topic
+             :chat-id chat-id)))))
 
 (handlers/register-handler-fx
  :chat.ui/remove-chat-pressed

src/status_im/mailserver/core.cljs

@@ -2,13 +2,11 @@
  status-im.mailserver.core
   (:require [re-frame.core :as re-frame]
             [status-im.accounts.db :as accounts.db]
-            [status-im.data-store.core :as data-store]
             [status-im.fleet.core :as fleet]
             [status-im.native-module.core :as status]
             [status-im.utils.platform :as platform]
             [status-im.transport.utils :as transport.utils]
             [status-im.utils.fx :as fx]
-            [status-im.constants :as constants]
             [status-im.utils.utils :as utils]
             [taoensso.timbre :as log]
             [status-im.transport.db :as transport.db]
@@ -37,6 +35,7 @@
 
 (def one-day (* 24 3600))
 (def seven-days (* 7 one-day))
+(def max-request-range one-day)
 (def maximum-number-of-attempts 2)
 (def request-timeout 30)
 (def min-limit 100)
@@ -298,7 +297,7 @@
                  (assoc-in [:mailserver/current-request :request-id] request-id))}
         {:db (assoc-in db [:mailserver/current-request :request-id] request-id)}))))
 
-(defn request-messages! [web3 {:keys [sym-key-id address]} {:keys [topics cursor to from] :as request}]
+(defn request-messages! [web3 {:keys [sym-key-id address]} {:keys [topics cursor to from force-to?] :as request}]
   ;; Add some room to from, unless we break day boundaries so that messages that have
   ;; been received after the last request are also fetched
   (let [actual-from   (adjust-request-for-transit-time from)
@@ -307,16 +306,20 @@
     (log/info "mailserver: request-messages for: "
               " topics " topics
               " from " actual-from
+              " force-to? " force-to?
+              " to " to
               " cursor " cursor
               " limit " actual-limit)
     (.requestMessages (transport.utils/shh web3)
-                      (clj->js {:topics         topics
+                      (clj->js (cond-> {:topics         topics
-                                :mailServerPeer address
+                                        :mailServerPeer address
-                                :symKeyID       sym-key-id
+                                        :symKeyID       sym-key-id
-                                :timeout        request-timeout
+                                        :timeout        request-timeout
-                                :limit          actual-limit
+                                        :limit          actual-limit
-                                :cursor         cursor
+                                        :cursor         cursor
-                                :from           actual-from})
+                                        :from           actual-from}
+                                 force-to?
+                                 (assoc :to to)))
                       (fn [error request-id]
                         (if-not error
                           (do
@@ -341,25 +344,43 @@
                sym-key-id)
       mailserver)))
 
-(defn ->request
+(defn topic->request
-  [now-in-s [last-request topics]]
+  [default-request-to requests-from requests-to]
-  (when (< last-request now-in-s)
+  (fn [[topic {:keys [last-request]}]]
-    {:topics topics
+    (let [force-request-from (get requests-from topic)
-     ;; To is currently not sent to the mailserver, but we use to calculate
+          force-request-to (get requests-to topic)]
-     ;; when the last-request was sent.
+      (when (or force-request-from
-     :to     now-in-s
+                (> default-request-to last-request))
-     :from   (max last-request
+        (let [from (or force-request-from
-                  (- now-in-s one-day))}))
+                       (max last-request
+                            (- default-request-to max-request-range)))
+              to   (or force-request-to default-request-to)]
+          {:topic     topic
+           :from      from
+           :to        to
+           :force-to? (not (nil? force-request-to))})))))
+
+(defn aggregate-requests
+  [acc {:keys [topic from to force-to?]}]
+  (update acc [from to force-to?]
+          (fn [{:keys [topics]}]
+            {:topics    ((fnil conj #{}) topics topic)
+             :from      from
+             :to        to
+             ;; To is sent to the mailserver only when force-to? is true,
+             ;; also we use to calculate when the last-request was sent.
+             :force-to? force-to?})))
 
 (defn prepare-messages-requests
-  [{:keys [db now] :as cofx} request-to]
+  [{{:keys [:mailserver/requests-from
-  (let [web3     (:web3 db)]
+            :mailserver/requests-to
-    (->>
+            :mailserver/topics]} :db}
-     (:mailserver/topics db)
+   default-request-to]
-     (reduce (fn [acc [topic {:keys [last-request]}]]
+  (transduce
-               (update acc last-request conj topic))
+   (keep (topic->request default-request-to requests-from requests-to))
-             {})
+   (completing aggregate-requests vals)
-     (keep (partial ->request request-to)))))
+   {}
+   topics))
 
 (fx/defn process-next-messages-request
   [{:keys [db now] :as cofx}]
@@ -372,6 +393,7 @@
                            (quot now 1000))
             requests   (prepare-messages-requests cofx request-to)
             web3       (:web3 db)]
+        (log/debug "Mailserver: planned requests " requests)
         (if-let [request (first requests)]
           {:db                          (assoc db
                                                :mailserver/pending-requests (count requests)
@@ -382,7 +404,9 @@
                                          :request    request}}
           {:db (dissoc db
                        :mailserver/pending-requests
-                       :mailserver/request-to)})))))
+                       :mailserver/request-to
+                       :mailserver/requests-from
+                       :mailserver/requests-to)})))))
 
 (fx/defn add-mailserver-trusted
   "the current mailserver has been trusted
@@ -485,14 +509,100 @@
                         {:topic topic
                          :mailserver-topic mailserver-topic})]})))
 
-(defn get-updated-mailserver-topics [db topics last-request]
+(defn calculate-gap
-  (reduce (fn [acc topic]
+  [{:keys [gap-from
-            (if-let [mailserver-topic (some-> (get-in db [:mailserver/topics topic])
+           gap-to
-                                              (assoc :last-request last-request))]
+           last-request] :as config}
-              (assoc acc topic mailserver-topic)
+   {:keys [request-from
-              acc))
+           request-to]}]
-          {}
+  (merge config
-          topics))
+         (cond
+           (nil? gap-from)
+           {:gap-from     request-to
+            :gap-to       request-to
+            :last-request request-to}
+
+           ;;------GF     GT--------LRT     F---T
+           (> request-from last-request)
+           {:gap-from     last-request
+            :gap-to       request-from
+            :last-request request-to}
+
+           ;;------GF     GT--------LRT
+           ;;                  F----------T
+           (and (>= last-request request-from gap-to)
+                (> request-to last-request))
+           {:last-request request-to}
+
+           ;;------GF     GT--------LRT
+           ;;                F----T
+           (and (>= last-request request-from gap-to)
+                (>= last-request request-to gap-to))
+           config
+
+           ;;------GF     GT--------LRT
+           ;;          F-------T
+           (and (> gap-to request-from gap-from)
+                (>= last-request request-to gap-to))
+           {:gap-to request-from}
+
+           ;;------GF     GT--------LRT
+           ;;         F-T
+           (and (> gap-to request-from gap-from)
+                (> gap-to request-to gap-from))
+           config
+
+           ;;------GF     GT--------LRT
+           ;;   F------T
+           (and (>= gap-from request-from)
+                (> gap-to request-to gap-from))
+           {:gap-from request-to}
+
+           ;;---------GF=GT=LRT
+           ;; F---T
+           (and (>= gap-from request-from)
+                (>= gap-from request-to)
+                (= gap-from last-request))
+           {:gap-from request-to}
+
+           ;;------GF     GT--------LRT
+           ;; F---T
+           (and (>= gap-from request-from)
+                (>= gap-from request-to))
+           config
+
+           ;;------GF     GT--------LRT
+           ;;   F-------------T
+           (and (>= gap-from request-from)
+                (>= last-request request-to gap-to))
+           {:gap-from     last-request
+            :gap-to       last-request
+            :last-request last-request}
+
+           ;;------GF     GT--------LRT
+           ;;   F------------------------T
+           (and (>= gap-from request-from)
+                (>= request-to last-request))
+           {:gap-from     request-to
+            :gap-to       request-to
+            :last-request request-to}
+
+           ;;------GF     GT--------LRT
+           ;;          F-----------------T
+           (and (> gap-to request-from gap-from)
+                (>= request-to last-request))
+           {:gap-to       request-from
+            :last-request request-to})))
+
+(defn get-updated-mailserver-topics [db requested-topics from to]
+  (into
+   {}
+   (keep (fn [topic]
+           (when-let [config (get-in db [:mailserver/topics topic])]
+             [topic (calculate-gap config
+                                   {:request-from from
+                                    :request-to   to})])))
+   requested-topics))
 
 (fx/defn update-mailserver-topics
   "TODO: add support for cursors
@@ -500,7 +610,7 @@
   [{:keys [db now] :as cofx} {:keys [request-id cursor]}]
   (when-let [request (get db :mailserver/current-request)]
     (let [{:keys [from to topics]} request
-          mailserver-topics (get-updated-mailserver-topics db topics to)]
+          mailserver-topics (get-updated-mailserver-topics db topics from to)]
       (log/info "mailserver: message request " request-id
                 "completed for mailserver topics" topics "from" from "to" to)
       (if (empty? mailserver-topics)
@@ -516,16 +626,31 @@
                :mailserver/request-messages {:web3 (:web3 db)
                                              :mailserver    mailserver
                                              :request request-with-cursor}}))
-          (fx/merge cofx
+          (let [{:keys [chat-id] :as current-request} (db :mailserver/current-request)
-                    {:db (-> db
+                gaps                    (db :mailserver/fetching-gaps-in-progress)
-                             (dissoc :mailserver/current-request)
+                fetching-gap-completed? (= (get gaps chat-id)
-                             (update :mailserver/topics merge mailserver-topics))
+                                           (select-keys
-                     :data-store/tx (mapv (fn [[topic mailserver-topic]]
+                                            current-request
-                                            (data-store.mailservers/save-mailserver-topic-tx
+                                            [:from :to :force-to? :topics :chat-id]))]
-                                             {:topic topic
+            (fx/merge cofx
-                                              :mailserver-topic mailserver-topic}))
+                      {:db            (-> db
-                                          mailserver-topics)}
+                                          (dissoc :mailserver/current-request)
-                    (process-next-messages-request)))))))
+                                          (update :mailserver/requests-from
+                                                  #(apply dissoc % topics))
+                                          (update :mailserver/requests-to
+                                                  #(apply dissoc % topics))
+                                          (update :mailserver/topics merge mailserver-topics)
+                                          (update :mailserver/fetching-gaps-in-progress
+                                                  (fn [gaps]
+                                                    (if fetching-gap-completed?
+                                                      (dissoc gaps chat-id)
+                                                      gaps))))
+                       :data-store/tx (mapv (fn [[topic mailserver-topic]]
+                                              (data-store.mailservers/save-mailserver-topic-tx
+                                               {:topic            topic
+                                                :mailserver-topic mailserver-topic}))
+                                            mailserver-topics)}
+                      (process-next-messages-request))))))))
 
 (fx/defn retry-next-messages-request
   [{:keys [db] :as cofx}]
@@ -543,8 +668,8 @@
                    :mailserver/pending-requests))})
 
 (fx/defn handle-request-completed
-  [{{:keys [chats] :as db} :db :as cofx}
+  [{{:keys [chats]} :db :as cofx}
-   {:keys [requestID lastEnvelopeHash cursor errorMessage] :as event}]
+   {:keys [requestID lastEnvelopeHash cursor errorMessage]}]
   (when (accounts.db/logged-in? cofx)
     (if (empty? errorMessage)
       (let [never-synced-chats-in-request
@@ -596,12 +721,13 @@
       add the chat-id to the topic and reset last-request
       there was no filter for the chat and messages for that
       so the whole history for that topic needs to be re-fetched"
-  [{:keys [db] :as cofx} {:keys [topic chat-id]}]
+  [{:keys [db now] :as cofx} {:keys [topic chat-id]}]
   (let [{:keys [chat-ids last-request] :as current-mailserver-topic}
         (get-in db [:mailserver/topics topic] {:chat-ids #{}})]
     (when-let [mailserver-topic (when-not (chat-ids chat-id)
                                   (-> current-mailserver-topic
-                                      (assoc :last-request 1)
+                                      (assoc :last-request (- (quot now 1000)
+                                                              (* 24 60 60)))
                                       (update :chat-ids conj chat-id)))]
       (fx/merge cofx
                 {:db (assoc-in db [:mailserver/topics topic] mailserver-topic)
@@ -610,21 +736,42 @@
                                    :mailserver-topic mailserver-topic})]}))))
 
 (fx/defn fetch-history
-  [{:keys [db] :as cofx} chat-id from-timestamp]
+  [{:keys [db] :as cofx} chat-id {:keys [from to]}]
-  (log/debug "fetch-history" "chat-id:" chat-id "from-timestamp:" from-timestamp)
+
+  (log/debug "fetch-history" "chat-id:" chat-id "from-timestamp:" from)
   (let [public-key (accounts.db/current-public-key cofx)
         topic  (or (get-in db [:transport/chats chat-id :topic])
-                   (transport.topic/public-key->discovery-topic-hash public-key))
+                   (transport.topic/public-key->discovery-topic-hash public-key))]
-        {:keys [chat-ids last-request] :as current-mailserver-topic}
+    (fx/merge cofx
-        (get-in db [:mailserver/topics topic] {:chat-ids #{}})]
+              {:db (cond-> (assoc-in db [:mailserver/requests-from topic] from)
-    (let [mailserver-topic (-> current-mailserver-topic
+
-                               (assoc :last-request (min from-timestamp last-request)))]
+                     to
-      (fx/merge cofx
+                     (assoc-in [:mailserver/requests-to topic] to))}
-                {:db (assoc-in db [:mailserver/topics topic] mailserver-topic)
+              (process-next-messages-request))))
-                 :data-store/tx [(data-store.mailservers/save-mailserver-topic-tx
+
-                                  {:topic topic
+(fx/defn fill-the-gap
-                                   :mailserver-topic mailserver-topic})]}
+  [{:keys [db] :as cofx} {:keys [exists? from to topic chat-id]}]
-                (process-next-messages-request)))))
+  (let [mailserver (get-mailserver-when-ready cofx)
+        request    {:from      from
+                    :to        to
+                    :force-to? true
+                    :topics    [topic]
+                    :chat-id   chat-id}]
+    (when exists?
+      {:db
+       (-> db
+           (assoc
+            :mailserver/pending-requests 1
+            :mailserver/current-request request
+            :mailserver/request-to to)
+
+           (update :mailserver/fetching-gaps-in-progress
+                   assoc chat-id request))
+
+       :mailserver/request-messages
+       {:web3       (:web3 db)
+        :mailserver mailserver
+        :request    request}})))
 
 (fx/defn resend-request
   [{:keys [db] :as cofx} {:keys [request-id]}]

src/status_im/mailserver/subs.cljs

@@ -43,6 +43,13 @@
         (pos-int? pending-requests)
         (not (or connecting? connection-error? request-error?)))))
 
+(re-frame/reg-sub
+ :mailserver/connected?
+ (fn [db]
+   (let [connected? (= :connected (:mailserver/state db))
+         online?    (= :online (:network-status db))]
+     (and connected? online?))))
+
 (re-frame/reg-sub
  :mailserver/current-id
  (fn [db]

src/status_im/transport/message/protocol.cljs

@@ -113,7 +113,7 @@
         (fx/merge cofx
                   (send-direct-message current-public-key nil this)
                   (send-with-pubkey params)))))
-  (receive [this chat-id signature _ cofx]
+  (receive [this chat-id signature timestamp cofx]
     {:chat-received-message/add-fx
      [(assoc (into {} this)
              :old-message-id (transport.utils/old-message-id this)
@@ -121,6 +121,7 @@
                           signature
                           (.-payload (:js-obj cofx)))
              :chat-id chat-id
+             :whisper-timestamp timestamp
              :raw-payload-hash (transport.utils/sha3
                                 (.-payload (:js-obj cofx)))
              :from signature

src/status_im/ui/screens/chat/views.cljs

@@ -70,6 +70,37 @@
   [{{:keys [value]} :row}]
   [message-datemark/chat-datemark-mobile value])
 
+(defview gap []
+  (letsubs [in-progress? [:chats/fetching-gap-in-progress?]
+            connected?   [:mailserver/connected?]]
+    [react/view {:align-self          :stretch
+                 :margin-top          24
+                 :margin-bottom       24
+                 :height              48
+                 :align-items         :center
+                 :justify-content     :center
+                 :border-color        colors/gray-light
+                 :border-top-width    1
+                 :border-bottom-width 1
+                 :background-color    :white}
+     [react/touchable-highlight
+      {:on-press (when (and connected? (not in-progress?))
+                   #(re-frame/dispatch [:chat.ui/fill-the-gap]))}
+      [react/view {:flex            1
+                   :align-items     :center
+                   :justify-content :center}
+       (if in-progress?
+         [react/activity-indicator]
+         [react/text
+          {:style {:color (if connected?
+                            colors/blue
+                            colors/gray)}}
+          (i18n/label :t/fetch-messages)])]]]))
+
+(defmethod message-row :gap
+  [_]
+  [gap])
+
 (defmethod message-row :default
   [{:keys [group-chat current-public-key modal? row]}]
   [message/chat-message (assoc row

test/cljs/status_im/test/chat/db.cljs

@@ -1,35 +1,35 @@
 (ns status-im.test.chat.db
   (:require [cljs.test :refer-macros [deftest is testing]]
-            [status-im.chat.db :as s]))
+            [status-im.chat.db :as db]))
 
 (deftest group-chat-name
   (testing "it prepends # if it's a public chat"
-    (is (= "#withhash" (s/group-chat-name {:group-chat true
+    (is (= "#withhash" (db/group-chat-name {:group-chat true
-                                           :chat-id "1"
+                                            :chat-id    "1"
-                                           :public? true
+                                            :public?    true
-                                           :name "withhash"}))))
+                                            :name       "withhash"}))))
   (testing "it leaves the name unchanged if it's a group chat"
-    (is (= "unchanged" (s/group-chat-name {:group-chat true
+    (is (= "unchanged" (db/group-chat-name {:group-chat true
-                                           :chat-id "1"
+                                            :chat-id    "1"
-                                           :name "unchanged"})))))
+                                            :name       "unchanged"})))))
 
 (deftest message-stream-tests
   (testing "messages with no interspersed datemarks"
-    (let [m1 {:from "1"
+    (let [m1       {:from     "1"
-              :datemark "a"
+                    :datemark "a"
-              :outgoing false}
+                    :outgoing false}
-          m2   {:from "2"
+          m2       {:from     "2"
-                :datemark "a"
+                    :datemark "a"
-                :outgoing true}
+                    :outgoing true}
-          m3    {:from "2"
+          m3       {:from     "2"
-                 :datemark "a"
+                    :datemark "a"
-                 :outgoing true}
+                    :outgoing true}
-          dm1  {:type :datemark
+          dm1      {:type  :datemark
-                :value "a"}
+                    :value "a"}
           messages [m1 m2 m3 dm1]
           [actual-m1
            actual-m2
-           actual-m3] (s/messages-stream messages)]
+           actual-m3] (db/messages-stream messages)]
       (testing "it marks only the first message as :last?"
         (is (:last? actual-m1))
         (is (not (:last? actual-m2)))
@@ -59,29 +59,29 @@
         (is (:last-in-group? actual-m2))
         (is (not (:last-in-group? actual-m3))))))
   (testing "messages with interspersed datemarks"
-    (let [m1 {:from "2"          ; first & last in group
+    (let [m1       {:from      "2"                          ; first & last in group
-              :timestamp 63000
+                    :timestamp 63000
-              :outgoing true}
+                    :outgoing  true}
-          dm1 {:type :datemark
+          dm1      {:type  :datemark
-               :value "a"}
+                    :value "a"}
-          m2  {:from "2"         ; first & last in group as more than 1 minute after previous message
+          m2       {:from      "2"                          ; first & last in group as more than 1 minute after previous message
-               :timestamp 62000
+                    :timestamp 62000
-               :outgoing false}
+                    :outgoing  false}
-          m3  {:from "2"         ; last in group
+          m3       {:from      "2"                          ; last in group
-               :timestamp 1
+                    :timestamp 1
-               :outgoing false}
+                    :outgoing  false}
-          m4  {:from "2"         ; first in group
+          m4       {:from      "2"                          ; first in group
-               :timestamp 0
+                    :timestamp 0
-               :outgoing false}
+                    :outgoing  false}
-          dm2 {:type :datemark
+          dm2      {:type  :datemark
-               :value "b"}
+                    :value "b"}
           messages [m1 dm1 m2 m3 m4 dm2]
           [actual-m1
            _
            actual-m2
            actual-m3
            actual-m4
-           _] (s/messages-stream messages)]
+           _] (db/messages-stream messages)]
       (testing "it marks the first outgoing message as :last-outgoing?"
         (is (:last-outgoing? actual-m1))
         (is (not (:last-outgoing? actual-m2)))
@@ -103,12 +103,12 @@
         (is (:last-in-group? actual-m3))
         (is (not (:last-in-group? actual-m4))))))
   (testing "system-messages"
-    (let [m1 {:from "system"
+    (let [m1       {:from         "system"
-              :message-type :system-message
+                    :message-type :system-message
-              :datemark "a"
+                    :datemark     "a"
-              :outgoing false}
+                    :outgoing     false}
           messages [m1]
-          [actual-m1] (s/messages-stream messages)]
+          [actual-m1] (db/messages-stream messages)]
       (testing "it does display the photo"
         (is (:display-photo? actual-m1))
         (testing "it does not display the username"
@@ -122,4 +122,218 @@
                        "3" {:is-active false :chat-id "3"}}]
     (testing "it returns only chats with is-active"
       (is (= #{"1" "2"}
-             (set (keys (s/active-chats {} chats {}))))))))
+             (set (keys (db/active-chats {} chats {}))))))))
+
+(deftest messages-with-datemarks-and-statuses
+  (testing "empty state"
+    (is (empty?
+         (db/messages-with-datemarks-and-statuses
+          nil
+          nil
+          nil
+          nil
+          nil))))
+  (testing "simple case"
+    (is (=
+         '({:whisper-timestamp 40
+            :timestamp         40
+            :content           nil
+            :timestamp-str     "14:00"
+            :user-statuses     nil
+            :datemark          "today"}
+           {:whisper-timestamp 30
+            :timestamp         30
+            :content           nil
+            :timestamp-str     "13:00"
+            :user-statuses     nil
+            :datemark          "today"}
+           {:value             "today"
+            :type              :datemark
+            :whisper-timestamp 30
+            :timestamp         30}
+           {:whisper-timestamp 20
+            :timestamp         20
+            :content           nil
+            :timestamp-str     "12:00"
+            :user-statuses     nil
+            :datemark          "yesterday"}
+           {:whisper-timestamp 10
+            :timestamp         10
+            :content           nil
+            :timestamp-str     "11:00"
+            :user-statuses     nil
+            :datemark          "yesterday"}
+           {:value             "yesterday"
+            :type              :datemark
+            :whisper-timestamp 10
+            :timestamp         10})
+         (db/messages-with-datemarks-and-statuses
+          {"yesterday"
+           (list
+            {:message-id        :m1
+             :timestamp-str     "11:00"
+             :whisper-timestamp 10
+             :timestamp         10}
+            {:message-id        :m2
+             :timestamp-str     "12:00"
+             :whisper-timestamp 20
+             :timestamp         20})
+           "today"
+           (list
+            {:message-id        :m3
+             :timestamp-str     "13:00"
+             :whisper-timestamp 30
+             :timestamp         30}
+            {:message-id        :m4
+             :timestamp-str     "14:00"
+             :whisper-timestamp 40
+             :timestamp         40})}
+          {:m1 {:whisper-timestamp 10
+                :timestamp         10}
+           :m2 {:whisper-timestamp 20
+                :timestamp         20}
+           :m3 {:whisper-timestamp 30
+                :timestamp         30}
+           :m4 {:whisper-timestamp 40
+                :timestamp         40}}
+          nil
+          nil
+          nil))))
+  (testing "simple case with gap"
+    (is (=
+         '({:whisper-timestamp 40
+            :timestamp         40
+            :content           nil
+            :timestamp-str     "14:00"
+            :user-statuses     nil
+            :datemark          "today"}
+           {:whisper-timestamp 30
+            :timestamp         30
+            :content           nil
+            :timestamp-str     "13:00"
+            :user-statuses     nil
+            :datemark          "today"}
+           {:value             "today"
+            :type              :datemark
+            :whisper-timestamp 30
+            :timestamp         30}
+           {:type              :gap
+            :value             "25"}
+           {:whisper-timestamp 20
+            :timestamp         20
+            :content           nil
+            :timestamp-str     "12:00"
+            :user-statuses     nil
+            :datemark          "yesterday"}
+           {:whisper-timestamp 10
+            :timestamp         10
+            :content           nil
+            :timestamp-str     "11:00"
+            :user-statuses     nil
+            :datemark          "yesterday"}
+           {:value             "yesterday"
+            :type              :datemark
+            :whisper-timestamp 10
+            :timestamp         10})
+         (db/messages-with-datemarks-and-statuses
+          {"yesterday"
+           (list
+            {:message-id        :m1
+             :timestamp-str     "11:00"
+             :whisper-timestamp 10
+             :timestamp         10}
+            {:message-id        :m2
+             :timestamp-str     "12:00"
+             :whisper-timestamp 20
+             :timestamp         20})
+           "today"
+           (list
+            {:message-id        :m3
+             :timestamp-str     "13:00"
+             :whisper-timestamp 30
+             :timestamp         30}
+            {:message-id        :m4
+             :timestamp-str     "14:00"
+             :whisper-timestamp 40
+             :timestamp         40})}
+          {:m1 {:whisper-timestamp 10
+                :timestamp         10}
+           :m2 {:whisper-timestamp 20
+                :timestamp         20}
+           :m3 {:whisper-timestamp 30
+                :timestamp         30}
+           :m4 {:whisper-timestamp 40
+                :timestamp         40}}
+          nil
+          nil
+          {:from    25
+           :exists? true}))))
+  (testing "simple case with gap after all messages"
+    (is (=
+         '({:type  :gap
+            :value "100"}
+           {:whisper-timestamp 40
+            :timestamp         40
+            :content           nil
+            :timestamp-str     "14:00"
+            :user-statuses     nil
+            :datemark          "today"}
+           {:whisper-timestamp 30
+            :timestamp         30
+            :content           nil
+            :timestamp-str     "13:00"
+            :user-statuses     nil
+            :datemark          "today"}
+           {:value             "today"
+            :type              :datemark
+            :whisper-timestamp 30
+            :timestamp         30}
+           {:whisper-timestamp 20
+            :timestamp         20
+            :content           nil
+            :timestamp-str     "12:00"
+            :user-statuses     nil
+            :datemark          "yesterday"}
+           {:whisper-timestamp 10
+            :timestamp         10
+            :content           nil
+            :timestamp-str     "11:00"
+            :user-statuses     nil
+            :datemark          "yesterday"}
+           {:value             "yesterday"
+            :type              :datemark
+            :whisper-timestamp 10
+            :timestamp         10})
+         (db/messages-with-datemarks-and-statuses
+          {"yesterday"
+           (list
+            {:message-id        :m1
+             :timestamp-str     "11:00"
+             :whisper-timestamp 10
+             :timestamp         10}
+            {:message-id        :m2
+             :timestamp-str     "12:00"
+             :whisper-timestamp 20
+             :timestamp         20})
+           "today"
+           (list
+            {:message-id        :m3
+             :timestamp-str     "13:00"
+             :whisper-timestamp 30
+             :timestamp         30}
+            {:message-id        :m4
+             :timestamp-str     "14:00"
+             :whisper-timestamp 40
+             :timestamp         40})}
+          {:m1 {:whisper-timestamp 10
+                :timestamp         10}
+           :m2 {:whisper-timestamp 20
+                :timestamp         20}
+           :m3 {:whisper-timestamp 30
+                :timestamp         30}
+           :m4 {:whisper-timestamp 40
+                :timestamp         40}}
+          nil
+          nil
+          {:from    100
+           :exists? true})))))

test/cljs/status_im/test/mailserver/core.cljs

@@ -627,3 +627,187 @@
         (is (not (-> (mailserver/connect-to-mailserver {:db mailserver-with-sym-key-db})
                      :shh/generate-sym-key-from-password
                      first)))))))
+
+(deftest calculate-gap
+  (testing "new topic"
+    (is (= {:gap-from     10
+            :gap-to       10
+            :last-request 10}
+
+           (mailserver/calculate-gap
+            {:gap-from     nil
+             :gap-to       nil
+             :last-request nil}
+            {:request-from 5
+             :request-to   10}))))
+  (testing "calculate-gap#1"
+    (is (= {:gap-from     3
+            :gap-to       4
+            :last-request 5}
+
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 3}
+            {:request-from 4
+             :request-to   5}))))
+  (testing "calculate-gap#2"
+    (is (= {:gap-from     1
+            :gap-to       2
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 4}
+            {:request-from 3
+             :request-to   5}))))
+  (testing "calculate-gap#2-1"
+    (is (= {:gap-from     1
+            :gap-to       2
+            :last-request 4}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 3}
+            {:request-from 3
+             :request-to   4}))))
+  (testing "calculate-gap#3"
+    (is (= {:gap-from     1
+            :gap-to       2
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 5}
+            {:request-from 3
+             :request-to   4}))))
+  (testing "calculate-gap#3-1"
+    (is (= {:gap-from     1
+            :gap-to       2
+            :last-request 3}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 3}
+            {:request-from 2
+             :request-to   3}))))
+  (testing "calculate-gap#4"
+    (is (= {:gap-from     1
+            :gap-to       2
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 5}
+            {:request-from 3
+             :request-to   4}))))
+  (testing "calculate-gap#5"
+    (is (= {:gap-from     1
+            :gap-to       4
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       4
+             :last-request 5}
+            {:request-from 2
+             :request-to   3}))))
+  (testing "calculate-gap#6"
+    (is (= {:gap-from     2
+            :gap-to       3
+            :last-request 4}
+           (mailserver/calculate-gap
+            {:gap-from     2
+             :gap-to       3
+             :last-request 4}
+            {:request-from 1
+             :request-to   2}))))
+  (testing "calculate-gap#6-1"
+    (is (= {:gap-from     1
+            :gap-to       4
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       4
+             :last-request 5}
+            {:request-from 2
+             :request-to   3}))))
+  (testing "calculate-gap#0"
+    (is (= {:gap-from     2
+            :gap-to       3
+            :last-request 3}
+           (mailserver/calculate-gap
+            {:gap-from     3
+             :gap-to       3
+             :last-request 3}
+            {:request-from 1
+             :request-to   2}))))
+  (testing "calculate-gap#7"
+    (is (= {:gap-from     3
+            :gap-to       4
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     3
+             :gap-to       4
+             :last-request 5}
+            {:request-from 1
+             :request-to   2}))))
+  (testing "calculate-gap#8"
+    (is (= {:gap-from     5
+            :gap-to       5
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     2
+             :gap-to       3
+             :last-request 5}
+            {:request-from 1
+             :request-to   4}))))
+  (testing "calculate-gap#8-1"
+    (is (= {:gap-from     3
+            :gap-to       3
+            :last-request 3}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 3}
+            {:request-from 1
+             :request-to   2}))))
+  (testing "calculate-gap#9"
+    (is (= {:gap-from     5
+            :gap-to       5
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     2
+             :gap-to       3
+             :last-request 4}
+            {:request-from 1
+             :request-to   5}))))
+  (testing "calculate-gap#9-1"
+    (is (= {:gap-from     3
+            :gap-to       3
+            :last-request 3}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 3}
+            {:request-from 1
+             :request-to   3}))))
+  (testing "calculate-gap#10"
+    (is (= {:gap-from     1
+            :gap-to       2
+            :last-request 5}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       3
+             :last-request 4}
+            {:request-from 2
+             :request-to   5}))))
+  (testing "calculate-gap#10-1"
+    (is (= {:gap-from     1
+            :gap-to       2
+            :last-request 3}
+           (mailserver/calculate-gap
+            {:gap-from     1
+             :gap-to       2
+             :last-request 3}
+            {:request-from 2
+             :request-to   3})))))

translations/en.json

@@ -997,5 +997,6 @@
     "chaos-unicorn-day-details": "\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83D\uDE80!",
     "invalid-format": "Invalid format\nMust be {{format}}",
     "mailserver-format": "enode://{enode-id}:{password}@{ip-address}:{port}",
-    "bootnode-format": "enode://{enode-id}@{ip-address}:{port}"
+    "bootnode-format": "enode://{enode-id}@{ip-address}:{port}",
+    "fetch-messages": "↓ Fetch messages"
 }