From a4d8f57b09ed1a48a17f5971e9556520fbb19e9a Mon Sep 17 00:00:00 2001 From: Roman Volosovskyi Date: Thu, 4 Apr 2019 16:55:35 +0300 Subject: [PATCH] Fetch missed range of messages --- src/status_im/chat/db.cljs | 124 ++++++-- src/status_im/chat/models/loading.cljs | 8 +- src/status_im/chat/models/message.cljs | 2 +- src/status_im/chat/subs.cljs | 24 +- .../realm/schemas/account/core.cljs | 20 +- .../schemas/account/mailserver_topic.cljs | 11 + .../realm/schemas/account/message.cljs | 6 + .../account/transport_inbox_topic.cljs | 1 + src/status_im/events.cljs | 23 +- src/status_im/mailserver/core.cljs | 275 ++++++++++++---- src/status_im/mailserver/subs.cljs | 7 + src/status_im/transport/message/protocol.cljs | 3 +- src/status_im/ui/screens/chat/views.cljs | 31 ++ test/cljs/status_im/test/chat/db.cljs | 300 +++++++++++++++--- test/cljs/status_im/test/mailserver/core.cljs | 184 +++++++++++ translations/en.json | 3 +- 16 files changed, 882 insertions(+), 140 deletions(-) diff --git a/src/status_im/chat/db.cljs b/src/status_im/chat/db.cljs index 2d6e20deeb..ab5571a924 100644 --- a/src/status_im/chat/db.cljs +++ b/src/status_im/chat/db.cljs @@ -5,7 +5,8 @@ [status-im.chat.commands.core :as commands] [status-im.chat.commands.input :as commands.input] [status-im.group-chats.db :as group-chats.db] - [status-im.utils.gfycat.core :as gfycat])) + [status-im.utils.gfycat.core :as gfycat] + [status-im.transport.partitioned-topic :as topic])) (defn group-chat-name [{:keys [public? name]}] @@ -49,11 +50,28 @@ {} chats)) +(defn topic-by-current-chat + [{:keys [current-chat-id chats] :as db}] + (let [{:keys [public?]} (get chats current-chat-id) + public-key (get-in db [:account/account :public-key])] + (if public? + (get-in db [:transport/chats current-chat-id :topic]) + (topic/public-key->discovery-topic-hash public-key)))) + +(defn messages-gap + [mailserver-topics topic] + (let [{:keys [gap-from gap-to]} + (get mailserver-topics topic)] + {:from gap-from + :to gap-to + :exists? (and gap-from gap-to + (> gap-to gap-from))})) + (defn sort-message-groups "Sorts message groups according to timestamp of first message in group" [message-groups messages] (sort-by - (comp unchecked-negate :timestamp (partial get messages) :message-id first second) + (comp :timestamp (partial get messages) :message-id first second) message-groups)) (defn quoted-message-data @@ -64,27 +82,93 @@ {:from from :text (:text content)})) +(defn add-datemark + [[datemark + message-references]] + (let [{:keys [whisper-timestamp timestamp]} (first message-references)] + (conj message-references + {:value datemark + :type :datemark + :whisper-timestamp whisper-timestamp + :timestamp timestamp}))) + +(defn datemark? [{:keys [type]}] + (= type :datemark)) + +(defn transform-message + [messages message-statuses referenced-messages] + (fn [{:keys [message-id timestamp-str] :as reference}] + (if (datemark? reference) + reference + (let [{:keys [content] :as message} (get messages message-id) + {:keys [response-to response-to-v2]} content + quote (some-> (or response-to-v2 response-to) + (quoted-message-data messages referenced-messages))] + (cond-> (-> message + (update :content dissoc :response-to :response-to-v2) + (assoc :timestamp-str timestamp-str + :user-statuses (get message-statuses message-id))) + ;; quoted message reference + quote + (assoc-in [:content :response-to] quote)))))) + +(defn check-gap + [{:keys [exists? from]} previous-message next-message gap-added?] + (let [previous-timestamp (:whisper-timestamp previous-message) + next-whisper-timestamp (:whisper-timestamp next-message) + next-timestamp (quot (:timestamp next-message) 1000) + ignore-next-message? (> (js/Math.abs + (- next-whisper-timestamp next-timestamp)) + 120)] + (and (not gap-added?) + (or (and exists? previous-timestamp next-whisper-timestamp + (not ignore-next-message?) + (< previous-timestamp from next-whisper-timestamp)) + (and exists? (nil? next-message)))))) + +(defn add-gap [messages gap] + (conj messages + {:type :gap + :value (str (:from gap))})) + (defn messages-with-datemarks-and-statuses "Converts message groups into sequence of messages interspersed with datemarks, with correct user statuses associated into message" - [message-groups messages message-statuses referenced-messages] - (mapcat (fn [[datemark message-references]] - (into (list {:value datemark - :type :datemark}) - (map (fn [{:keys [message-id timestamp-str]}] - (let [{:keys [content] :as message} (get messages message-id) - {:keys [response-to response-to-v2]} content - quote (some-> (or response-to-v2 response-to) - (quoted-message-data messages referenced-messages))] - (cond-> (-> message - (update :content dissoc :response-to :response-to-v2) - (assoc :datemark datemark - :timestamp-str timestamp-str - :user-statuses (get message-statuses message-id))) - quote ;; quoted message reference - (assoc-in [:content :response-to] quote))))) - message-references)) - message-groups)) + [message-groups messages message-statuses referenced-messages messages-gap] + (transduce + (comp + (mapcat add-datemark) + (map (transform-message messages message-statuses referenced-messages))) + (completing + (fn [{:keys [messages datemark-reference previous-message gap-added?]} + message] + (let [new-datemark? (datemark? message) + add-gap? (check-gap messages-gap previous-message message gap-added?)] + {:messages (cond-> messages + + add-gap? + (add-gap messages-gap) + + :always + (conj + (cond-> message + (not new-datemark?) + (assoc + :datemark + (:value datemark-reference))))) + :previous-message message + :datemark-reference (if new-datemark? + message + datemark-reference) + :gap-added? (or gap-added? add-gap?)})) + (fn [{:keys [messages previous-message gap-added?]}] + (let [add-gap? (check-gap messages-gap previous-message nil gap-added?)] + (cond-> messages + add-gap? + (add-gap messages-gap))))) + {:messages (list) + :previous-message nil} + message-groups)) (defn- set-previous-message-info [stream] (let [{:keys [display-photo? message-type] :as previous-message} (peek stream)] diff --git a/src/status_im/chat/models/loading.cljs b/src/status_im/chat/models/loading.cljs index 41d74bfc33..d05239f355 100644 --- a/src/status_im/chat/models/loading.cljs +++ b/src/status_im/chat/models/loading.cljs @@ -27,9 +27,11 @@ (update-in db [:chats chat-id :message-groups datemark] (fn [message-references] (->> grouped-messages - (map (fn [{:keys [message-id timestamp]}] - {:message-id message-id - :timestamp-str (time/timestamp->time timestamp)})) + (map (fn [{:keys [message-id timestamp whisper-timestamp]}] + {:message-id message-id + :timestamp-str (time/timestamp->time timestamp) + :timestamp timestamp + :whisper-timestamp whisper-timestamp})) (into (or message-references '())) (sort-references (get-in db [:chats chat-id :messages])))))) db diff --git a/src/status_im/chat/models/message.cljs b/src/status_im/chat/models/message.cljs index b8e25ce785..a8f5e10297 100644 --- a/src/status_im/chat/models/message.cljs +++ b/src/status_im/chat/models/message.cljs @@ -273,7 +273,7 @@ (get all-chats chat-id) {:keys [content content-type clock-value]} (->> (chat.db/sort-message-groups message-groups messages) - first + last second last :message-id diff --git a/src/status_im/chat/subs.cljs b/src/status_im/chat/subs.cljs index b3facb584a..02a442bb9c 100644 --- a/src/status_im/chat/subs.cljs +++ b/src/status_im/chat/subs.cljs @@ -166,17 +166,37 @@ (fn [{:keys [referenced-messages]}] (or referenced-messages {}))) +(re-frame/reg-sub + :chats/current-chat-topic + (fn [db] + (chat.db/topic-by-current-chat db))) + +(re-frame/reg-sub + :chats/messages-gap + :<- [:get-in [:mailserver/topics]] + :<- [:chats/current-chat-topic] + (fn [[mailserver-topics topic]] + (chat.db/messages-gap mailserver-topics topic))) + (re-frame/reg-sub :chats/current-chat-messages-stream :<- [:chats/current-chat-messages] :<- [:chats/current-chat-message-groups] :<- [:chats/current-chat-message-statuses] :<- [:chats/current-chat-referenced-messages] - (fn [[messages message-groups message-statuses referenced-messages]] + :<- [:chats/messages-gap] + (fn [[messages message-groups message-statuses referenced-messages messages-gap]] (-> (chat.db/sort-message-groups message-groups messages) - (chat.db/messages-with-datemarks-and-statuses messages message-statuses referenced-messages) + (chat.db/messages-with-datemarks-and-statuses messages message-statuses referenced-messages messages-gap) chat.db/messages-stream))) +(re-frame/reg-sub + :chats/fetching-gap-in-progress? + (fn [db] + (let [chat-id (:current-chat-id db) + gaps (:mailserver/fetching-gaps-in-progress db)] + (contains? gaps chat-id)))) + (re-frame/reg-sub :chats/current-chat-intro-status :<- [:chats/current-chat] diff --git a/src/status_im/data_store/realm/schemas/account/core.cljs b/src/status_im/data_store/realm/schemas/account/core.cljs index e69cf4a2d1..ce9d053009 100644 --- a/src/status_im/data_store/realm/schemas/account/core.cljs +++ b/src/status_im/data_store/realm/schemas/account/core.cljs @@ -461,6 +461,21 @@ contact-device-info/v1 contact-recovery/v1]) +(def v41 [chat/v14 + transport/v8 + contact/v7 + message/v10 + mailserver/v11 + mailserver-topic/v2 + user-status/v2 + membership-update/v1 + installation/v3 + local-storage/v1 + browser/v8 + dapp-permissions/v9 + contact-device-info/v1 + contact-recovery/v1]) + ;; put schemas ordered by version (def schemas [{:schema v1 :schemaVersion 1 @@ -581,4 +596,7 @@ :migration (constantly nil)} {:schema v40 :schemaVersion 40 - :migration migrations/v40}]) + :migration migrations/v40} + {:schema v41 + :schemaVersion 41 + :migration (constantly nil)}]) diff --git a/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs b/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs index b11d4591b0..d437140952 100644 --- a/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs +++ b/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs @@ -5,3 +5,14 @@ :properties {:topic :string :chat-ids :string :last-request {:type :int :default 1}}}) + +(def v2 + (-> v1 + (assoc-in + [:properties :gap-from] + {:type :int + :optional true}) + (assoc-in + [:properties :gap-to] + {:type :int + :optional true}))) diff --git a/src/status_im/data_store/realm/schemas/account/message.cljs b/src/status_im/data_store/realm/schemas/account/message.cljs index beee2ae7e4..f943b0a221 100644 --- a/src/status_im/data_store/realm/schemas/account/message.cljs +++ b/src/status_im/data_store/realm/schemas/account/message.cljs @@ -62,3 +62,9 @@ (-> v8 (assoc-in [:properties :raw-payload-hash] {:type :string}))) + +(def v10 + (-> v9 + (assoc-in [:properties :whisper-timestamp] + {:type :int + :optional true}))) diff --git a/src/status_im/data_store/realm/schemas/account/transport_inbox_topic.cljs b/src/status_im/data_store/realm/schemas/account/transport_inbox_topic.cljs index d14f6002ce..8a86a721de 100644 --- a/src/status_im/data_store/realm/schemas/account/transport_inbox_topic.cljs +++ b/src/status_im/data_store/realm/schemas/account/transport_inbox_topic.cljs @@ -1,3 +1,4 @@ +; This entity is not used in the newer version of schema (ns status-im.data-store.realm.schemas.account.transport-inbox-topic) (def v1 {:name :transport-inbox-topic diff --git a/src/status_im/events.cljs b/src/status_im/events.cljs index f337023179..314d9bf4c2 100644 --- a/src/status_im/events.cljs +++ b/src/status_im/events.cljs @@ -54,7 +54,8 @@ [status-im.utils.config :as config] [status-im.ui.components.bottom-sheet.core :as bottom-sheet] [status-im.ui.components.react :as react] - [status-im.utils.build :as build])) + [status-im.utils.build :as build] + [status-im.chat.db :as chat.db])) ;; init module @@ -445,7 +446,7 @@ (handlers/register-handler-fx :mailserver/fetch-history (fn [cofx [_ chat-id from-timestamp]] - (mailserver/fetch-history cofx chat-id from-timestamp))) + (mailserver/fetch-history cofx chat-id {:from from-timestamp}))) (handlers/register-handler-fx :mailserver.callback/generate-mailserver-symkey-success @@ -721,8 +722,22 @@ (handlers/register-handler-fx :chat.ui/fetch-history-pressed - (fn [cofx [_ chat-id]] - (mailserver/fetch-history cofx chat-id 1))) + (fn [{:keys [now] :as cofx} [_ chat-id]] + (mailserver/fetch-history cofx chat-id + {:from (- (quot now 1000) (* 24 3600))}))) + +(handlers/register-handler-fx + :chat.ui/fill-the-gap + (fn [{:keys [db] :as cofx}] + (let [mailserver-topics (:mailserver/topics db) + chat-id (:current-chat-id db) + topic (chat.db/topic-by-current-chat db) + gap (chat.db/messages-gap mailserver-topics topic)] + (mailserver/fill-the-gap + cofx + (assoc gap + :topic topic + :chat-id chat-id))))) (handlers/register-handler-fx :chat.ui/remove-chat-pressed diff --git a/src/status_im/mailserver/core.cljs b/src/status_im/mailserver/core.cljs index 1d9ecf0350..d6560fd367 100644 --- a/src/status_im/mailserver/core.cljs +++ b/src/status_im/mailserver/core.cljs @@ -2,13 +2,11 @@ status-im.mailserver.core (:require [re-frame.core :as re-frame] [status-im.accounts.db :as accounts.db] - [status-im.data-store.core :as data-store] [status-im.fleet.core :as fleet] [status-im.native-module.core :as status] [status-im.utils.platform :as platform] [status-im.transport.utils :as transport.utils] [status-im.utils.fx :as fx] - [status-im.constants :as constants] [status-im.utils.utils :as utils] [taoensso.timbre :as log] [status-im.transport.db :as transport.db] @@ -37,6 +35,7 @@ (def one-day (* 24 3600)) (def seven-days (* 7 one-day)) +(def max-request-range one-day) (def maximum-number-of-attempts 2) (def request-timeout 30) (def min-limit 100) @@ -298,7 +297,7 @@ (assoc-in [:mailserver/current-request :request-id] request-id))} {:db (assoc-in db [:mailserver/current-request :request-id] request-id)})))) -(defn request-messages! [web3 {:keys [sym-key-id address]} {:keys [topics cursor to from] :as request}] +(defn request-messages! [web3 {:keys [sym-key-id address]} {:keys [topics cursor to from force-to?] :as request}] ;; Add some room to from, unless we break day boundaries so that messages that have ;; been received after the last request are also fetched (let [actual-from (adjust-request-for-transit-time from) @@ -307,16 +306,20 @@ (log/info "mailserver: request-messages for: " " topics " topics " from " actual-from + " force-to? " force-to? + " to " to " cursor " cursor " limit " actual-limit) (.requestMessages (transport.utils/shh web3) - (clj->js {:topics topics - :mailServerPeer address - :symKeyID sym-key-id - :timeout request-timeout - :limit actual-limit - :cursor cursor - :from actual-from}) + (clj->js (cond-> {:topics topics + :mailServerPeer address + :symKeyID sym-key-id + :timeout request-timeout + :limit actual-limit + :cursor cursor + :from actual-from} + force-to? + (assoc :to to))) (fn [error request-id] (if-not error (do @@ -341,25 +344,43 @@ sym-key-id) mailserver))) -(defn ->request - [now-in-s [last-request topics]] - (when (< last-request now-in-s) - {:topics topics - ;; To is currently not sent to the mailserver, but we use to calculate - ;; when the last-request was sent. - :to now-in-s - :from (max last-request - (- now-in-s one-day))})) +(defn topic->request + [default-request-to requests-from requests-to] + (fn [[topic {:keys [last-request]}]] + (let [force-request-from (get requests-from topic) + force-request-to (get requests-to topic)] + (when (or force-request-from + (> default-request-to last-request)) + (let [from (or force-request-from + (max last-request + (- default-request-to max-request-range))) + to (or force-request-to default-request-to)] + {:topic topic + :from from + :to to + :force-to? (not (nil? force-request-to))}))))) + +(defn aggregate-requests + [acc {:keys [topic from to force-to?]}] + (update acc [from to force-to?] + (fn [{:keys [topics]}] + {:topics ((fnil conj #{}) topics topic) + :from from + :to to + ;; To is sent to the mailserver only when force-to? is true, + ;; also we use to calculate when the last-request was sent. + :force-to? force-to?}))) (defn prepare-messages-requests - [{:keys [db now] :as cofx} request-to] - (let [web3 (:web3 db)] - (->> - (:mailserver/topics db) - (reduce (fn [acc [topic {:keys [last-request]}]] - (update acc last-request conj topic)) - {}) - (keep (partial ->request request-to))))) + [{{:keys [:mailserver/requests-from + :mailserver/requests-to + :mailserver/topics]} :db} + default-request-to] + (transduce + (keep (topic->request default-request-to requests-from requests-to)) + (completing aggregate-requests vals) + {} + topics)) (fx/defn process-next-messages-request [{:keys [db now] :as cofx}] @@ -372,6 +393,7 @@ (quot now 1000)) requests (prepare-messages-requests cofx request-to) web3 (:web3 db)] + (log/debug "Mailserver: planned requests " requests) (if-let [request (first requests)] {:db (assoc db :mailserver/pending-requests (count requests) @@ -382,7 +404,9 @@ :request request}} {:db (dissoc db :mailserver/pending-requests - :mailserver/request-to)}))))) + :mailserver/request-to + :mailserver/requests-from + :mailserver/requests-to)}))))) (fx/defn add-mailserver-trusted "the current mailserver has been trusted @@ -485,14 +509,100 @@ {:topic topic :mailserver-topic mailserver-topic})]}))) -(defn get-updated-mailserver-topics [db topics last-request] - (reduce (fn [acc topic] - (if-let [mailserver-topic (some-> (get-in db [:mailserver/topics topic]) - (assoc :last-request last-request))] - (assoc acc topic mailserver-topic) - acc)) - {} - topics)) +(defn calculate-gap + [{:keys [gap-from + gap-to + last-request] :as config} + {:keys [request-from + request-to]}] + (merge config + (cond + (nil? gap-from) + {:gap-from request-to + :gap-to request-to + :last-request request-to} + + ;;------GF GT--------LRT F---T + (> request-from last-request) + {:gap-from last-request + :gap-to request-from + :last-request request-to} + + ;;------GF GT--------LRT + ;; F----------T + (and (>= last-request request-from gap-to) + (> request-to last-request)) + {:last-request request-to} + + ;;------GF GT--------LRT + ;; F----T + (and (>= last-request request-from gap-to) + (>= last-request request-to gap-to)) + config + + ;;------GF GT--------LRT + ;; F-------T + (and (> gap-to request-from gap-from) + (>= last-request request-to gap-to)) + {:gap-to request-from} + + ;;------GF GT--------LRT + ;; F-T + (and (> gap-to request-from gap-from) + (> gap-to request-to gap-from)) + config + + ;;------GF GT--------LRT + ;; F------T + (and (>= gap-from request-from) + (> gap-to request-to gap-from)) + {:gap-from request-to} + + ;;---------GF=GT=LRT + ;; F---T + (and (>= gap-from request-from) + (>= gap-from request-to) + (= gap-from last-request)) + {:gap-from request-to} + + ;;------GF GT--------LRT + ;; F---T + (and (>= gap-from request-from) + (>= gap-from request-to)) + config + + ;;------GF GT--------LRT + ;; F-------------T + (and (>= gap-from request-from) + (>= last-request request-to gap-to)) + {:gap-from last-request + :gap-to last-request + :last-request last-request} + + ;;------GF GT--------LRT + ;; F------------------------T + (and (>= gap-from request-from) + (>= request-to last-request)) + {:gap-from request-to + :gap-to request-to + :last-request request-to} + + ;;------GF GT--------LRT + ;; F-----------------T + (and (> gap-to request-from gap-from) + (>= request-to last-request)) + {:gap-to request-from + :last-request request-to}))) + +(defn get-updated-mailserver-topics [db requested-topics from to] + (into + {} + (keep (fn [topic] + (when-let [config (get-in db [:mailserver/topics topic])] + [topic (calculate-gap config + {:request-from from + :request-to to})]))) + requested-topics)) (fx/defn update-mailserver-topics "TODO: add support for cursors @@ -500,7 +610,7 @@ [{:keys [db now] :as cofx} {:keys [request-id cursor]}] (when-let [request (get db :mailserver/current-request)] (let [{:keys [from to topics]} request - mailserver-topics (get-updated-mailserver-topics db topics to)] + mailserver-topics (get-updated-mailserver-topics db topics from to)] (log/info "mailserver: message request " request-id "completed for mailserver topics" topics "from" from "to" to) (if (empty? mailserver-topics) @@ -516,16 +626,31 @@ :mailserver/request-messages {:web3 (:web3 db) :mailserver mailserver :request request-with-cursor}})) - (fx/merge cofx - {:db (-> db - (dissoc :mailserver/current-request) - (update :mailserver/topics merge mailserver-topics)) - :data-store/tx (mapv (fn [[topic mailserver-topic]] - (data-store.mailservers/save-mailserver-topic-tx - {:topic topic - :mailserver-topic mailserver-topic})) - mailserver-topics)} - (process-next-messages-request))))))) + (let [{:keys [chat-id] :as current-request} (db :mailserver/current-request) + gaps (db :mailserver/fetching-gaps-in-progress) + fetching-gap-completed? (= (get gaps chat-id) + (select-keys + current-request + [:from :to :force-to? :topics :chat-id]))] + (fx/merge cofx + {:db (-> db + (dissoc :mailserver/current-request) + (update :mailserver/requests-from + #(apply dissoc % topics)) + (update :mailserver/requests-to + #(apply dissoc % topics)) + (update :mailserver/topics merge mailserver-topics) + (update :mailserver/fetching-gaps-in-progress + (fn [gaps] + (if fetching-gap-completed? + (dissoc gaps chat-id) + gaps)))) + :data-store/tx (mapv (fn [[topic mailserver-topic]] + (data-store.mailservers/save-mailserver-topic-tx + {:topic topic + :mailserver-topic mailserver-topic})) + mailserver-topics)} + (process-next-messages-request)))))))) (fx/defn retry-next-messages-request [{:keys [db] :as cofx}] @@ -543,8 +668,8 @@ :mailserver/pending-requests))}) (fx/defn handle-request-completed - [{{:keys [chats] :as db} :db :as cofx} - {:keys [requestID lastEnvelopeHash cursor errorMessage] :as event}] + [{{:keys [chats]} :db :as cofx} + {:keys [requestID lastEnvelopeHash cursor errorMessage]}] (when (accounts.db/logged-in? cofx) (if (empty? errorMessage) (let [never-synced-chats-in-request @@ -596,12 +721,13 @@ add the chat-id to the topic and reset last-request there was no filter for the chat and messages for that so the whole history for that topic needs to be re-fetched" - [{:keys [db] :as cofx} {:keys [topic chat-id]}] + [{:keys [db now] :as cofx} {:keys [topic chat-id]}] (let [{:keys [chat-ids last-request] :as current-mailserver-topic} (get-in db [:mailserver/topics topic] {:chat-ids #{}})] (when-let [mailserver-topic (when-not (chat-ids chat-id) (-> current-mailserver-topic - (assoc :last-request 1) + (assoc :last-request (- (quot now 1000) + (* 24 60 60))) (update :chat-ids conj chat-id)))] (fx/merge cofx {:db (assoc-in db [:mailserver/topics topic] mailserver-topic) @@ -610,21 +736,42 @@ :mailserver-topic mailserver-topic})]})))) (fx/defn fetch-history - [{:keys [db] :as cofx} chat-id from-timestamp] - (log/debug "fetch-history" "chat-id:" chat-id "from-timestamp:" from-timestamp) + [{:keys [db] :as cofx} chat-id {:keys [from to]}] + + (log/debug "fetch-history" "chat-id:" chat-id "from-timestamp:" from) (let [public-key (accounts.db/current-public-key cofx) topic (or (get-in db [:transport/chats chat-id :topic]) - (transport.topic/public-key->discovery-topic-hash public-key)) - {:keys [chat-ids last-request] :as current-mailserver-topic} - (get-in db [:mailserver/topics topic] {:chat-ids #{}})] - (let [mailserver-topic (-> current-mailserver-topic - (assoc :last-request (min from-timestamp last-request)))] - (fx/merge cofx - {:db (assoc-in db [:mailserver/topics topic] mailserver-topic) - :data-store/tx [(data-store.mailservers/save-mailserver-topic-tx - {:topic topic - :mailserver-topic mailserver-topic})]} - (process-next-messages-request))))) + (transport.topic/public-key->discovery-topic-hash public-key))] + (fx/merge cofx + {:db (cond-> (assoc-in db [:mailserver/requests-from topic] from) + + to + (assoc-in [:mailserver/requests-to topic] to))} + (process-next-messages-request)))) + +(fx/defn fill-the-gap + [{:keys [db] :as cofx} {:keys [exists? from to topic chat-id]}] + (let [mailserver (get-mailserver-when-ready cofx) + request {:from from + :to to + :force-to? true + :topics [topic] + :chat-id chat-id}] + (when exists? + {:db + (-> db + (assoc + :mailserver/pending-requests 1 + :mailserver/current-request request + :mailserver/request-to to) + + (update :mailserver/fetching-gaps-in-progress + assoc chat-id request)) + + :mailserver/request-messages + {:web3 (:web3 db) + :mailserver mailserver + :request request}}))) (fx/defn resend-request [{:keys [db] :as cofx} {:keys [request-id]}] diff --git a/src/status_im/mailserver/subs.cljs b/src/status_im/mailserver/subs.cljs index 0ace2cb161..5cbdf39ffa 100644 --- a/src/status_im/mailserver/subs.cljs +++ b/src/status_im/mailserver/subs.cljs @@ -43,6 +43,13 @@ (pos-int? pending-requests) (not (or connecting? connection-error? request-error?))))) +(re-frame/reg-sub + :mailserver/connected? + (fn [db] + (let [connected? (= :connected (:mailserver/state db)) + online? (= :online (:network-status db))] + (and connected? online?)))) + (re-frame/reg-sub :mailserver/current-id (fn [db] diff --git a/src/status_im/transport/message/protocol.cljs b/src/status_im/transport/message/protocol.cljs index 60b45fb141..dba4e60ff4 100644 --- a/src/status_im/transport/message/protocol.cljs +++ b/src/status_im/transport/message/protocol.cljs @@ -113,7 +113,7 @@ (fx/merge cofx (send-direct-message current-public-key nil this) (send-with-pubkey params))))) - (receive [this chat-id signature _ cofx] + (receive [this chat-id signature timestamp cofx] {:chat-received-message/add-fx [(assoc (into {} this) :old-message-id (transport.utils/old-message-id this) @@ -121,6 +121,7 @@ signature (.-payload (:js-obj cofx))) :chat-id chat-id + :whisper-timestamp timestamp :raw-payload-hash (transport.utils/sha3 (.-payload (:js-obj cofx))) :from signature diff --git a/src/status_im/ui/screens/chat/views.cljs b/src/status_im/ui/screens/chat/views.cljs index a6a4c450ed..bfff44c816 100644 --- a/src/status_im/ui/screens/chat/views.cljs +++ b/src/status_im/ui/screens/chat/views.cljs @@ -70,6 +70,37 @@ [{{:keys [value]} :row}] [message-datemark/chat-datemark-mobile value]) +(defview gap [] + (letsubs [in-progress? [:chats/fetching-gap-in-progress?] + connected? [:mailserver/connected?]] + [react/view {:align-self :stretch + :margin-top 24 + :margin-bottom 24 + :height 48 + :align-items :center + :justify-content :center + :border-color colors/gray-light + :border-top-width 1 + :border-bottom-width 1 + :background-color :white} + [react/touchable-highlight + {:on-press (when (and connected? (not in-progress?)) + #(re-frame/dispatch [:chat.ui/fill-the-gap]))} + [react/view {:flex 1 + :align-items :center + :justify-content :center} + (if in-progress? + [react/activity-indicator] + [react/text + {:style {:color (if connected? + colors/blue + colors/gray)}} + (i18n/label :t/fetch-messages)])]]])) + +(defmethod message-row :gap + [_] + [gap]) + (defmethod message-row :default [{:keys [group-chat current-public-key modal? row]}] [message/chat-message (assoc row diff --git a/test/cljs/status_im/test/chat/db.cljs b/test/cljs/status_im/test/chat/db.cljs index b3f2ec4f09..8524af01d9 100644 --- a/test/cljs/status_im/test/chat/db.cljs +++ b/test/cljs/status_im/test/chat/db.cljs @@ -1,35 +1,35 @@ (ns status-im.test.chat.db (:require [cljs.test :refer-macros [deftest is testing]] - [status-im.chat.db :as s])) + [status-im.chat.db :as db])) (deftest group-chat-name (testing "it prepends # if it's a public chat" - (is (= "#withhash" (s/group-chat-name {:group-chat true - :chat-id "1" - :public? true - :name "withhash"})))) + (is (= "#withhash" (db/group-chat-name {:group-chat true + :chat-id "1" + :public? true + :name "withhash"})))) (testing "it leaves the name unchanged if it's a group chat" - (is (= "unchanged" (s/group-chat-name {:group-chat true - :chat-id "1" - :name "unchanged"}))))) + (is (= "unchanged" (db/group-chat-name {:group-chat true + :chat-id "1" + :name "unchanged"}))))) (deftest message-stream-tests (testing "messages with no interspersed datemarks" - (let [m1 {:from "1" - :datemark "a" - :outgoing false} - m2 {:from "2" - :datemark "a" - :outgoing true} - m3 {:from "2" - :datemark "a" - :outgoing true} - dm1 {:type :datemark - :value "a"} + (let [m1 {:from "1" + :datemark "a" + :outgoing false} + m2 {:from "2" + :datemark "a" + :outgoing true} + m3 {:from "2" + :datemark "a" + :outgoing true} + dm1 {:type :datemark + :value "a"} messages [m1 m2 m3 dm1] [actual-m1 actual-m2 - actual-m3] (s/messages-stream messages)] + actual-m3] (db/messages-stream messages)] (testing "it marks only the first message as :last?" (is (:last? actual-m1)) (is (not (:last? actual-m2))) @@ -59,29 +59,29 @@ (is (:last-in-group? actual-m2)) (is (not (:last-in-group? actual-m3)))))) (testing "messages with interspersed datemarks" - (let [m1 {:from "2" ; first & last in group - :timestamp 63000 - :outgoing true} - dm1 {:type :datemark - :value "a"} - m2 {:from "2" ; first & last in group as more than 1 minute after previous message - :timestamp 62000 - :outgoing false} - m3 {:from "2" ; last in group - :timestamp 1 - :outgoing false} - m4 {:from "2" ; first in group - :timestamp 0 - :outgoing false} - dm2 {:type :datemark - :value "b"} + (let [m1 {:from "2" ; first & last in group + :timestamp 63000 + :outgoing true} + dm1 {:type :datemark + :value "a"} + m2 {:from "2" ; first & last in group as more than 1 minute after previous message + :timestamp 62000 + :outgoing false} + m3 {:from "2" ; last in group + :timestamp 1 + :outgoing false} + m4 {:from "2" ; first in group + :timestamp 0 + :outgoing false} + dm2 {:type :datemark + :value "b"} messages [m1 dm1 m2 m3 m4 dm2] [actual-m1 _ actual-m2 actual-m3 actual-m4 - _] (s/messages-stream messages)] + _] (db/messages-stream messages)] (testing "it marks the first outgoing message as :last-outgoing?" (is (:last-outgoing? actual-m1)) (is (not (:last-outgoing? actual-m2))) @@ -103,12 +103,12 @@ (is (:last-in-group? actual-m3)) (is (not (:last-in-group? actual-m4)))))) (testing "system-messages" - (let [m1 {:from "system" - :message-type :system-message - :datemark "a" - :outgoing false} + (let [m1 {:from "system" + :message-type :system-message + :datemark "a" + :outgoing false} messages [m1] - [actual-m1] (s/messages-stream messages)] + [actual-m1] (db/messages-stream messages)] (testing "it does display the photo" (is (:display-photo? actual-m1)) (testing "it does not display the username" @@ -122,4 +122,218 @@ "3" {:is-active false :chat-id "3"}}] (testing "it returns only chats with is-active" (is (= #{"1" "2"} - (set (keys (s/active-chats {} chats {})))))))) + (set (keys (db/active-chats {} chats {})))))))) + +(deftest messages-with-datemarks-and-statuses + (testing "empty state" + (is (empty? + (db/messages-with-datemarks-and-statuses + nil + nil + nil + nil + nil)))) + (testing "simple case" + (is (= + '({:whisper-timestamp 40 + :timestamp 40 + :content nil + :timestamp-str "14:00" + :user-statuses nil + :datemark "today"} + {:whisper-timestamp 30 + :timestamp 30 + :content nil + :timestamp-str "13:00" + :user-statuses nil + :datemark "today"} + {:value "today" + :type :datemark + :whisper-timestamp 30 + :timestamp 30} + {:whisper-timestamp 20 + :timestamp 20 + :content nil + :timestamp-str "12:00" + :user-statuses nil + :datemark "yesterday"} + {:whisper-timestamp 10 + :timestamp 10 + :content nil + :timestamp-str "11:00" + :user-statuses nil + :datemark "yesterday"} + {:value "yesterday" + :type :datemark + :whisper-timestamp 10 + :timestamp 10}) + (db/messages-with-datemarks-and-statuses + {"yesterday" + (list + {:message-id :m1 + :timestamp-str "11:00" + :whisper-timestamp 10 + :timestamp 10} + {:message-id :m2 + :timestamp-str "12:00" + :whisper-timestamp 20 + :timestamp 20}) + "today" + (list + {:message-id :m3 + :timestamp-str "13:00" + :whisper-timestamp 30 + :timestamp 30} + {:message-id :m4 + :timestamp-str "14:00" + :whisper-timestamp 40 + :timestamp 40})} + {:m1 {:whisper-timestamp 10 + :timestamp 10} + :m2 {:whisper-timestamp 20 + :timestamp 20} + :m3 {:whisper-timestamp 30 + :timestamp 30} + :m4 {:whisper-timestamp 40 + :timestamp 40}} + nil + nil + nil)))) + (testing "simple case with gap" + (is (= + '({:whisper-timestamp 40 + :timestamp 40 + :content nil + :timestamp-str "14:00" + :user-statuses nil + :datemark "today"} + {:whisper-timestamp 30 + :timestamp 30 + :content nil + :timestamp-str "13:00" + :user-statuses nil + :datemark "today"} + {:value "today" + :type :datemark + :whisper-timestamp 30 + :timestamp 30} + {:type :gap + :value "25"} + {:whisper-timestamp 20 + :timestamp 20 + :content nil + :timestamp-str "12:00" + :user-statuses nil + :datemark "yesterday"} + {:whisper-timestamp 10 + :timestamp 10 + :content nil + :timestamp-str "11:00" + :user-statuses nil + :datemark "yesterday"} + {:value "yesterday" + :type :datemark + :whisper-timestamp 10 + :timestamp 10}) + (db/messages-with-datemarks-and-statuses + {"yesterday" + (list + {:message-id :m1 + :timestamp-str "11:00" + :whisper-timestamp 10 + :timestamp 10} + {:message-id :m2 + :timestamp-str "12:00" + :whisper-timestamp 20 + :timestamp 20}) + "today" + (list + {:message-id :m3 + :timestamp-str "13:00" + :whisper-timestamp 30 + :timestamp 30} + {:message-id :m4 + :timestamp-str "14:00" + :whisper-timestamp 40 + :timestamp 40})} + {:m1 {:whisper-timestamp 10 + :timestamp 10} + :m2 {:whisper-timestamp 20 + :timestamp 20} + :m3 {:whisper-timestamp 30 + :timestamp 30} + :m4 {:whisper-timestamp 40 + :timestamp 40}} + nil + nil + {:from 25 + :exists? true})))) + (testing "simple case with gap after all messages" + (is (= + '({:type :gap + :value "100"} + {:whisper-timestamp 40 + :timestamp 40 + :content nil + :timestamp-str "14:00" + :user-statuses nil + :datemark "today"} + {:whisper-timestamp 30 + :timestamp 30 + :content nil + :timestamp-str "13:00" + :user-statuses nil + :datemark "today"} + {:value "today" + :type :datemark + :whisper-timestamp 30 + :timestamp 30} + {:whisper-timestamp 20 + :timestamp 20 + :content nil + :timestamp-str "12:00" + :user-statuses nil + :datemark "yesterday"} + {:whisper-timestamp 10 + :timestamp 10 + :content nil + :timestamp-str "11:00" + :user-statuses nil + :datemark "yesterday"} + {:value "yesterday" + :type :datemark + :whisper-timestamp 10 + :timestamp 10}) + (db/messages-with-datemarks-and-statuses + {"yesterday" + (list + {:message-id :m1 + :timestamp-str "11:00" + :whisper-timestamp 10 + :timestamp 10} + {:message-id :m2 + :timestamp-str "12:00" + :whisper-timestamp 20 + :timestamp 20}) + "today" + (list + {:message-id :m3 + :timestamp-str "13:00" + :whisper-timestamp 30 + :timestamp 30} + {:message-id :m4 + :timestamp-str "14:00" + :whisper-timestamp 40 + :timestamp 40})} + {:m1 {:whisper-timestamp 10 + :timestamp 10} + :m2 {:whisper-timestamp 20 + :timestamp 20} + :m3 {:whisper-timestamp 30 + :timestamp 30} + :m4 {:whisper-timestamp 40 + :timestamp 40}} + nil + nil + {:from 100 + :exists? true}))))) diff --git a/test/cljs/status_im/test/mailserver/core.cljs b/test/cljs/status_im/test/mailserver/core.cljs index e8f681c1a3..83b6d05f3b 100644 --- a/test/cljs/status_im/test/mailserver/core.cljs +++ b/test/cljs/status_im/test/mailserver/core.cljs @@ -627,3 +627,187 @@ (is (not (-> (mailserver/connect-to-mailserver {:db mailserver-with-sym-key-db}) :shh/generate-sym-key-from-password first))))))) + +(deftest calculate-gap + (testing "new topic" + (is (= {:gap-from 10 + :gap-to 10 + :last-request 10} + + (mailserver/calculate-gap + {:gap-from nil + :gap-to nil + :last-request nil} + {:request-from 5 + :request-to 10})))) + (testing "calculate-gap#1" + (is (= {:gap-from 3 + :gap-to 4 + :last-request 5} + + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 3} + {:request-from 4 + :request-to 5})))) + (testing "calculate-gap#2" + (is (= {:gap-from 1 + :gap-to 2 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 4} + {:request-from 3 + :request-to 5})))) + (testing "calculate-gap#2-1" + (is (= {:gap-from 1 + :gap-to 2 + :last-request 4} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 3} + {:request-from 3 + :request-to 4})))) + (testing "calculate-gap#3" + (is (= {:gap-from 1 + :gap-to 2 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 5} + {:request-from 3 + :request-to 4})))) + (testing "calculate-gap#3-1" + (is (= {:gap-from 1 + :gap-to 2 + :last-request 3} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 3} + {:request-from 2 + :request-to 3})))) + (testing "calculate-gap#4" + (is (= {:gap-from 1 + :gap-to 2 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 5} + {:request-from 3 + :request-to 4})))) + (testing "calculate-gap#5" + (is (= {:gap-from 1 + :gap-to 4 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 4 + :last-request 5} + {:request-from 2 + :request-to 3})))) + (testing "calculate-gap#6" + (is (= {:gap-from 2 + :gap-to 3 + :last-request 4} + (mailserver/calculate-gap + {:gap-from 2 + :gap-to 3 + :last-request 4} + {:request-from 1 + :request-to 2})))) + (testing "calculate-gap#6-1" + (is (= {:gap-from 1 + :gap-to 4 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 4 + :last-request 5} + {:request-from 2 + :request-to 3})))) + (testing "calculate-gap#0" + (is (= {:gap-from 2 + :gap-to 3 + :last-request 3} + (mailserver/calculate-gap + {:gap-from 3 + :gap-to 3 + :last-request 3} + {:request-from 1 + :request-to 2})))) + (testing "calculate-gap#7" + (is (= {:gap-from 3 + :gap-to 4 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 3 + :gap-to 4 + :last-request 5} + {:request-from 1 + :request-to 2})))) + (testing "calculate-gap#8" + (is (= {:gap-from 5 + :gap-to 5 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 2 + :gap-to 3 + :last-request 5} + {:request-from 1 + :request-to 4})))) + (testing "calculate-gap#8-1" + (is (= {:gap-from 3 + :gap-to 3 + :last-request 3} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 3} + {:request-from 1 + :request-to 2})))) + (testing "calculate-gap#9" + (is (= {:gap-from 5 + :gap-to 5 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 2 + :gap-to 3 + :last-request 4} + {:request-from 1 + :request-to 5})))) + (testing "calculate-gap#9-1" + (is (= {:gap-from 3 + :gap-to 3 + :last-request 3} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 3} + {:request-from 1 + :request-to 3})))) + (testing "calculate-gap#10" + (is (= {:gap-from 1 + :gap-to 2 + :last-request 5} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 3 + :last-request 4} + {:request-from 2 + :request-to 5})))) + (testing "calculate-gap#10-1" + (is (= {:gap-from 1 + :gap-to 2 + :last-request 3} + (mailserver/calculate-gap + {:gap-from 1 + :gap-to 2 + :last-request 3} + {:request-from 2 + :request-to 3}))))) diff --git a/translations/en.json b/translations/en.json index 26829e6c2e..cbd6319bf1 100644 --- a/translations/en.json +++ b/translations/en.json @@ -997,5 +997,6 @@ "chaos-unicorn-day-details": "\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83E\uDD84\uD83D\uDE80!", "invalid-format": "Invalid format\nMust be {{format}}", "mailserver-format": "enode://{enode-id}:{password}@{ip-address}:{port}", - "bootnode-format": "enode://{enode-id}@{ip-address}:{port}" + "bootnode-format": "enode://{enode-id}@{ip-address}:{port}", + "fetch-messages": "↓ Fetch messages" }