diff --git a/src/status_im/chat/db.cljs b/src/status_im/chat/db.cljs index ab5571a924..64acb033b5 100644 --- a/src/status_im/chat/db.cljs +++ b/src/status_im/chat/db.cljs @@ -58,15 +58,6 @@ (get-in db [:transport/chats current-chat-id :topic]) (topic/public-key->discovery-topic-hash public-key)))) -(defn messages-gap - [mailserver-topics topic] - (let [{:keys [gap-from gap-to]} - (get mailserver-topics topic)] - {:from gap-from - :to gap-to - :exists? (and gap-from gap-to - (> gap-to gap-from))})) - (defn sort-message-groups "Sorts message groups according to timestamp of first message in group" [message-groups messages] @@ -113,41 +104,54 @@ (assoc-in [:content :response-to] quote)))))) (defn check-gap - [{:keys [exists? from]} previous-message next-message gap-added?] - (let [previous-timestamp (:whisper-timestamp previous-message) - next-whisper-timestamp (:whisper-timestamp next-message) - next-timestamp (quot (:timestamp next-message) 1000) - ignore-next-message? (> (js/Math.abs - (- next-whisper-timestamp next-timestamp)) - 120)] - (and (not gap-added?) - (or (and exists? previous-timestamp next-whisper-timestamp - (not ignore-next-message?) - (< previous-timestamp from next-whisper-timestamp)) - (and exists? (nil? next-message)))))) + [gaps previous-message next-message] + (let [previous-timestamp (:whisper-timestamp previous-message) + next-whisper-timestamp (:whisper-timestamp next-message) + next-timestamp (quot (:timestamp next-message) 1000) + ignore-next-message? (> (js/Math.abs + (- next-whisper-timestamp next-timestamp)) + 120)] + (reduce + (fn [acc {:keys [from id]}] + (if (and next-message + (not ignore-next-message?) + (or + (and (nil? previous-timestamp) + (< from next-whisper-timestamp)) + (< previous-timestamp from next-whisper-timestamp))) + (-> acc + (update :gaps-number inc) + (update-in [:gap :ids] conj id)) + (reduced acc))) + {:gaps-number 0 + :gap nil} + gaps))) -(defn add-gap [messages gap] +(defn add-gap [messages gaps] (conj messages {:type :gap - :value (str (:from gap))})) + :value (clojure.string/join (:ids gaps)) + :gaps gaps})) (defn messages-with-datemarks-and-statuses "Converts message groups into sequence of messages interspersed with datemarks, with correct user statuses associated into message" - [message-groups messages message-statuses referenced-messages messages-gap] + [message-groups messages message-statuses referenced-messages messages-gaps] (transduce (comp (mapcat add-datemark) (map (transform-message messages message-statuses referenced-messages))) (completing - (fn [{:keys [messages datemark-reference previous-message gap-added?]} + (fn [{:keys [messages datemark-reference previous-message gaps]} message] (let [new-datemark? (datemark? message) - add-gap? (check-gap messages-gap previous-message message gap-added?)] + {:keys [gaps-number gap]} + (check-gap gaps previous-message message) + add-gap? (pos? gaps-number)] {:messages (cond-> messages add-gap? - (add-gap messages-gap) + (add-gap gap) :always (conj @@ -160,14 +164,16 @@ :datemark-reference (if new-datemark? message datemark-reference) - :gap-added? (or gap-added? add-gap?)})) - (fn [{:keys [messages previous-message gap-added?]}] - (let [add-gap? (check-gap messages-gap previous-message nil gap-added?)] - (cond-> messages - add-gap? - (add-gap messages-gap))))) + :gaps (if add-gap? + (drop gaps-number gaps) + gaps)})) + (fn [{:keys [messages gaps]}] + (cond-> messages + (seq gaps) + (add-gap {:ids (map :id gaps)})))) {:messages (list) - :previous-message nil} + :previous-message nil + :gaps messages-gaps} message-groups)) (defn- set-previous-message-info [stream] diff --git a/src/status_im/chat/models.cljs b/src/status_im/chat/models.cljs index d534ec5af6..cf5fc6b053 100644 --- a/src/status_im/chat/models.cljs +++ b/src/status_im/chat/models.cljs @@ -19,7 +19,9 @@ [status-im.utils.gfycat.core :as gfycat] [status-im.utils.platform :as platform] [status-im.utils.priority-map :refer [empty-message-map]] - [status-im.utils.utils :as utils])) + [status-im.utils.utils :as utils] + [status-im.mailserver.core :as mailserver] + [status-im.transport.partitioned-topic :as transport.topic])) (defn- get-chat [cofx chat-id] (get-in cofx [:db :chats chat-id])) @@ -163,6 +165,10 @@ (fx/merge cofx #(when (public-chat? % chat-id) (transport.chat/unsubscribe-from-chat % chat-id)) + #(when (group-chat? % chat-id) + (mailserver/remove-chat-from-mailserver-topic % chat-id)) + (mailserver/remove-gaps chat-id) + (mailserver/remove-range chat-id) (deactivate-chat chat-id) (clear-history chat-id) #(when (one-to-one-chat? % chat-id) @@ -242,13 +248,12 @@ (fx/defn preload-chat-data "Takes chat-id and coeffects map, returns effects necessary when navigating to chat" [{:keys [db] :as cofx} chat-id] - (let [chat (get-in db [:chats chat-id])] - (fx/merge cofx - {:db (-> (assoc db :current-chat-id chat-id) - (set-chat-ui-props {:validation-messages nil}))} - (contact-code/listen-to-chat chat-id) - (when platform/desktop? - (mark-messages-seen chat-id))))) + (fx/merge cofx + {:db (-> (assoc db :current-chat-id chat-id) + (set-chat-ui-props {:validation-messages nil}))} + (contact-code/listen-to-chat chat-id) + (when platform/desktop? + (mark-messages-seen chat-id)))) (fx/defn navigate-to-chat "Takes coeffects map and chat-id, returns effects necessary for navigation and preloading data" @@ -277,7 +282,7 @@ ;; don't allow to open chat with yourself (when (not= (accounts.db/current-public-key cofx) chat-id) (fx/merge cofx - (upsert-chat {:chat-id chat-id + (upsert-chat {:chat-id chat-id :is-active true}) (navigate-to-chat chat-id opts)))) diff --git a/src/status_im/chat/models/loading.cljs b/src/status_im/chat/models/loading.cljs index 34ae7aab25..b077defbfd 100644 --- a/src/status_im/chat/models/loading.cljs +++ b/src/status_im/chat/models/loading.cljs @@ -6,7 +6,8 @@ [status-im.data-store.user-statuses :as user-statuses-store] [status-im.utils.datetime :as time] [status-im.utils.fx :as fx] - [status-im.utils.priority-map :refer [empty-message-map]])) + [status-im.utils.priority-map :refer [empty-message-map]] + [status-im.mailserver.core :as mailserver])) (def index-messages (partial into empty-message-map (map (juxt :message-id identity)))) @@ -114,5 +115,6 @@ (chat-model/update-chats-unviewed-messages-count {:chat-id current-chat-id :new-loaded-unviewed-messages-ids loaded-unviewed-messages}) + (mailserver/load-gaps current-chat-id) (group-chat-messages current-chat-id new-messages) (chat-model/mark-messages-seen current-chat-id))))) diff --git a/src/status_im/chat/models/message.cljs b/src/status_im/chat/models/message.cljs index bfe0cd524f..c64af9f578 100644 --- a/src/status_im/chat/models/message.cljs +++ b/src/status_im/chat/models/message.cljs @@ -466,8 +466,9 @@ (let [{:keys [chats]} db {:keys [last-clock-value] :as chat} (get chats chat-id) message-data (-> message - (assoc :from (accounts.db/current-public-key cofx) - :timestamp now + (assoc :from (accounts.db/current-public-key cofx) + :timestamp now + :whisper-timestamp (quot now 1000) :clock-value (utils.clocks/send last-clock-value)) (add-message-type chat))] diff --git a/src/status_im/chat/subs.cljs b/src/status_im/chat/subs.cljs index a4ba494482..d286b8842e 100644 --- a/src/status_im/chat/subs.cljs +++ b/src/status_im/chat/subs.cljs @@ -172,11 +172,11 @@ (chat.db/topic-by-current-chat db))) (re-frame/reg-sub - :chats/messages-gap - :<- [:get-in [:mailserver/topics]] - :<- [:chats/current-chat-topic] - (fn [[mailserver-topics topic]] - (chat.db/messages-gap mailserver-topics topic))) + :chats/messages-gaps + :<- [:get-in [:mailserver/gaps]] + :<- [:chats/current-chat-id] + (fn [[gaps chat-id]] + (sort-by :from (vals (get gaps chat-id))))) (re-frame/reg-sub :chats/current-chat-messages-stream @@ -184,18 +184,18 @@ :<- [:chats/current-chat-message-groups] :<- [:chats/current-chat-message-statuses] :<- [:chats/current-chat-referenced-messages] - :<- [:chats/messages-gap] - (fn [[messages message-groups message-statuses referenced-messages messages-gap]] + :<- [:chats/messages-gaps] + (fn [[messages message-groups message-statuses referenced-messages messages-gaps]] (-> (chat.db/sort-message-groups message-groups messages) - (chat.db/messages-with-datemarks-and-statuses messages message-statuses referenced-messages messages-gap) + (chat.db/messages-with-datemarks-and-statuses messages message-statuses referenced-messages messages-gaps) chat.db/messages-stream))) (re-frame/reg-sub :chats/fetching-gap-in-progress? - (fn [db] + (fn [db [_ ids]] (let [chat-id (:current-chat-id db) gaps (:mailserver/fetching-gaps-in-progress db)] - (contains? gaps chat-id)))) + (seq (select-keys (get gaps chat-id) ids))))) (re-frame/reg-sub :chats/current-chat-intro-status diff --git a/src/status_im/contact/core.cljs b/src/status_im/contact/core.cljs index b4db07b6ee..c654c2b982 100644 --- a/src/status_im/contact/core.cljs +++ b/src/status_im/contact/core.cljs @@ -21,7 +21,8 @@ [status-im.transport.partitioned-topic :as transport.topic] [status-im.utils.config :as config] [status-im.chat.models.loading :as chat.models.loading] - [status-im.chat.models.message :as chat.models.message])) + [status-im.chat.models.message :as chat.models.message] + [status-im.mailserver.core :as mailserver])) (fx/defn load-contacts [{:keys [db all-contacts]}] @@ -74,7 +75,12 @@ (fx/merge cofx {:db (assoc-in db [:contacts/new-identity] "")} (upsert-contact contact) - (send-contact-request contact))))) + (mailserver/upsert-mailserver-topic + {:chat-ids [public-key] + :topic transport.topic/discovery-topic-hash + :fetch? false}) + (send-contact-request contact) + (mailserver/process-next-messages-request))))) (fx/defn add-contacts-filter [{:keys [db]} public-key action] (when (not= (get-in db [:account/account :public-key]) public-key) diff --git a/src/status_im/contact_code/core.cljs b/src/status_im/contact_code/core.cljs index ab61b1e14c..f5c438762d 100644 --- a/src/status_im/contact_code/core.cljs +++ b/src/status_im/contact_code/core.cljs @@ -10,7 +10,8 @@ [status-im.transport.message.public-chat :as transport.public-chat] [status-im.data-store.accounts :as data-store.accounts] [status-im.transport.chat.core :as transport.chat] - [status-im.accounts.db :as accounts.db])) + [status-im.accounts.db :as accounts.db] + [status-im.mailserver.core :as mailserver])) (defn topic [pk] (str pk "-contact-code")) @@ -46,15 +47,17 @@ (and is-active (contains? members-joined my-public-key) (contains? members their-public-key))) - (vals (:chats db)))] + (vals (:chats db))) + their-topic (topic their-public-key)] (when (and (not (contact.db/active? db their-public-key)) (not= my-public-key their-public-key) (not (get-in db [:chats their-public-key :is-active])) (empty? active-group-chats)) - (fx/merge cofx - (transport.chat/unsubscribe-from-chat (topic their-public-key)))))) + (mailserver/remove-gaps their-topic) + (mailserver/remove-range their-topic) + (transport.chat/unsubscribe-from-chat their-topic))))) ;; Publish contact code every 12hrs (def publish-contact-code-interval (* 12 60 60 1000)) diff --git a/src/status_im/data_store/mailservers.cljs b/src/status_im/data_store/mailservers.cljs index 0a1ae17f9b..4295b699b3 100644 --- a/src/status_im/data_store/mailservers.cljs +++ b/src/status_im/data_store/mailservers.cljs @@ -64,3 +64,57 @@ (let [mailserver-topic (core/single (core/get-by-field realm :mailserver-topic :topic topic))] (core/delete realm mailserver-topic)))) + +(defn save-chat-requests-range + [chat-requests-range] + (fn [realm] + (core/create realm :chat-requests-range chat-requests-range true))) + +(re-frame/reg-cofx + :data-store/all-chat-requests-ranges + (fn [cofx _] + (assoc cofx + :data-store/all-chat-requests-ranges + (reduce (fn [acc {:keys [chat-id] :as range}] + (assoc acc chat-id range)) + {} + (-> @core/account-realm + (core/get-all :chat-requests-range) + (core/all-clj :chat-requests-range)))))) +(re-frame/reg-cofx + :data-store/all-gaps + (fn [cofx _] + (assoc cofx + :data-store/all-gaps + (fn [chat-id] + (reduce (fn [acc {:keys [id] :as gap}] + (assoc acc id gap)) + {} + (-> @core/account-realm + (core/get-by-field :mailserver-requests-gap :chat-id chat-id) + (core/all-clj :mailserver-requests-gap))))))) + +(defn save-mailserver-requests-gap + [gap] + (fn [realm] + (core/create realm :mailserver-requests-gap gap true))) + +(defn delete-mailserver-requests-gaps + [ids] + (fn [realm] + (doseq [id ids] + (core/delete + realm + (core/get-by-field realm :mailserver-requests-gap :id id))))) + +(defn delete-all-gaps-by-chat + [chat-id] + (fn [realm] + (core/delete realm + (core/get-by-field realm :mailserver-requests-gap :chat-id chat-id)))) + +(defn delete-range + [chat-id] + (fn [realm] + (core/delete realm + (core/get-by-field realm :chat-requests-range :chat-id chat-id)))) diff --git a/src/status_im/data_store/realm/schemas/account/chat_requests_range.cljs b/src/status_im/data_store/realm/schemas/account/chat_requests_range.cljs new file mode 100644 index 0000000000..163f6fe6e0 --- /dev/null +++ b/src/status_im/data_store/realm/schemas/account/chat_requests_range.cljs @@ -0,0 +1,9 @@ +(ns status-im.data-store.realm.schemas.account.chat-requests-range) + +(def v1 {:name :chat-requests-range + :primaryKey :chat-id + :properties {:chat-id :string + :lowest-request-from {:type :int + :optional true} + :highest-request-to {:type :int + :optional true}}}) diff --git a/src/status_im/data_store/realm/schemas/account/core.cljs b/src/status_im/data_store/realm/schemas/account/core.cljs index ce9d053009..4bc2c64da0 100644 --- a/src/status_im/data_store/realm/schemas/account/core.cljs +++ b/src/status_im/data_store/realm/schemas/account/core.cljs @@ -15,7 +15,9 @@ [status-im.data-store.realm.schemas.account.membership-update :as membership-update] [status-im.data-store.realm.schemas.account.installation :as installation] [status-im.data-store.realm.schemas.account.contact-recovery :as contact-recovery] - [status-im.data-store.realm.schemas.account.migrations :as migrations])) + [status-im.data-store.realm.schemas.account.mailserver-requests-gap :as mailserver-requests-gap] + [status-im.data-store.realm.schemas.account.migrations :as migrations] + [status-im.data-store.realm.schemas.account.chat-requests-range :as chat-requests-range])) (def v1 [chat/v1 transport/v1 @@ -476,6 +478,23 @@ contact-device-info/v1 contact-recovery/v1]) +(def v42 [chat/v14 + chat-requests-range/v1 + transport/v8 + contact/v7 + message/v10 + mailserver/v11 + mailserver-topic/v2 + user-status/v2 + membership-update/v1 + installation/v3 + local-storage/v1 + browser/v8 + dapp-permissions/v9 + contact-device-info/v1 + contact-recovery/v1 + mailserver-requests-gap/v1]) + ;; put schemas ordered by version (def schemas [{:schema v1 :schemaVersion 1 @@ -599,4 +618,7 @@ :migration migrations/v40} {:schema v41 :schemaVersion 41 - :migration (constantly nil)}]) + :migration (constantly nil)} + {:schema v42 + :schemaVersion 42 + :migration migrations/v42}]) diff --git a/src/status_im/data_store/realm/schemas/account/mailserver_requests_gap.cljs b/src/status_im/data_store/realm/schemas/account/mailserver_requests_gap.cljs new file mode 100644 index 0000000000..1d5ba805b9 --- /dev/null +++ b/src/status_im/data_store/realm/schemas/account/mailserver_requests_gap.cljs @@ -0,0 +1,10 @@ +(ns status-im.data-store.realm.schemas.account.mailserver-requests-gap) + +(def v1 {:name :mailserver-requests-gap + :primaryKey :id + :properties {:id :string + :chat-id {:type :string + :indexed true} + :from {:type :int + :indexed true} + :to :int}}) diff --git a/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs b/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs index d437140952..d6e9c6dc86 100644 --- a/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs +++ b/src/status_im/data_store/realm/schemas/account/mailserver_topic.cljs @@ -16,3 +16,6 @@ [:properties :gap-to] {:type :int :optional true}))) + +(def v3 + (update v2 :properties dissoc :gap-to :gap-from)) diff --git a/src/status_im/data_store/realm/schemas/account/migrations.cljs b/src/status_im/data_store/realm/schemas/account/migrations.cljs index 127b66b933..80852662a6 100644 --- a/src/status_im/data_store/realm/schemas/account/migrations.cljs +++ b/src/status_im/data_store/realm/schemas/account/migrations.cljs @@ -389,3 +389,31 @@ (or (true? pending?) (zero? last-updated)) (conj ":contact/request-received"))] (aset new-contact "system-tags" (clj->js system-tags)))))) + +(defn private-chats-ids [chats] + (set + (keep + (fn [i] + (let [chat (aget chats i)] + (when-not (aget chat "public?") + (aget chat "chat-id")))) + (range (.-length chats))))) + +(def one-day (* 24 60 60)) + +(def discovery-topic-hash "0xf8946aac") + +(defn v42 + "Add all private chats to :discovery mailserver topic" + [old-realm new-realm] + (log/debug "migrating v40 account database") + (let [mailserver-topic (-> (.objects new-realm "mailserver-topic") + (.filtered (str "topic=\"" discovery-topic-hash "\"")) + (aget 0)) + old-chat-ids (edn/read-string (aget mailserver-topic "chat-ids")) + new-chats-ids (private-chats-ids (.objects old-realm "chat")) + all-chat-ids (clojure.set/union old-chat-ids new-chats-ids) + chat-ids-str (pr-str all-chat-ids)] + (when mailserver-topic + (aset mailserver-topic "chat-ids" chat-ids-str)))) + diff --git a/src/status_im/events.cljs b/src/status_im/events.cljs index 457502da76..4c5f0b07fa 100644 --- a/src/status_im/events.cljs +++ b/src/status_im/events.cljs @@ -121,6 +121,7 @@ (accounts.login/login) (node/initialize (get-in db [:accounts/login :address]))) (init/initialize-account address) + (mailserver/initialize-ranges) (chat-loading/initialize-chats {:to 10})))) (handlers/register-handler-fx @@ -130,7 +131,8 @@ (re-frame/inject-cofx :data-store/get-all-installations) (re-frame/inject-cofx :data-store/all-browsers) (re-frame/inject-cofx :data-store/all-dapp-permissions) - (re-frame/inject-cofx :data-store/all-chats)] + (re-frame/inject-cofx :data-store/all-chats) + (re-frame/inject-cofx :data-store/all-chat-requests-ranges)] account-change-success) (handlers/register-handler-fx @@ -723,20 +725,38 @@ :chat.ui/fetch-history-pressed (fn [{:keys [now] :as cofx} [_ chat-id]] (mailserver/fetch-history cofx chat-id - {:from (- (quot now 1000) (* 24 3600))}))) + {:from (- (quot now 1000) mailserver/one-day)}))) (handlers/register-handler-fx - :chat.ui/fill-the-gap - (fn [{:keys [db] :as cofx}] - (let [mailserver-topics (:mailserver/topics db) - chat-id (:current-chat-id db) + :chat.ui/fetch-history-pressed48-60 + (fn [{:keys [now] :as cofx} [_ chat-id]] + (let [now (quot now 1000)] + (mailserver/fetch-history cofx chat-id + {:from (- now (* 2.5 mailserver/one-day)) + :to (- now (* 2 mailserver/one-day))})))) + +(handlers/register-handler-fx + :chat.ui/fetch-history-pressed84-96 + (fn [{:keys [now] :as cofx} [_ chat-id]] + (let [now (quot now 1000)] + (mailserver/fetch-history cofx chat-id + {:from (- now (* 4 mailserver/one-day)) + :to (- now (* 3.5 mailserver/one-day))})))) + +(handlers/register-handler-fx + :chat.ui/fill-gaps + (fn [{:keys [db] :as cofx} [_ gap-ids]] + (let [chat-id (:current-chat-id db) topic (chat.db/topic-by-current-chat db) - gap (chat.db/messages-gap mailserver-topics topic)] + gaps (keep + (fn [id] + (get-in db [:mailserver/gaps chat-id id])) + gap-ids)] (mailserver/fill-the-gap cofx - (assoc gap - :topic topic - :chat-id chat-id))))) + {:gaps gaps + :topic topic + :chat-id chat-id})))) (handlers/register-handler-fx :chat.ui/remove-chat-pressed @@ -777,7 +797,8 @@ :chat.ui/load-more-messages [(re-frame/inject-cofx :data-store/get-messages) (re-frame/inject-cofx :data-store/get-user-statuses) - (re-frame/inject-cofx :data-store/get-referenced-messages)] + (re-frame/inject-cofx :data-store/get-referenced-messages) + (re-frame/inject-cofx :data-store/all-gaps)] (fn [cofx _] (chat.loading/load-more-messages cofx))) diff --git a/src/status_im/group_chats/core.cljs b/src/status_im/group_chats/core.cljs index 4182e89fff..bd7b5ef9ac 100644 --- a/src/status_im/group_chats/core.cljs +++ b/src/status_im/group_chats/core.cljs @@ -12,14 +12,13 @@ [status-im.group-chats.db :as group-chats.db] [status-im.i18n :as i18n] [status-im.native-module.core :as native-module] - [status-im.transport.chat.core :as transport.chat] [status-im.transport.message.group-chat :as message.group-chat] [status-im.transport.message.protocol :as protocol] - [status-im.transport.message.public-chat :as transport.public-chat] [status-im.transport.partitioned-topic :as transport.topic] [status-im.utils.clocks :as utils.clocks] - [status-im.utils.config :as config] - [status-im.utils.fx :as fx])) + [status-im.utils.fx :as fx] + [status-im.mailserver.core :as mailserver] + [taoensso.timbre :as log])) ;; Description of the flow: ;; the flow is complicated a bit by 2 asynchronous call to status-go, which might make the logic a bit more opaque. @@ -213,10 +212,17 @@ events [create-event (members-added-event clock-value selected-contacts)]] - {:group-chats/sign-membership {:chat-id chat-id - :from my-public-key - :events events} - :db (assoc db :group/selected-contacts #{})})) + (fx/merge + cofx + {:group-chats/sign-membership {:chat-id chat-id + :from my-public-key + :events events} + :db (assoc db :group/selected-contacts #{})} + (mailserver/upsert-mailserver-topic + {:chat-ids [chat-id] + :topic transport.topic/discovery-topic-hash + :fetch? false}) + (mailserver/process-next-messages-request)))) (fx/defn remove-member "Format group update message and sign membership" @@ -236,7 +242,7 @@ (fx/defn join-chat "Format group update message and sign membership" - [{:keys [db] :as cofx} chat-id] + [cofx chat-id] (let [my-public-key (accounts.db/current-public-key cofx) last-clock-value (get-last-clock-value cofx chat-id) chat (get-in cofx [:db :chats chat-id]) @@ -244,9 +250,16 @@ (when (valid-event? chat (assoc event :from my-public-key)) - {:group-chats/sign-membership {:chat-id chat-id - :from my-public-key - :events [event]}}))) + (fx/merge + cofx + {:group-chats/sign-membership {:chat-id chat-id + :from my-public-key + :events [event]}} + (mailserver/upsert-mailserver-topic + {:chat-ids [chat-id] + :topic transport.topic/discovery-topic-hash + :fetch false}) + (mailserver/process-next-messages-request))))) (fx/defn make-admin "Format group update with make admin message and sign membership" diff --git a/src/status_im/mailserver/core.cljs b/src/status_im/mailserver/core.cljs index 80e152a799..f66fbaa423 100644 --- a/src/status_im/mailserver/core.cljs +++ b/src/status_im/mailserver/core.cljs @@ -18,7 +18,8 @@ [status-im.accounts.update.core :as accounts.update] [status-im.ui.screens.navigation :as navigation] [status-im.transport.partitioned-topic :as transport.topic] - [status-im.ui.screens.mobile-network-settings.utils :as mobile-network-utils])) + [status-im.ui.screens.mobile-network-settings.utils :as mobile-network-utils] + [status-im.utils.random :as rand])) ;; How do mailserver work ? ;; @@ -35,6 +36,7 @@ (def one-day (* 24 3600)) (def seven-days (* 7 one-day)) +(def max-gaps-range (* 30 one-day)) (def max-request-range one-day) (def maximum-number-of-attempts 2) (def request-timeout 30) @@ -249,7 +251,8 @@ (fx/merge cofx {:db (cond-> (dissoc db :mailserver/current-request) gap-request? - (assoc :mailserver/fetching-gaps-in-progress {}))} + (-> (assoc :mailserver/fetching-gaps-in-progress {}) + (dissoc :mailserver/planned-gap-requests)))} (if added? (mark-trusted-peer) (add-peer))))) @@ -371,25 +374,32 @@ :force-to? (not (nil? force-request-to))}))))) (defn aggregate-requests - [acc {:keys [topic from to force-to?]}] - (update acc [from to force-to?] - (fn [{:keys [topics]}] - {:topics ((fnil conj #{}) topics topic) - :from from - :to to - ;; To is sent to the mailserver only when force-to? is true, - ;; also we use to calculate when the last-request was sent. - :force-to? force-to?}))) + [acc {:keys [topic from to force-to? gap chat-id]}] + (when from + (update acc [from to force-to?] + (fn [{:keys [topics]}] + {:topics ((fnil conj #{}) topics topic) + :from from + :to to + ;; To is sent to the mailserver only when force-to? is true, + ;; also we use to calculate when the last-request was sent. + :force-to? force-to? + :gap gap + :chat-id chat-id})))) (defn prepare-messages-requests [{{:keys [:mailserver/requests-from :mailserver/requests-to - :mailserver/topics]} :db} + :mailserver/topics + :mailserver/planned-gap-requests]} :db} default-request-to] (transduce (keep (topic->request default-request-to requests-from requests-to)) (completing aggregate-requests vals) - {} + (reduce + aggregate-requests + {} + (vals planned-gap-requests)) topics)) (fx/defn process-next-messages-request @@ -504,8 +514,11 @@ "if the chat is the only chat of the mailserver topic delete the mailserver topic and process-next-messages-requests again to remove pending request for that topic otherwise remove the chat-id of the chat from the mailserver topic and save" - [{:keys [db now] :as cofx} chat-id] - (let [topic (get-in db [:transport/chats chat-id :topic]) + [{:keys [db] :as cofx} chat-id] + (let [{:keys [public?] :as chat} (get-in db [:chats chat-id]) + topic (if (and chat (not public?)) + transport.topic/discovery-topic-hash + (get-in db [:transport/chats chat-id :topic])) {:keys [chat-ids] :as mailserver-topic} (update (get-in db [:mailserver/topics topic]) :chat-ids disj chat-id)] @@ -519,99 +532,222 @@ {:topic topic :mailserver-topic mailserver-topic})]}))) -(defn calculate-gap - [{:keys [gap-from - gap-to - last-request] :as config} - {:keys [request-from - request-to]}] - (merge config - (cond - (nil? gap-from) - {:gap-from request-to - :gap-to request-to - :last-request request-to} +(fx/defn remove-gaps + [{:keys [db]} chat-id] + {:db (update db :mailserver/gaps dissoc chat-id) + :data-store/tx [(data-store.mailservers/delete-all-gaps-by-chat chat-id)]}) - ;;------GF GT--------LRT F---T - (> request-from last-request) - {:gap-from last-request - :gap-to request-from - :last-request request-to} +(fx/defn remove-range + [{:keys [db]} chat-id] + {:db (update db :mailserver/ranges dissoc chat-id) + :data-store/tx [(data-store.mailservers/delete-range chat-id)]}) - ;;------GF GT--------LRT - ;; F----------T - (and (>= last-request request-from gap-to) - (> request-to last-request)) - {:last-request request-to} +(defn update-mailserver-topic + [{:keys [last-request] :as config} + {:keys [request-to]}] + (cond-> config + (> request-to last-request) + (assoc :last-request request-to))) - ;;------GF GT--------LRT - ;; F----T - (and (>= last-request request-from gap-to) - (>= last-request request-to gap-to)) - config +(defn check-existing-gaps + [chat-id chat-gaps request] + (let [request-from (:from request) + request-to (:to request)] + (reduce + (fn [acc {:keys [from to id] :as gap}] + (cond + ;; F----T + ;; RF---RT + (< to request-from) + acc - ;;------GF GT--------LRT - ;; F-------T - (and (> gap-to request-from gap-from) - (>= last-request request-to gap-to)) - {:gap-to request-from} + ;; F----T + ;; RF---RT + (< request-to from) + (reduced acc) - ;;------GF GT--------LRT - ;; F-T - (and (> gap-to request-from gap-from) - (> gap-to request-to gap-from)) - config + ;; F------T + ;; RF-----RT + (and (<= request-from from) + (< from request-to to)) + (let [updated-gap (assoc gap + :from request-to + :to to)] + (reduced + (update acc :updated-gaps assoc id updated-gap))) - ;;------GF GT--------LRT - ;; F------T - (and (>= gap-from request-from) - (> gap-to request-to gap-from)) - {:gap-from request-to} + ;; F------T + ;; RF----------RT + (and (<= request-from from) + (<= to request-to)) + (update acc :deleted-gaps conj (:id gap)) - ;;---------GF=GT=LRT - ;; F---T - (and (>= gap-from request-from) - (>= gap-from request-to) - (= gap-from last-request)) - {:gap-from request-to} + ;; F---------T + ;; RF-------RT + (and (< from request-from to) + (<= to request-to)) + (let [updated-gap (assoc gap + :from from + :to request-from)] + (update acc :updated-gaps assoc id updated-gap)) - ;;------GF GT--------LRT - ;; F---T - (and (>= gap-from request-from) - (>= gap-from request-to)) - config + ;; F---------T + ;; RF---RT + (and (< from request-from) + (< request-to to)) + (reduced + (-> acc + (update :deleted-gaps conj (:id gap)) + (update :new-gaps concat [{:chat-id chat-id + :from from + :to request-from} + {:chat-id chat-id + :from request-to + :to to}]))) - ;;------GF GT--------LRT - ;; F-------------T - (and (>= gap-from request-from) - (>= last-request request-to gap-to)) - {:gap-from last-request - :gap-to last-request - :last-request last-request} + :else acc)) + {} + (sort-by :from (vals chat-gaps))))) - ;;------GF GT--------LRT - ;; F------------------------T - (and (>= gap-from request-from) - (>= request-to last-request)) - {:gap-from request-to - :gap-to request-to - :last-request request-to} +(defn check-all-gaps + [gaps chat-ids request] + (transduce + (map (fn [chat-id] + (let [chat-gaps (get gaps chat-id)] + [chat-id (check-existing-gaps chat-id chat-gaps request)]))) + (completing + (fn [acc [chat-id {:keys [new-gaps updated-gaps deleted-gaps]}]] + (cond-> acc + (seq new-gaps) + (assoc-in [:new-gaps chat-id] new-gaps) - ;;------GF GT--------LRT - ;; F-----------------T - (and (> gap-to request-from gap-from) - (>= request-to last-request)) - {:gap-to request-from - :last-request request-to}))) + (seq updated-gaps) + (assoc-in [:updated-gaps chat-id] updated-gaps) + + (seq deleted-gaps) + (assoc-in [:deleted-gaps chat-id] deleted-gaps)))) + {} + chat-ids)) + +(fx/defn update-ranges + [{:keys [db] :as cofx}] + (let [{:keys [topics from to]} + (get db :mailserver/current-request) + chat-ids (mapcat + :chat-ids + (-> (:mailserver/topics db) + (select-keys topics) + vals)) + ranges (:mailserver/ranges db) + updated-ranges (into + {} + (keep + (fn [chat-id] + (let [chat-id (str chat-id) + {:keys [lowest-request-from + highest-request-to] + :as range} + (get ranges chat-id)] + [chat-id + (cond-> (assoc range :chat-id chat-id) + (or (nil? highest-request-to) + (> to highest-request-to)) + (assoc :highest-request-to to) + + (or (nil? lowest-request-from) + (< from lowest-request-from)) + (assoc :lowest-request-from from))]))) + chat-ids)] + (fx/merge + cofx + {:db (update db :mailserver/ranges merge updated-ranges) + :data-store/tx (map data-store.mailservers/save-chat-requests-range + (vals updated-ranges))}))) + +(defn prepare-new-gaps [new-gaps ranges {:keys [from to]} chat-ids] + (into + {} + (comp + (map (fn [chat-id] + (let [gaps (get new-gaps chat-id) + {:keys [highest-request-to lowest-request-from]} + (get ranges chat-id)] + [chat-id (cond-> gaps + (and + (not (nil? highest-request-to)) + (< highest-request-to from)) + (conj {:chat-id chat-id + :from highest-request-to + :to from}) + (and + (not (nil? lowest-request-from)) + (< to lowest-request-from)) + (conj {:chat-id chat-id + :from to + :to lowest-request-from}))]))) + (keep (fn [[chat-id gaps]] + [chat-id + (into {} + (map (fn [gap] + (let [id (rand/guid)] + [id (assoc gap :id id)]))) + gaps)]))) + chat-ids)) + +(fx/defn update-gaps + [{:keys [db]}] + (let [{:keys [topics] :as request} (get db :mailserver/current-request) + chat-ids (into #{} + (comp + (keep #(get-in db [:mailserver/topics %])) + (mapcat :chat-ids) + (map str)) + topics) + + {:keys [updated-gaps new-gaps deleted-gaps]} + (check-all-gaps (get db :mailserver/gaps) chat-ids request) + + ranges (:mailserver/ranges db) + prepared-new-gaps (prepare-new-gaps new-gaps ranges request chat-ids)] + {:db + (reduce (fn [db chat-id] + (let [chats-deleted-gaps (get deleted-gaps chat-id) + chats-updated-gaps (merge (get updated-gaps chat-id) + (get prepared-new-gaps chat-id))] + (update-in db [:mailserver/gaps chat-id] + (fn [chat-gaps] + (-> (apply dissoc chat-gaps chats-deleted-gaps) + (merge chats-updated-gaps)))))) + db + chat-ids) + + :data-store/tx + (conj + (map + data-store.mailservers/save-mailserver-requests-gap + (concat (mapcat vals (vals updated-gaps)) + (mapcat vals (vals prepared-new-gaps)))) + (data-store.mailservers/delete-mailserver-requests-gaps + (mapcat val deleted-gaps)))})) + +(fx/defn update-chats-and-gaps + [cofx cursor] + (when (or (nil? cursor) + (and (string? cursor) + (clojure.string/blank? cursor))) + (fx/merge + cofx + (update-gaps) + (update-ranges)))) (defn get-updated-mailserver-topics [db requested-topics from to] (into {} (keep (fn [topic] (when-let [config (get-in db [:mailserver/topics topic])] - [topic (calculate-gap config - {:request-from from - :request-to to})]))) + [topic (update-mailserver-topic config + {:request-from from + :request-to to})]))) requested-topics)) (fx/defn update-mailserver-topics @@ -632,29 +768,32 @@ (if (seq cursor) (when-let [mailserver (get-mailserver-when-ready cofx)] (let [request-with-cursor (assoc request :cursor cursor)] - {:db (assoc db :mailserver/current-request request-with-cursor) - :mailserver/request-messages {:web3 (:web3 db) - :mailserver mailserver - :request request-with-cursor}})) - (fx/merge cofx - {:db (-> db - (dissoc :mailserver/current-request) - (update :mailserver/requests-from - #(apply dissoc % topics)) - (update :mailserver/requests-to - #(apply dissoc % topics)) - (update :mailserver/topics merge mailserver-topics) - (update :mailserver/fetching-gaps-in-progress - (fn [gaps] - (if (executing-gap-request? db) - (dissoc gaps (:chat-id request)) - gaps)))) - :data-store/tx (mapv (fn [[topic mailserver-topic]] - (data-store.mailservers/save-mailserver-topic-tx - {:topic topic - :mailserver-topic mailserver-topic})) - mailserver-topics)} - (process-next-messages-request))))))) + {:db (assoc db :mailserver/current-request request-with-cursor) + :mailserver/request-messages {:web3 (:web3 db) + :mailserver mailserver + :request request-with-cursor}})) + (let [{:keys [gap chat-id]} request] + (fx/merge cofx + {:db (-> db + (dissoc :mailserver/current-request) + (update :mailserver/requests-from + #(apply dissoc % topics)) + (update :mailserver/requests-to + #(apply dissoc % topics)) + (update :mailserver/topics merge mailserver-topics) + (update :mailserver/fetching-gaps-in-progress + (fn [gaps] + (if gap + (update gaps chat-id dissoc gap) + gaps))) + (update :mailserver/planned-gap-requests + dissoc gap)) + :data-store/tx (mapv (fn [[topic mailserver-topic]] + (data-store.mailservers/save-mailserver-topic-tx + {:topic topic + :mailserver-topic mailserver-topic})) + mailserver-topics)} + (process-next-messages-request)))))))) (fx/defn retry-next-messages-request [{:keys [db] :as cofx}] @@ -689,6 +828,7 @@ :dispatch-n (map #(identity [:chat.ui/join-time-messages-checked %]) never-synced-chats-in-request)} + (update-chats-and-gaps cursor) (update-mailserver-topics {:request-id requestID :cursor cursor})) (fx/merge @@ -700,11 +840,13 @@ {:ms 1000 :dispatch [:chat.ui/join-time-messages-checked %]}) never-synced-chats-in-request))} + (update-chats-and-gaps cursor) (update-mailserver-topics {:request-id requestID :cursor cursor}))) (fx/merge cofx {:mailserver/increase-limit []} + (update-chats-and-gaps cursor) (update-mailserver-topics {:request-id requestID :cursor cursor})))) (handle-request-error cofx errorMessage)))) @@ -725,24 +867,29 @@ add the chat-id to the topic and reset last-request there was no filter for the chat and messages for that so the whole history for that topic needs to be re-fetched" - [{:keys [db now] :as cofx} {:keys [topic chat-id]}] - (let [{:keys [chat-ids] :as current-mailserver-topic} - (get-in db [:mailserver/topics topic] {:chat-ids #{}})] - (when-not (contains? chat-ids chat-id) + [{:keys [db now] :as cofx} {:keys [topic chat-ids fetch?] + :or {fetch? true}}] + (let [current-mailserver-topic (get-in db [:mailserver/topics topic] + {:chat-ids #{}}) + existing-ids (:chat-ids current-mailserver-topic) + chat-id (first chat-ids)] + (when-not (every? (partial contains? existing-ids) chat-ids) (let [{:keys [new-account? public-key]} (:account/account db) now-s (quot now 1000) - last-request (if (and new-account? - (or (= chat-id :discovery-topic) - (and - (string? chat-id) - (string/starts-with? - chat-id - public-key)))) + last-request (if (or + (not fetch?) + (and new-account? + (or (= chat-id :discovery-topic) + (and + (string? chat-id) + (string/starts-with? + chat-id + public-key))))) (- now-s 10) (- now-s max-request-range)) mailserver-topic (-> current-mailserver-topic (assoc :last-request last-request) - (update :chat-ids conj chat-id))] + (update :chat-ids clojure.set/union (set chat-ids)))] (fx/merge cofx {:db (assoc-in db [:mailserver/topics topic] mailserver-topic) :data-store/tx [(data-store.mailservers/save-mailserver-topic-tx @@ -751,7 +898,6 @@ (fx/defn fetch-history [{:keys [db] :as cofx} chat-id {:keys [from to]}] - (log/debug "fetch-history" "chat-id:" chat-id "from-timestamp:" from) (let [public-key (accounts.db/current-public-key cofx) topic (or (get-in db [:transport/chats chat-id :topic]) @@ -764,28 +910,33 @@ (process-next-messages-request)))) (fx/defn fill-the-gap - [{:keys [db] :as cofx} {:keys [exists? from to topic chat-id]}] - (let [mailserver (get-mailserver-when-ready cofx) - request {:from from - :to to - :force-to? true - :topics [topic] - :chat-id chat-id}] - (when exists? - {:db - (-> db - (assoc - :mailserver/pending-requests 1 - :mailserver/current-request request - :mailserver/request-to to) - - (update :mailserver/fetching-gaps-in-progress - assoc chat-id request)) - - :mailserver/request-messages - {:web3 (:web3 db) - :mailserver mailserver - :request request}}))) + [{:keys [db] :as cofx} {:keys [gaps topic chat-id]}] + (let [mailserver (get-mailserver-when-ready cofx) + requests (into {} + (map + (fn [{:keys [from to id]}] + [id + {:from (max from + (- to max-request-range)) + :to to + :force-to? true + :topics [topic] + :topic topic + :chat-id chat-id + :gap id}])) + gaps) + first-request (val (first requests)) + current-request (:mailserver/current-request db)] + (cond-> {:db (-> db + (assoc :mailserver/planned-gap-requests requests) + (update :mailserver/fetching-gaps-in-progress + assoc chat-id requests))} + (not current-request) + (-> (assoc-in [:db :mailserver/current-request] first-request) + (assoc :mailserver/request-messages + {:web3 (:web3 db) + :mailserver mailserver + :request first-request}))))) (fx/defn resend-request [{:keys [db] :as cofx} {:keys [request-id]}] @@ -814,7 +965,8 @@ {:db (-> db (dissoc :mailserver/current-request) (update :mailserver/fetching-gaps-in-progress - dissoc (:chat-id current-request)))} + dissoc (:chat-id current-request)) + (dissoc :mailserver/planned-gap-requests))} mailserver (let [{:keys [topics from to cursor limit] :as request} current-request @@ -981,3 +1133,25 @@ (fx/merge cofx (accounts.update/update-settings (assoc-in settings [:mailserver current-fleet] mailserver-id) {})))) + +(fx/defn initialize-ranges + [{:keys [:data-store/all-chat-requests-ranges db]}] + {:db (assoc db :mailserver/ranges all-chat-requests-ranges)}) + +(fx/defn load-gaps + [{:keys [db now :data-store/all-gaps]} chat-id] + (when-not (get-in db [:chats chat-id :gaps-loaded?]) + (let [now-s (quot now 1000) + gaps (all-gaps chat-id) + outdated-gaps (into [] (comp (filter #(< (:to %) + (- now-s max-gaps-range))) + (map :id)) + (vals gaps)) + gaps (apply dissoc gaps outdated-gaps)] + {:db + (-> db + (assoc-in [:chats chat-id :gaps-loaded?] true) + (assoc-in [:mailserver/gaps chat-id] gaps)) + :data-store/tx + [(data-store.mailservers/delete-mailserver-requests-gaps + outdated-gaps)]}))) diff --git a/src/status_im/transport/chat/core.cljs b/src/status_im/transport/chat/core.cljs index 7dc92f6680..22424e050e 100644 --- a/src/status_im/transport/chat/core.cljs +++ b/src/status_im/transport/chat/core.cljs @@ -15,6 +15,4 @@ (fx/defn unsubscribe-from-chat "Unsubscribe from chat on transport layer" [cofx chat-id] - (fx/merge cofx - (mailserver/remove-chat-from-mailserver-topic chat-id) - (remove-transport-chat chat-id))) + (remove-transport-chat cofx chat-id)) diff --git a/src/status_im/transport/filters.cljs b/src/status_im/transport/filters.cljs index 1dd69af0bf..0c33d74385 100644 --- a/src/status_im/transport/filters.cljs +++ b/src/status_im/transport/filters.cljs @@ -84,8 +84,8 @@ (mapcat (fn [{:keys [topic chat-id filter]}] [(add-filter chat-id filter) - (mailserver/upsert-mailserver-topic {:topic topic - :chat-id chat-id})]) + (mailserver/upsert-mailserver-topic {:topic topic + :chat-ids [chat-id]})]) filters)] (apply fx/merge cofx {:db (dissoc db :filters/after-adding-discovery-filter)} diff --git a/src/status_im/ui/screens/chat/actions.cljs b/src/status_im/ui/screens/chat/actions.cljs index 03a769c6c5..109b1b97c4 100644 --- a/src/status_im/ui/screens/chat/actions.cljs +++ b/src/status_im/ui/screens/chat/actions.cljs @@ -26,6 +26,14 @@ {:label (i18n/label :t/fetch-history) :action #(re-frame/dispatch [:chat.ui/fetch-history-pressed chat-id])}) +(defn fetch-history48-60 [chat-id] + {:label "Fetch 48-60h" + :action #(re-frame/dispatch [:chat.ui/fetch-history-pressed48-60 chat-id])}) + +(defn fetch-history84-96 [chat-id] + {:label "Fetch 84-96h" + :action #(re-frame/dispatch [:chat.ui/fetch-history-pressed84-96 chat-id])}) + (defn- delete-chat [chat-id group?] {:label (i18n/label :t/delete-chat) :action #(re-frame/dispatch [(if group? @@ -37,18 +45,24 @@ [(view-profile chat-id) (clear-history) (fetch-history chat-id) + #_(fetch-history48-60 chat-id) + #_(fetch-history84-96 chat-id) (delete-chat chat-id false)]) (defn- group-chat-actions [chat-id] [(group-info chat-id) (clear-history) (fetch-history chat-id) + #_(fetch-history48-60 chat-id) + #_(fetch-history84-96 chat-id) (delete-chat chat-id true)]) (defn- public-chat-actions [chat-id] [(share-chat chat-id) (clear-history) (fetch-history chat-id) + #_(fetch-history48-60 chat-id) + #_(fetch-history84-96 chat-id) (delete-chat chat-id false)]) (defn actions [group-chat? chat-id public?] diff --git a/src/status_im/ui/screens/chat/views.cljs b/src/status_im/ui/screens/chat/views.cljs index bfff44c816..51b00eecd4 100644 --- a/src/status_im/ui/screens/chat/views.cljs +++ b/src/status_im/ui/screens/chat/views.cljs @@ -70,8 +70,8 @@ [{{:keys [value]} :row}] [message-datemark/chat-datemark-mobile value]) -(defview gap [] - (letsubs [in-progress? [:chats/fetching-gap-in-progress?] +(defview gap [{:keys [ids]}] + (letsubs [in-progress? [:chats/fetching-gap-in-progress? ids] connected? [:mailserver/connected?]] [react/view {:align-self :stretch :margin-top 24 @@ -85,7 +85,7 @@ :background-color :white} [react/touchable-highlight {:on-press (when (and connected? (not in-progress?)) - #(re-frame/dispatch [:chat.ui/fill-the-gap]))} + #(re-frame/dispatch [:chat.ui/fill-gaps ids]))} [react/view {:flex 1 :align-items :center :justify-content :center} @@ -98,8 +98,8 @@ (i18n/label :t/fetch-messages)])]]])) (defmethod message-row :gap - [_] - [gap]) + [{:keys [row]}] + [gap (:gaps row)]) (defmethod message-row :default [{:keys [group-chat current-public-key modal? row]}] diff --git a/test/cljs/status_im/test/chat/db.cljs b/test/cljs/status_im/test/chat/db.cljs index 8524af01d9..89395aa3b5 100644 --- a/test/cljs/status_im/test/chat/db.cljs +++ b/test/cljs/status_im/test/chat/db.cljs @@ -217,8 +217,9 @@ :type :datemark :whisper-timestamp 30 :timestamp 30} - {:type :gap - :value "25"} + {:type :gap + :value ":gapid1" + :gaps {:ids [:gapid1]}} {:whisper-timestamp 20 :timestamp 20 :content nil @@ -266,12 +267,14 @@ :timestamp 40}} nil nil - {:from 25 - :exists? true})))) + [{:from 25 + :to 30 + :id :gapid1}])))) (testing "simple case with gap after all messages" (is (= '({:type :gap - :value "100"} + :value ":gapid1" + :gaps {:ids (:gapid1)}} {:whisper-timestamp 40 :timestamp 40 :content nil @@ -335,5 +338,6 @@ :timestamp 40}} nil nil - {:from 100 - :exists? true}))))) + [{:from 100 + :to 110 + :id :gapid1}]))))) diff --git a/test/cljs/status_im/test/chat/models.cljs b/test/cljs/status_im/test/chat/models.cljs index 0f37a1c83b..2f3314183d 100644 --- a/test/cljs/status_im/test/chat/models.cljs +++ b/test/cljs/status_im/test/chat/models.cljs @@ -168,7 +168,7 @@ (testing "it adds the relevant transactions for realm" (let [actual (chat/remove-chat cofx chat-id)] (is (:data-store/tx actual)) - (is (= 5 (count (:data-store/tx actual)))))))) + (is (= 8 (count (:data-store/tx actual)))))))) (deftest multi-user-chat? (let [chat-id "1"] diff --git a/test/cljs/status_im/test/mailserver/core.cljs b/test/cljs/status_im/test/mailserver/core.cljs index 83b6d05f3b..cf00d218cd 100644 --- a/test/cljs/status_im/test/mailserver/core.cljs +++ b/test/cljs/status_im/test/mailserver/core.cljs @@ -1,7 +1,8 @@ (ns status-im.test.mailserver.core (:require [cljs.test :refer-macros [deftest is testing]] [status-im.transport.utils :as utils] - [status-im.mailserver.core :as mailserver])) + [status-im.mailserver.core :as mailserver] + [status-im.utils.random :as rand])) (def enode "enode://08d8eb6177b187049f6c97ed3f6c74fbbefb94c7ad10bafcaf4b65ce89c314dcfee0a8bc4e7a5b824dfa08b45b360cc78f34f0aff981f8386caa07652d2e601b@163.172.177.138:40404") (def enode2 "enode://12d8eb6177b187049f6c97ed3f6c74fbbefb94c7ad10bafcaf4b65ce89c314dcfee0a8bc4e7a5b824dfa08b45b360cc78f34f0aff981f8386caa07652d2e601b@163.172.177.138:40404") @@ -628,186 +629,244 @@ :shh/generate-sym-key-from-password first))))))) -(deftest calculate-gap - (testing "new topic" - (is (= {:gap-from 10 - :gap-to 10 - :last-request 10} +(deftest check-existing-gaps + (let [] + (testing "no gaps" + (is (= {} + (mailserver/check-existing-gaps + :chat-id + nil + {:from 1 + :to 2})))) + (testing "request before gaps" + (is (= {} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 1 + :to 2})))) + (testing "request between gaps" + (is (= {} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 22 + :to 28})))) + (testing "request between gaps" + (is (= {} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 22 + :to 28})))) + (testing "request after gaps" + (is (= {} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 70 + :to 80})))) + (testing "request covers all gaps" + (is (= {:deleted-gaps [:g3 :g2 :g1]} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 10 + :to 60})))) + (testing "request splits gap in two" + (is (= {:deleted-gaps [:g1] + :new-gaps [{:chat-id :chat-id :from 10 :to 12} + {:chat-id :chat-id :from 18 :to 20}]} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 12 + :to 18})))) + (testing "request partially covers one gap #1" + (is (= {:updated-gaps {:g1 {:from 15 + :to 20 + :id :g1}}} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 8 + :to 15})))) + (testing "request partially covers one gap #2" + (is (= {:updated-gaps {:g1 {:from 10 + :to 15 + :id :g1}}} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 15 + :to 25})))) + (testing "request partially covers two gaps #2" + (is (= {:updated-gaps {:g1 {:from 10 + :to 15 + :id :g1} + :g2 {:from 35 + :to 40 + :id :g2}}} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 15 + :to 35})))) + (testing "request covers one gap and two other partially" + (is (= {:updated-gaps {:g1 {:from 10 + :to 15 + :id :g1} + :g3 {:from 55 + :to 60 + :id :g3}} + :deleted-gaps [:g2]} + (mailserver/check-existing-gaps + :chat-id + {:g1 {:from 10 + :to 20 + :id :g1} + :g2 {:from 30 + :to 40 + :id :g2} + :g3 {:from 50 + :to 60 + :id :g3}} + {:from 15 + :to 55})))))) - (mailserver/calculate-gap - {:gap-from nil - :gap-to nil - :last-request nil} - {:request-from 5 - :request-to 10})))) - (testing "calculate-gap#1" - (is (= {:gap-from 3 - :gap-to 4 - :last-request 5} +(defn rand-guid [] + (let [gap-id (atom 0)] + (fn [] + (swap! gap-id inc) + (str "gap" @gap-id)))) - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 3} - {:request-from 4 - :request-to 5})))) - (testing "calculate-gap#2" - (is (= {:gap-from 1 - :gap-to 2 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 4} - {:request-from 3 - :request-to 5})))) - (testing "calculate-gap#2-1" - (is (= {:gap-from 1 - :gap-to 2 - :last-request 4} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 3} - {:request-from 3 - :request-to 4})))) - (testing "calculate-gap#3" - (is (= {:gap-from 1 - :gap-to 2 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 5} - {:request-from 3 - :request-to 4})))) - (testing "calculate-gap#3-1" - (is (= {:gap-from 1 - :gap-to 2 - :last-request 3} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 3} - {:request-from 2 - :request-to 3})))) - (testing "calculate-gap#4" - (is (= {:gap-from 1 - :gap-to 2 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 5} - {:request-from 3 - :request-to 4})))) - (testing "calculate-gap#5" - (is (= {:gap-from 1 - :gap-to 4 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 4 - :last-request 5} - {:request-from 2 - :request-to 3})))) - (testing "calculate-gap#6" - (is (= {:gap-from 2 - :gap-to 3 - :last-request 4} - (mailserver/calculate-gap - {:gap-from 2 - :gap-to 3 - :last-request 4} - {:request-from 1 - :request-to 2})))) - (testing "calculate-gap#6-1" - (is (= {:gap-from 1 - :gap-to 4 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 4 - :last-request 5} - {:request-from 2 - :request-to 3})))) - (testing "calculate-gap#0" - (is (= {:gap-from 2 - :gap-to 3 - :last-request 3} - (mailserver/calculate-gap - {:gap-from 3 - :gap-to 3 - :last-request 3} - {:request-from 1 - :request-to 2})))) - (testing "calculate-gap#7" - (is (= {:gap-from 3 - :gap-to 4 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 3 - :gap-to 4 - :last-request 5} - {:request-from 1 - :request-to 2})))) - (testing "calculate-gap#8" - (is (= {:gap-from 5 - :gap-to 5 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 2 - :gap-to 3 - :last-request 5} - {:request-from 1 - :request-to 4})))) - (testing "calculate-gap#8-1" - (is (= {:gap-from 3 - :gap-to 3 - :last-request 3} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 3} - {:request-from 1 - :request-to 2})))) - (testing "calculate-gap#9" - (is (= {:gap-from 5 - :gap-to 5 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 2 - :gap-to 3 - :last-request 4} - {:request-from 1 - :request-to 5})))) - (testing "calculate-gap#9-1" - (is (= {:gap-from 3 - :gap-to 3 - :last-request 3} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 3} - {:request-from 1 - :request-to 3})))) - (testing "calculate-gap#10" - (is (= {:gap-from 1 - :gap-to 2 - :last-request 5} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 3 - :last-request 4} - {:request-from 2 - :request-to 5})))) - (testing "calculate-gap#10-1" - (is (= {:gap-from 1 - :gap-to 2 - :last-request 3} - (mailserver/calculate-gap - {:gap-from 1 - :gap-to 2 - :last-request 3} - {:request-from 2 - :request-to 3}))))) +(deftest prepare-new-gaps + (testing "prepare-new-gaps" + (with-redefs [rand/guid (rand-guid)] + (is (= {"chat1" {"gap1" {:id "gap1" + :chat-id "chat1" + :from 20 + :to 30}}} + (mailserver/prepare-new-gaps + nil + {"chat1" + {:chat-id "chat1" + :lowest-request-from 10 + :highest-request-to 20}} + {:from 30 + :to 50} + #{"chat1"}))))) + (testing "prepare-new-gaps request after known range" + (with-redefs [rand/guid (rand-guid)] + (is (= {"chat1" {"gap1" {:id "gap1" + :chat-id "chat1" + :from 12 + :to 14} + "gap2" {:chat-id "chat1" + :from 20 + :to 30 + :id "gap2"}}} + (mailserver/prepare-new-gaps + {"chat1" [{:chat-id "chat1" + :from 12 + :to 14}]} + {"chat1" + {:chat-id "chat1" + :lowest-request-from 10 + :highest-request-to 20}} + {:from 30 + :to 50} + #{"chat1"}))))) + (testing "prepare-new-gaps request before known range" + (with-redefs [rand/guid (rand-guid)] + (is (= {"chat1" {"gap1" {:chat-id "chat1" + :from 12 + :to 14 + :id "gap1"} + "gap2" {:chat-id "chat1" + :from 8 + :to 10 + :id "gap2"}}} + (mailserver/prepare-new-gaps + {"chat1" [{:chat-id "chat1" + :from 12 + :to 14}]} + {"chat1" + {:chat-id "chat1" + :lowest-request-from 10 + :highest-request-to 20}} + {:from 2 + :to 8} + #{"chat1"}))))))