From 9a9cd0d8d020db9a45e47b820168da156db05e3f Mon Sep 17 00:00:00 2001 From: Roman Volosovskyi Date: Sun, 16 Dec 2018 10:19:00 +0200 Subject: [PATCH] [slow sign in] Denormalize last-clock-value In order to get `:last-clock-value` one extra query was executed for each chat during initialization. Implementation: - `:last-clock-value` field was added to `chat` entity - this field is updated when the message is sent/received - extra query was removed --- src/status_im/chat/models/message.cljs | 34 ++++++++--- src/status_im/data_store/chats.cljs | 11 +--- src/status_im/data_store/realm/core.cljs | 2 +- .../realm/schemas/account/chat.cljs | 5 ++ .../realm/schemas/account/core.cljs | 18 +++++- .../realm/schemas/account/migrations.cljs | 23 ++++++- .../cljs/status_im/test/data_store/chats.cljs | 60 +++++++++---------- 7 files changed, 100 insertions(+), 53 deletions(-) diff --git a/src/status_im/chat/models/message.cljs b/src/status_im/chat/models/message.cljs index 7ca339b705..4bbf5f7b06 100644 --- a/src/status_im/chat/models/message.cljs +++ b/src/status_im/chat/models/message.cljs @@ -109,7 +109,9 @@ :prioritary? (not (chat-model/multi-user-chat? cofx chat-id))})) (fx/defn add-message - [{:keys [db] :as cofx} batch? {:keys [chat-id message-id clock-value timestamp content from] :as message} current-chat?] + [{:keys [db] :as cofx} + {{:keys [chat-id message-id clock-value timestamp from] :as message} :message + :keys [current-chat? batch? last-clock-value]}] (let [current-public-key (accounts.db/current-public-key cofx) prepared-message (-> message (prepare-message chat-id current-chat?) @@ -124,8 +126,11 @@ {:db (cond-> (-> db (update-in [:chats chat-id :messages] assoc message-id prepared-message) - ;; this will increase last-clock-value twice when sending our own messages - (update-in [:chats chat-id :last-clock-value] (partial utils.clocks/receive clock-value))) + (update-in [:chats chat-id :last-clock-value] + (fn [old-clock-value] + (or last-clock-value + (utils.clocks/receive clock-value + old-clock-value))))) (and (not current-chat?) (not= from current-public-key)) @@ -193,7 +198,9 @@ (fx/merge cofx {:transport/confirm-messages-processed [{:web3 web3 :js-obj js-obj}]} - (add-message true message current-chat?) + (add-message {:batch? true + :message message + :current-chat current-chat?}) ;; Checking :outgoing here only works for now as we don't have a :seen ;; status for public chats, if we add processing of our own messages ;; for 1-to-1 care needs to be taken not to override the :seen status @@ -269,7 +276,7 @@ (defn- update-last-message [all-chats chat-id] (let [{:keys [messages message-groups]} (get all-chats chat-id) - {:keys [content message-type]} + {:keys [content message-type clock-value]} (->> (chat.db/sort-message-groups message-groups messages) first second @@ -279,7 +286,8 @@ (chat-model/upsert-chat {:chat-id chat-id :last-message-content content - :last-message-type message-type}))) + :last-message-type message-type + :last-clock-value clock-value}))) (fx/defn update-last-messages [{:keys [db] :as cofx} chat-ids] @@ -372,8 +380,12 @@ {:chat-id chat-id :timestamp now :last-message-content (:content message) - :last-message-type (:message-type message)}) - (add-message false message-with-id true) + :last-message-type (:message-type message) + :last-clock-value (:clock-value message)}) + (add-message {:batch? false + :message message-with-id + :current-chat? true + :last-clock-value (:clock-value message)}) (add-own-status chat-id message-id :sending) (send chat-id message-id wrapped-record)))) @@ -433,7 +445,11 @@ (remove-message-from-group chat-id (get-in db [:chats chat-id :messages message-id])))) (fx/defn add-system-messages [cofx messages] - (let [messages-fx (map #(add-message false (system-message cofx %) true) messages)] + (let [messages-fx (map #(add-message + {:batch false + :message (system-message cofx %) + :current-chat? true}) + messages)] (apply fx/merge cofx messages-fx))) (fx/defn send-message diff --git a/src/status_im/data_store/chats.cljs b/src/status_im/data_store/chats.cljs index b84f412fa6..27b3dbd761 100644 --- a/src/status_im/data_store/chats.cljs +++ b/src/status_im/data_store/chats.cljs @@ -47,22 +47,13 @@ :signature signature :chat-id chat-id})))) -(defn- get-last-clock-value [chat-id] - (-> (core/get-by-field @core/account-realm - :message :chat-id chat-id) - (core/sorted :clock-value :desc) - (core/single-clj :message) - :clock-value - (utils.clocks/safe-timestamp))) - (defn- normalize-chat [{:keys [chat-id] :as chat}] (-> chat (update :admins #(into #{} %)) (update :contacts #(into #{} %)) (update :tags #(into #{} %)) (update :membership-updates (partial unmarshal-membership-updates chat-id)) - ;; We cap the clock value to a safe value in case the db has been polluted - (assoc :last-clock-value (get-last-clock-value chat-id)) + (update :last-clock-value utils.clocks/safe-timestamp) (update :last-message-type keyword) (update :last-message-content edn/read-string))) diff --git a/src/status_im/data_store/realm/core.cljs b/src/status_im/data_store/realm/core.cljs index a9ddb57aa3..29ccb60665 100644 --- a/src/status_im/data_store/realm/core.cljs +++ b/src/status_im/data_store/realm/core.cljs @@ -149,7 +149,7 @@ (when (pos? current-version) (doseq [schema schemas :when (> (:schemaVersion schema) current-version)] - (migration-log :current-version current-version) + (migration-log :current-version (:schemaVersion schema)) (let [migrated-realm (open-realm schema file-name encryption-key)] (close migrated-realm)))) (open-realm (last schemas) file-name encryption-key)) diff --git a/src/status_im/data_store/realm/schemas/account/chat.cljs b/src/status_im/data_store/realm/schemas/account/chat.cljs index d917af0c6f..afa0eb6578 100644 --- a/src/status_im/data_store/realm/schemas/account/chat.cljs +++ b/src/status_im/data_store/realm/schemas/account/chat.cljs @@ -230,3 +230,8 @@ :optional true} :last-message-type {:type :string :optional true}})) + +(def v11 + (update v10 :properties merge + {:last-clock-value {:type :int + :optional true}})) diff --git a/src/status_im/data_store/realm/schemas/account/core.cljs b/src/status_im/data_store/realm/schemas/account/core.cljs index 09c2c95a4e..94cb7ef800 100644 --- a/src/status_im/data_store/realm/schemas/account/core.cljs +++ b/src/status_im/data_store/realm/schemas/account/core.cljs @@ -304,6 +304,19 @@ browser/v8 dapp-permissions/v9]) +(def v29 [chat/v11 + transport/v7 + contact/v3 + message/v9 + mailserver/v11 + mailserver-topic/v1 + user-status/v2 + membership-update/v1 + installation/v2 + local-storage/v1 + browser/v8 + dapp-permissions/v9]) + ;; put schemas ordered by version (def schemas [{:schema v1 :schemaVersion 1 @@ -388,4 +401,7 @@ :migration migrations/v27} {:schema v28 :schemaVersion 28 - :migration migrations/v28}]) + :migration migrations/v28} + {:schema v29 + :schemaVersion 29 + :migration migrations/v29}]) diff --git a/src/status_im/data_store/realm/schemas/account/migrations.cljs b/src/status_im/data_store/realm/schemas/account/migrations.cljs index 7e8f838c09..09f832e151 100644 --- a/src/status_im/data_store/realm/schemas/account/migrations.cljs +++ b/src/status_im/data_store/realm/schemas/account/migrations.cljs @@ -7,7 +7,8 @@ [clojure.string :as string] [status-im.constants :as constants] [cognitect.transit :as transit] - [status-im.js-dependencies :as dependencies])) + [status-im.js-dependencies :as dependencies] + [status-im.utils.clocks :as utils.clocks])) (defn v1 [old-realm new-realm] (log/debug "migrating v1 account database: " old-realm new-realm)) @@ -316,3 +317,23 @@ message-type (aget last-message "message-type")] (aset chat "last-message-content" content) (aset chat "last-message-type" message-type))))))) + +(defn get-last-clock-value [realm chat-id] + (if-let [last-message + (-> (.objects realm "message") + (.filtered (str "chat-id=\"" chat-id "\"")) + (.sorted "clock-value" true) + (aget 0))] + (-> + last-message + (aget "clock-value") + (utils.clocks/safe-timestamp)) + 0)) + +(defn v29 [old-realm new-realm] + (let [chats (.objects new-realm "chat")] + (dotimes [i (.-length chats)] + (let [chat (aget chats i) + chat-id (aget chat "chat-id")] + (when-let [last-clock-value (get-last-clock-value new-realm chat-id)] + (aset chat "last-clock-value" last-clock-value)))))) diff --git a/test/cljs/status_im/test/data_store/chats.cljs b/test/cljs/status_im/test/data_store/chats.cljs index 534e7b9743..a9fcfb50a1 100644 --- a/test/cljs/status_im/test/data_store/chats.cljs +++ b/test/cljs/status_im/test/data_store/chats.cljs @@ -5,39 +5,37 @@ (deftest normalize-chat-test (testing "admins & contacts" - (with-redefs [chats/get-last-clock-value (constantly 42)] - (is (= {:last-clock-value 42 - :admins #{4} - :contacts #{2} - :tags #{} - :membership-updates [] - :last-message-type :message-type - :last-message-content {:foo "bar"}} - (chats/normalize-chat - {:admins [4] - :contacts [2] - :last-message-type "message-type" - :last-message-content "{:foo \"bar\"}"}))))) + (is (= {:admins #{4} + :contacts #{2} + :tags #{} + :membership-updates [] + :last-message-type :message-type + :last-message-content {:foo "bar"} + :last-clock-value nil} + (chats/normalize-chat + {:admins [4] + :contacts [2] + :last-message-type "message-type" + :last-message-content "{:foo \"bar\"}"})))) (testing "membership-updates" - (with-redefs [chats/get-last-clock-value (constantly 42)] - (let [raw-events {"1" {:id "1" :type "members-added" :clock-value 10 :members [1 2] :signature "a" :from "id-1"} - "2" {:id "2" :type "member-removed" :clock-value 11 :member 1 :signature "a" :from "id-1"} - "3" {:id "3" :type "chat-created" :clock-value 0 :name "blah" :signature "b" :from "id-2"}} - expected #{{:chat-id "chat-id" - :from "id-2" - :signature "b" - :events [{:type "chat-created" :clock-value 0 :name "blah"}]} - {:chat-id "chat-id" - :signature "a" - :from "id-1" - :events [{:type "members-added" :clock-value 10 :members [1 2]} + (let [raw-events {"1" {:id "1" :type "members-added" :clock-value 10 :members [1 2] :signature "a" :from "id-1"} + "2" {:id "2" :type "member-removed" :clock-value 11 :member 1 :signature "a" :from "id-1"} + "3" {:id "3" :type "chat-created" :clock-value 0 :name "blah" :signature "b" :from "id-2"}} + expected #{{:chat-id "chat-id" + :from "id-2" + :signature "b" + :events [{:type "chat-created" :clock-value 0 :name "blah"}]} + {:chat-id "chat-id" + :signature "a" + :from "id-1" + :events [{:type "members-added" :clock-value 10 :members [1 2]} {:type "member-removed" :clock-value 11 :member 1}]}} - actual (->> (chats/normalize-chat {:chat-id "chat-id" - :membership-updates raw-events}) - :membership-updates - (into #{}))] - (is (= expected - actual)))))) + actual (->> (chats/normalize-chat {:chat-id "chat-id" + :membership-updates raw-events}) + :membership-updates + (into #{}))] + (is (= expected + actual))))) (deftest marshal-membership-updates-test (let [raw-updates [{:chat-id "chat-id"