[slow sign in] Denormalize last-clock-value

In order to get `:last-clock-value` one extra query was executed for
each chat during initialization.

Implementation:
- `:last-clock-value` field was added to `chat` entity
- this field is updated when the message is sent/received
- extra query was removed
This commit is contained in:
Roman Volosovskyi 2018-12-16 10:19:00 +02:00
parent 902dc3806c
commit 9a9cd0d8d0
No known key found for this signature in database
GPG Key ID: 0238A4B5ECEE70DE
7 changed files with 100 additions and 53 deletions

View File

@ -109,7 +109,9 @@
:prioritary? (not (chat-model/multi-user-chat? cofx chat-id))})) :prioritary? (not (chat-model/multi-user-chat? cofx chat-id))}))
(fx/defn add-message (fx/defn add-message
[{:keys [db] :as cofx} batch? {:keys [chat-id message-id clock-value timestamp content from] :as message} current-chat?] [{:keys [db] :as cofx}
{{:keys [chat-id message-id clock-value timestamp from] :as message} :message
:keys [current-chat? batch? last-clock-value]}]
(let [current-public-key (accounts.db/current-public-key cofx) (let [current-public-key (accounts.db/current-public-key cofx)
prepared-message (-> message prepared-message (-> message
(prepare-message chat-id current-chat?) (prepare-message chat-id current-chat?)
@ -124,8 +126,11 @@
{:db (cond-> {:db (cond->
(-> db (-> db
(update-in [:chats chat-id :messages] assoc message-id prepared-message) (update-in [:chats chat-id :messages] assoc message-id prepared-message)
;; this will increase last-clock-value twice when sending our own messages (update-in [:chats chat-id :last-clock-value]
(update-in [:chats chat-id :last-clock-value] (partial utils.clocks/receive clock-value))) (fn [old-clock-value]
(or last-clock-value
(utils.clocks/receive clock-value
old-clock-value)))))
(and (not current-chat?) (and (not current-chat?)
(not= from current-public-key)) (not= from current-public-key))
@ -193,7 +198,9 @@
(fx/merge cofx (fx/merge cofx
{:transport/confirm-messages-processed [{:web3 web3 {:transport/confirm-messages-processed [{:web3 web3
:js-obj js-obj}]} :js-obj js-obj}]}
(add-message true message current-chat?) (add-message {:batch? true
:message message
:current-chat current-chat?})
;; Checking :outgoing here only works for now as we don't have a :seen ;; Checking :outgoing here only works for now as we don't have a :seen
;; status for public chats, if we add processing of our own messages ;; status for public chats, if we add processing of our own messages
;; for 1-to-1 care needs to be taken not to override the :seen status ;; for 1-to-1 care needs to be taken not to override the :seen status
@ -269,7 +276,7 @@
(defn- update-last-message [all-chats chat-id] (defn- update-last-message [all-chats chat-id]
(let [{:keys [messages message-groups]} (let [{:keys [messages message-groups]}
(get all-chats chat-id) (get all-chats chat-id)
{:keys [content message-type]} {:keys [content message-type clock-value]}
(->> (chat.db/sort-message-groups message-groups messages) (->> (chat.db/sort-message-groups message-groups messages)
first first
second second
@ -279,7 +286,8 @@
(chat-model/upsert-chat (chat-model/upsert-chat
{:chat-id chat-id {:chat-id chat-id
:last-message-content content :last-message-content content
:last-message-type message-type}))) :last-message-type message-type
:last-clock-value clock-value})))
(fx/defn update-last-messages (fx/defn update-last-messages
[{:keys [db] :as cofx} chat-ids] [{:keys [db] :as cofx} chat-ids]
@ -372,8 +380,12 @@
{:chat-id chat-id {:chat-id chat-id
:timestamp now :timestamp now
:last-message-content (:content message) :last-message-content (:content message)
:last-message-type (:message-type message)}) :last-message-type (:message-type message)
(add-message false message-with-id true) :last-clock-value (:clock-value message)})
(add-message {:batch? false
:message message-with-id
:current-chat? true
:last-clock-value (:clock-value message)})
(add-own-status chat-id message-id :sending) (add-own-status chat-id message-id :sending)
(send chat-id message-id wrapped-record)))) (send chat-id message-id wrapped-record))))
@ -433,7 +445,11 @@
(remove-message-from-group chat-id (get-in db [:chats chat-id :messages message-id])))) (remove-message-from-group chat-id (get-in db [:chats chat-id :messages message-id]))))
(fx/defn add-system-messages [cofx messages] (fx/defn add-system-messages [cofx messages]
(let [messages-fx (map #(add-message false (system-message cofx %) true) messages)] (let [messages-fx (map #(add-message
{:batch false
:message (system-message cofx %)
:current-chat? true})
messages)]
(apply fx/merge cofx messages-fx))) (apply fx/merge cofx messages-fx)))
(fx/defn send-message (fx/defn send-message

View File

@ -47,22 +47,13 @@
:signature signature :signature signature
:chat-id chat-id})))) :chat-id chat-id}))))
(defn- get-last-clock-value [chat-id]
(-> (core/get-by-field @core/account-realm
:message :chat-id chat-id)
(core/sorted :clock-value :desc)
(core/single-clj :message)
:clock-value
(utils.clocks/safe-timestamp)))
(defn- normalize-chat [{:keys [chat-id] :as chat}] (defn- normalize-chat [{:keys [chat-id] :as chat}]
(-> chat (-> chat
(update :admins #(into #{} %)) (update :admins #(into #{} %))
(update :contacts #(into #{} %)) (update :contacts #(into #{} %))
(update :tags #(into #{} %)) (update :tags #(into #{} %))
(update :membership-updates (partial unmarshal-membership-updates chat-id)) (update :membership-updates (partial unmarshal-membership-updates chat-id))
;; We cap the clock value to a safe value in case the db has been polluted (update :last-clock-value utils.clocks/safe-timestamp)
(assoc :last-clock-value (get-last-clock-value chat-id))
(update :last-message-type keyword) (update :last-message-type keyword)
(update :last-message-content edn/read-string))) (update :last-message-content edn/read-string)))

View File

@ -149,7 +149,7 @@
(when (pos? current-version) (when (pos? current-version)
(doseq [schema schemas (doseq [schema schemas
:when (> (:schemaVersion schema) current-version)] :when (> (:schemaVersion schema) current-version)]
(migration-log :current-version current-version) (migration-log :current-version (:schemaVersion schema))
(let [migrated-realm (open-realm schema file-name encryption-key)] (let [migrated-realm (open-realm schema file-name encryption-key)]
(close migrated-realm)))) (close migrated-realm))))
(open-realm (last schemas) file-name encryption-key)) (open-realm (last schemas) file-name encryption-key))

View File

@ -230,3 +230,8 @@
:optional true} :optional true}
:last-message-type {:type :string :last-message-type {:type :string
:optional true}})) :optional true}}))
(def v11
(update v10 :properties merge
{:last-clock-value {:type :int
:optional true}}))

View File

@ -304,6 +304,19 @@
browser/v8 browser/v8
dapp-permissions/v9]) dapp-permissions/v9])
(def v29 [chat/v11
transport/v7
contact/v3
message/v9
mailserver/v11
mailserver-topic/v1
user-status/v2
membership-update/v1
installation/v2
local-storage/v1
browser/v8
dapp-permissions/v9])
;; put schemas ordered by version ;; put schemas ordered by version
(def schemas [{:schema v1 (def schemas [{:schema v1
:schemaVersion 1 :schemaVersion 1
@ -388,4 +401,7 @@
:migration migrations/v27} :migration migrations/v27}
{:schema v28 {:schema v28
:schemaVersion 28 :schemaVersion 28
:migration migrations/v28}]) :migration migrations/v28}
{:schema v29
:schemaVersion 29
:migration migrations/v29}])

View File

@ -7,7 +7,8 @@
[clojure.string :as string] [clojure.string :as string]
[status-im.constants :as constants] [status-im.constants :as constants]
[cognitect.transit :as transit] [cognitect.transit :as transit]
[status-im.js-dependencies :as dependencies])) [status-im.js-dependencies :as dependencies]
[status-im.utils.clocks :as utils.clocks]))
(defn v1 [old-realm new-realm] (defn v1 [old-realm new-realm]
(log/debug "migrating v1 account database: " old-realm new-realm)) (log/debug "migrating v1 account database: " old-realm new-realm))
@ -316,3 +317,23 @@
message-type (aget last-message "message-type")] message-type (aget last-message "message-type")]
(aset chat "last-message-content" content) (aset chat "last-message-content" content)
(aset chat "last-message-type" message-type))))))) (aset chat "last-message-type" message-type)))))))
(defn get-last-clock-value [realm chat-id]
(if-let [last-message
(-> (.objects realm "message")
(.filtered (str "chat-id=\"" chat-id "\""))
(.sorted "clock-value" true)
(aget 0))]
(->
last-message
(aget "clock-value")
(utils.clocks/safe-timestamp))
0))
(defn v29 [old-realm new-realm]
(let [chats (.objects new-realm "chat")]
(dotimes [i (.-length chats)]
(let [chat (aget chats i)
chat-id (aget chat "chat-id")]
(when-let [last-clock-value (get-last-clock-value new-realm chat-id)]
(aset chat "last-clock-value" last-clock-value))))))

View File

@ -5,21 +5,19 @@
(deftest normalize-chat-test (deftest normalize-chat-test
(testing "admins & contacts" (testing "admins & contacts"
(with-redefs [chats/get-last-clock-value (constantly 42)] (is (= {:admins #{4}
(is (= {:last-clock-value 42
:admins #{4}
:contacts #{2} :contacts #{2}
:tags #{} :tags #{}
:membership-updates [] :membership-updates []
:last-message-type :message-type :last-message-type :message-type
:last-message-content {:foo "bar"}} :last-message-content {:foo "bar"}
:last-clock-value nil}
(chats/normalize-chat (chats/normalize-chat
{:admins [4] {:admins [4]
:contacts [2] :contacts [2]
:last-message-type "message-type" :last-message-type "message-type"
:last-message-content "{:foo \"bar\"}"}))))) :last-message-content "{:foo \"bar\"}"}))))
(testing "membership-updates" (testing "membership-updates"
(with-redefs [chats/get-last-clock-value (constantly 42)]
(let [raw-events {"1" {:id "1" :type "members-added" :clock-value 10 :members [1 2] :signature "a" :from "id-1"} (let [raw-events {"1" {:id "1" :type "members-added" :clock-value 10 :members [1 2] :signature "a" :from "id-1"}
"2" {:id "2" :type "member-removed" :clock-value 11 :member 1 :signature "a" :from "id-1"} "2" {:id "2" :type "member-removed" :clock-value 11 :member 1 :signature "a" :from "id-1"}
"3" {:id "3" :type "chat-created" :clock-value 0 :name "blah" :signature "b" :from "id-2"}} "3" {:id "3" :type "chat-created" :clock-value 0 :name "blah" :signature "b" :from "id-2"}}
@ -37,7 +35,7 @@
:membership-updates :membership-updates
(into #{}))] (into #{}))]
(is (= expected (is (= expected
actual)))))) actual)))))
(deftest marshal-membership-updates-test (deftest marshal-membership-updates-test
(let [raw-updates [{:chat-id "chat-id" (let [raw-updates [{:chat-id "chat-id"