[slow sign in] Denormalize last-clock-value

In order to get `:last-clock-value` one extra query was executed for
each chat during initialization.

Implementation:
- `:last-clock-value` field was added to `chat` entity
- this field is updated when the message is sent/received
- extra query was removed

src/status_im/chat/models/message.cljs

@@ -109,7 +109,9 @@
      :prioritary? (not (chat-model/multi-user-chat? cofx chat-id))}))
 
 (fx/defn add-message
-  [{:keys [db] :as cofx} batch? {:keys [chat-id message-id clock-value timestamp content from] :as message} current-chat?]
+  [{:keys [db] :as cofx}
+   {{:keys [chat-id message-id clock-value timestamp from] :as message} :message
+    :keys [current-chat? batch? last-clock-value]}]
   (let [current-public-key (accounts.db/current-public-key cofx)
         prepared-message (-> message
                              (prepare-message chat-id current-chat?)
@@ -124,8 +126,11 @@
               {:db            (cond->
                                (-> db
                                    (update-in [:chats chat-id :messages] assoc message-id prepared-message)
-                                   ;; this will increase last-clock-value twice when sending our own messages
+                                   (update-in [:chats chat-id :last-clock-value]
-                                   (update-in [:chats chat-id :last-clock-value] (partial utils.clocks/receive clock-value)))
+                                              (fn [old-clock-value]
+                                                (or last-clock-value
+                                                    (utils.clocks/receive clock-value
+                                                                          old-clock-value)))))
 
                                 (and (not current-chat?)
                                      (not= from current-public-key))
@@ -193,7 +198,9 @@
     (fx/merge cofx
               {:transport/confirm-messages-processed [{:web3   web3
                                                        :js-obj js-obj}]}
-              (add-message true message current-chat?)
+              (add-message {:batch?       true
+                            :message      message
+                            :current-chat current-chat?})
               ;; Checking :outgoing here only works for now as we don't have a :seen
               ;; status for public chats, if we add processing of our own messages
               ;; for 1-to-1 care needs to be taken not to override the :seen status
@@ -269,7 +276,7 @@
 (defn- update-last-message [all-chats chat-id]
   (let [{:keys [messages message-groups]}
         (get all-chats chat-id)
-        {:keys [content message-type]}
+        {:keys [content message-type clock-value]}
         (->> (chat.db/sort-message-groups message-groups messages)
              first
              second
@@ -279,7 +286,8 @@
     (chat-model/upsert-chat
      {:chat-id              chat-id
       :last-message-content content
-      :last-message-type    message-type})))
+      :last-message-type    message-type
+      :last-clock-value     clock-value})))
 
 (fx/defn update-last-messages
   [{:keys [db] :as cofx} chat-ids]
@@ -372,8 +380,12 @@
                {:chat-id              chat-id
                 :timestamp            now
                 :last-message-content (:content message)
-                :last-message-type    (:message-type message)})
+                :last-message-type    (:message-type message)
-              (add-message false message-with-id true)
+                :last-clock-value     (:clock-value message)})
+              (add-message {:batch?           false
+                            :message          message-with-id
+                            :current-chat?    true
+                            :last-clock-value (:clock-value message)})
               (add-own-status chat-id message-id :sending)
               (send chat-id message-id wrapped-record))))
 
@@ -433,7 +445,11 @@
             (remove-message-from-group chat-id (get-in db [:chats chat-id :messages message-id]))))
 
 (fx/defn add-system-messages [cofx messages]
-  (let [messages-fx (map #(add-message false (system-message cofx %) true) messages)]
+  (let [messages-fx (map #(add-message
+                           {:batch         false
+                            :message       (system-message cofx %)
+                            :current-chat? true})
+                         messages)]
     (apply fx/merge cofx messages-fx)))
 
 (fx/defn send-message

src/status_im/data_store/chats.cljs

@@ -47,22 +47,13 @@
                :signature signature
                :chat-id chat-id}))))
 
-(defn- get-last-clock-value [chat-id]
-  (-> (core/get-by-field @core/account-realm
-                         :message :chat-id chat-id)
-      (core/sorted :clock-value :desc)
-      (core/single-clj :message)
-      :clock-value
-      (utils.clocks/safe-timestamp)))
-
 (defn- normalize-chat [{:keys [chat-id] :as chat}]
   (-> chat
       (update :admins   #(into #{} %))
       (update :contacts #(into #{} %))
       (update :tags #(into #{} %))
       (update :membership-updates  (partial unmarshal-membership-updates chat-id))
-      ;; We cap the clock value to a safe value in case the db has been polluted
+      (update :last-clock-value utils.clocks/safe-timestamp)
-      (assoc :last-clock-value (get-last-clock-value chat-id))
       (update :last-message-type keyword)
       (update :last-message-content edn/read-string)))
 

src/status_im/data_store/realm/core.cljs

@@ -149,7 +149,7 @@
   (when (pos? current-version)
     (doseq [schema schemas
             :when (> (:schemaVersion schema) current-version)]
-      (migration-log :current-version current-version)
+      (migration-log :current-version (:schemaVersion schema))
       (let [migrated-realm (open-realm schema file-name encryption-key)]
         (close migrated-realm))))
   (open-realm (last schemas) file-name encryption-key))

src/status_im/data_store/realm/schemas/account/chat.cljs

@@ -230,3 +230,8 @@
                                   :optional true}
            :last-message-type    {:type     :string
                                   :optional true}}))
+
+(def v11
+  (update v10 :properties merge
+          {:last-clock-value {:type     :int
+                              :optional true}}))

src/status_im/data_store/realm/schemas/account/core.cljs

@@ -304,6 +304,19 @@
           browser/v8
           dapp-permissions/v9])
 
+(def v29 [chat/v11
+          transport/v7
+          contact/v3
+          message/v9
+          mailserver/v11
+          mailserver-topic/v1
+          user-status/v2
+          membership-update/v1
+          installation/v2
+          local-storage/v1
+          browser/v8
+          dapp-permissions/v9])
+
 ;; put schemas ordered by version
 (def schemas [{:schema        v1
                :schemaVersion 1
@@ -388,4 +401,7 @@
                :migration     migrations/v27}
               {:schema        v28
                :schemaVersion 28
-               :migration     migrations/v28}])
+               :migration     migrations/v28}
+              {:schema        v29
+               :schemaVersion 29
+               :migration     migrations/v29}])

src/status_im/data_store/realm/schemas/account/migrations.cljs

@@ -7,7 +7,8 @@
             [clojure.string :as string]
             [status-im.constants :as constants]
             [cognitect.transit :as transit]
-            [status-im.js-dependencies :as dependencies]))
+            [status-im.js-dependencies :as dependencies]
+            [status-im.utils.clocks :as utils.clocks]))
 
 (defn v1 [old-realm new-realm]
   (log/debug "migrating v1 account database: " old-realm new-realm))
@@ -316,3 +317,23 @@
                 message-type (aget last-message "message-type")]
             (aset chat "last-message-content" content)
             (aset chat "last-message-type" message-type)))))))
+
+(defn get-last-clock-value [realm chat-id]
+  (if-let [last-message
+           (-> (.objects realm "message")
+               (.filtered (str "chat-id=\"" chat-id "\""))
+               (.sorted "clock-value" true)
+               (aget 0))]
+    (->
+     last-message
+     (aget "clock-value")
+     (utils.clocks/safe-timestamp))
+    0))
+
+(defn v29 [old-realm new-realm]
+  (let [chats (.objects new-realm "chat")]
+    (dotimes [i (.-length chats)]
+      (let [chat (aget chats i)
+            chat-id (aget chat "chat-id")]
+        (when-let [last-clock-value (get-last-clock-value new-realm chat-id)]
+          (aset chat "last-clock-value" last-clock-value))))))

test/cljs/status_im/test/data_store/chats.cljs

@@ -5,21 +5,19 @@
 
 (deftest normalize-chat-test
   (testing "admins & contacts"
-    (with-redefs [chats/get-last-clock-value (constantly 42)]
+    (is (= {:admins               #{4}
-      (is (= {:last-clock-value 42
-              :admins #{4}
             :contacts             #{2}
             :tags                 #{}
             :membership-updates   []
             :last-message-type    :message-type
-              :last-message-content {:foo "bar"}}
+            :last-message-content {:foo "bar"}
+            :last-clock-value     nil}
            (chats/normalize-chat
             {:admins               [4]
              :contacts             [2]
              :last-message-type    "message-type"
-               :last-message-content "{:foo \"bar\"}"})))))
+             :last-message-content "{:foo \"bar\"}"}))))
   (testing "membership-updates"
-    (with-redefs [chats/get-last-clock-value (constantly 42)]
     (let [raw-events {"1" {:id "1" :type "members-added" :clock-value 10 :members [1 2] :signature "a" :from "id-1"}
                       "2" {:id "2" :type "member-removed" :clock-value 11 :member 1 :signature "a" :from "id-1"}
                       "3" {:id "3" :type "chat-created" :clock-value 0 :name "blah" :signature "b" :from "id-2"}}
@@ -37,7 +35,7 @@
                           :membership-updates
                           (into #{}))]
       (is (= expected
-               actual))))))
+             actual)))))
 
 (deftest marshal-membership-updates-test
   (let [raw-updates [{:chat-id "chat-id"