[#6460] [#6438] Deduplication-ids

src/status_im/chat/models/loading.cljs

@@ -79,7 +79,8 @@
 (fx/defn initialize-chats
   "Initialize all persisted chats on startup"
   [{:keys [db default-dapps all-stored-chats get-stored-messages get-stored-user-statuses
-           get-stored-unviewed-messages get-referenced-messages stored-message-ids] :as cofx}]
+           get-stored-unviewed-messages get-referenced-messages stored-message-ids
+           stored-deduplication-ids] :as cofx}]
   (let [stored-unviewed-messages (get-stored-unviewed-messages (:current-public-key db))
         chats (reduce (fn [acc {:keys [chat-id] :as chat}]
                         (let [chat-messages (index-messages (get-stored-messages chat-id))
@@ -90,6 +91,7 @@
                                         :unviewed-messages unviewed-ids
                                         :messages chat-messages
                                         :message-statuses (get-stored-user-statuses chat-id message-ids)
+                                        :deduplication-ids (get stored-deduplication-ids chat-id)
                                         :not-loaded-message-ids (set/difference (get stored-message-ids chat-id)
                                                                                 (set message-ids))
                                         :referenced-messages (index-messages

src/status_im/chat/models/message.cljs

@@ -152,14 +152,13 @@
             (chat-loading/group-chat-messages chat-id (get chat->message chat-id))))
 
 (defn- add-to-chat?
-  [{:keys [db]} {:keys [chat-id clock-value message-id message-id-old-format] :as message}]
+  [{:keys [db]} {:keys [chat-id clock-value message-id from] :as message}]
-  (let [{:keys [deleted-at-clock-value messages not-loaded-message-ids]}
+  (let [deduplication-id (messages-store/deduplication-id from chat-id clock-value)
+        {:keys [deleted-at-clock-value messages not-loaded-message-ids deduplication-ids]}
         (get-in db [:chats chat-id])]
     (not (or (get messages message-id)
-             ;; TODO(rasom): remove this condition
-             ;; on when 0.9.29 will not be available for users
-             (and message-id-old-format (get messages message-id-old-format))
              (get not-loaded-message-ids message-id)
+             (get deduplication-ids deduplication-id)
              (>= deleted-at-clock-value clock-value)))))
 
 (defn- filter-messages [cofx messages]

src/status_im/chat/specs.cljs

@@ -18,6 +18,7 @@
 (s/def :chat/message-statuses (s/nilable map?))                   ; message/user statuses indexed by two level index
 (s/def :chat/not-loaded-message-ids (s/nilable set?))             ; set of message-ids not yet fully loaded from persisted state
 (s/def :chat/referenced-messages (s/nilable map?))                ; map of messages indexed by message-id which are not displayed directly, but referenced by other messages
+(s/def :chat/deduplication-ids (s/nilable set?))                  ; set of helper deduplication ids
 (s/def :chat/last-clock-value (s/nilable number?))                ; last logical clock value of messages in chat
 (s/def :chat/loaded-chats (s/nilable seq?))
 (s/def :chat/bot-db (s/nilable map?))

src/status_im/data_store/messages.cljs

@@ -3,7 +3,8 @@
             [re-frame.core :as re-frame]
             [status-im.constants :as constants]
             [status-im.data-store.realm.core :as core]
-            [status-im.utils.core :as utils]))
+            [status-im.utils.core :as utils]
+            [status-im.js-dependencies :as dependencies]))
 
 (defn- transform-message [message]
   (-> message
@@ -52,6 +53,33 @@
                                                                   (aget msg "message-id"))))))
                                      @chat-id->message-id))))
 
+(defn- sha3 [s]
+  (.sha3 dependencies/Web3.prototype s))
+
+(defn deduplication-id
+  "Computes deduplication id from message sender-pk, chat-id and clock-value"
+  [sender-pk chat-id clock-value]
+  (sha3 (str sender-pk chat-id clock-value)))
+
+(re-frame/reg-cofx
+ :data-store/deduplication-ids
+ (fn [cofx _]
+   (assoc cofx :stored-deduplication-ids (let [chat-id->message-id (volatile! {})]
+                                           (-> @core/account-realm
+                                               (.objects "message")
+                                               (.map (fn [msg _ _]
+                                                       (let [chat-id     (aget msg "chat-id")
+                                                             sender-pk   (aget msg "from")
+                                                             clock-value (aget msg "clock-value")]
+                                                         (vswap! chat-id->message-id
+                                                                 #(update %
+                                                                          (aget msg "chat-id")
+                                                                          (fnil conj #{})
+                                                                          (deduplication-id sender-pk
+                                                                                            chat-id
+                                                                                            clock-value)))))))
+                                           @chat-id->message-id))))
+
 (defn- get-unviewed-messages
   [public-key]
   (into {}

src/status_im/events.cljs

@@ -92,6 +92,7 @@
   (re-frame/inject-cofx :data-store/get-unviewed-messages)
   (re-frame/inject-cofx :data-store/get-referenced-messages)
   (re-frame/inject-cofx :data-store/message-ids)
+  (re-frame/inject-cofx :data-store/deduplication-ids)
   (re-frame/inject-cofx :data-store/get-local-storage-data)
   (re-frame/inject-cofx :data-store/get-all-contacts)
   (re-frame/inject-cofx :data-store/get-all-mailservers)

src/status_im/transport/message/protocol.cljs

@@ -105,18 +105,13 @@
            this)
           (send-with-pubkey cofx params)))))
   (receive [this chat-id signature _ cofx]
-    (let [old-message (Message. (:text content) content-type message-type
+    {:chat-received-message/add-fx
-                                clock-value timestamp)]
+     [(assoc (into {} this)
-      {:chat-received-message/add-fx
+             :message-id (transport.utils/message-id this)
-       [(assoc (into {} this)
+             :show? true
-               :message-id (transport.utils/message-id this)
+             :chat-id chat-id
-               ;; TODO(rasom): remove this condition
+             :from signature
-               ;; on when 0.9.29 will not be available for users
+             :js-obj (:js-obj cofx))]})
-               :message-id-old-format (transport.utils/message-id-old-format old-message)
-               :show? true
-               :chat-id chat-id
-               :from signature
-               :js-obj (:js-obj cofx))]}))
   (validate [this]
     (if (spec/valid? :message/message this)
       this

src/status_im/transport/utils.cljs

@@ -22,15 +22,6 @@
   [message]
   (sha3 (pr-str message)))
 
-(defn message-id-old-format
-  "Get an old format message-id.
-   To be removed on 8th day after 0.9.30"
-  [message]
-  (-> message
-      pr-str
-      (clojure.string/replace "message.protocol" "message.v1.protocol")
-      sha3))
-
 (defn get-topic
   "Get the topic of a group chat or public chat from the chat-id"
   [chat-id]

src/status_im/ui/screens/db.cljs

@@ -296,6 +296,7 @@
                  :chat/message-groups
                  :chat/message-statuses
                  :chat/not-loaded-message-ids
+                 :chat/deduplication-ids
                  :chat/referenced-messages
                  :chat/last-clock-value
                  :chat/loaded-chats