[#6903] fix replies compatibility

Issue was caused by https://github.com/status-im/status-react/pull/6722

Implementation:
1. `old-message-id` field (indexed) was introduced in `message` entity
   and is calculated as `message-id` was calculated in `0.9.31`
```clojure
(defn old-message-id
  [message]
  (sha3 (pr-str message)))
```
2. When a reply message is sent from the PR version of app both `response-to`
   and `response-to-v2` fields are sent as a part of `message`'s `content`
   field, so that it can be recognized by `0.9.31`.
3. When PR version of app receives reply from `0.9.31` we check whether
   message's `content` contains `response-to` but doesn't contain
   `response-to-v2`, and if so we check whether DB contains message with
   `old-message-id=response-to`. If such message has been found we assoc
   `response-to-v2` to content.
4. If message from DB contains only `response-to` but not `response-to-v2`
   attempt to fetch the message by `old-message-id` is done.

src/status_im/chat/db.cljs

@@ -60,10 +60,11 @@
                          :type  :datemark})
                   (map (fn [{:keys [message-id timestamp-str]}]
                          (let [{:keys [content] :as message} (get messages message-id)
-                               quote (some-> (:response-to content)
+                               {:keys [response-to response-to-v2]} content
+                               quote (some-> (or response-to-v2 response-to)
                                              (quoted-message-data messages referenced-messages))]
                            (cond-> (-> message
-                                       (update :content dissoc :response-to)
+                                       (update :content dissoc :response-to :response-to-v2)
                                        (assoc :datemark      datemark
                                               :timestamp-str timestamp-str
                                               :user-statuses (get message-statuses message-id)))

src/status_im/chat/models/input.cljs

@@ -87,10 +87,12 @@
 
 (fx/defn reply-to-message
   "Sets reference to previous chat message and focuses on input"
-  [{:keys [db] :as cofx} message-id]
+  [{:keys [db] :as cofx} message-id old-message-id]
   (let [current-chat-id (:current-chat-id db)]
     (fx/merge cofx
-              {:db (assoc-in db [:chats current-chat-id :metadata :responding-to-message] message-id)}
+              {:db (assoc-in db [:chats current-chat-id :metadata :responding-to-message]
+                             {:message-id     message-id
+                              :old-message-id old-message-id})}
               (chat-input-focus :input-ref))))
 
 (fx/defn cancel-message-reply
@@ -121,15 +123,17 @@
   "no command detected, when not empty, proceed by sending text message without command processing"
   [input-text current-chat-id {:keys [db] :as cofx}]
   (when-not (string/blank? input-text)
-    (let [reply-to-message (get-in db [:chats current-chat-id :metadata :responding-to-message])]
+    (let [{:keys [message-id old-message-id]}
+          (get-in db [:chats current-chat-id :metadata :responding-to-message])]
       (fx/merge cofx
                 {:db (assoc-in db [:chats current-chat-id :metadata :responding-to-message] nil)}
                 (chat.message/send-message {:chat-id      current-chat-id
                                             :content-type constants/content-type-text
                                             :content      (cond-> {:chat-id current-chat-id
                                                                    :text    input-text}
-                                                            reply-to-message
+                                                            message-id
-                                                            (assoc :response-to reply-to-message))})
+                                                            (assoc :response-to old-message-id
+                                                                   :response-to-v2 message-id))})
                 (commands.input/set-command-reference nil)
                 (set-chat-input-text nil)
                 (process-cooldown)))))

src/status_im/chat/models/loading.cljs

@@ -44,12 +44,17 @@
              (:chats db)))
 
 (defn- get-referenced-ids
-  "Takes map of message-id->messages and returns set of message ids which are referenced by the original messages,
+  "Takes map of `message-id->messages` and returns set of maps of
-  excluding any message id, which is already in the original map"
+  `{:response-to old-message-id :response-to-v2 message-id}`,
+   excluding any `message-id` which is already in the original map"
   [message-id->messages]
   (into #{}
-        (comp (keep (comp :response-to :content))
+        (comp (keep (fn [{:keys [content]}]
-              (filter #(not (contains? message-id->messages %))))
+                      (let [response-to-id
+                            (select-keys content [:response-to :response-to-v2])]
+                        (when (some (complement nil?) (vals response-to-id))
+                          response-to-id))))
+              (remove #(some message-id->messages (vals %))))
         (vals message-id->messages)))
 
 (defn get-unviewed-messages-ids
@@ -86,9 +91,8 @@
                  :message-statuses statuses
                  :loaded-unviewed-messages-ids unviewed-messages-ids
                  :referenced-messages (into {}
-                                            (map (juxt :message-id identity)
+                                            (get-referenced-messages
-                                                 (get-referenced-messages
+                                             (get-referenced-ids chat-messages))))))
-                                                  (get-referenced-ids chat-messages)))))))
             chats
             (keys chats)))}
      (group-messages))))
@@ -137,8 +141,8 @@
     (let [loaded-count        (count (get-in db [:chats current-chat-id :messages]))
           new-messages        (get-stored-messages current-chat-id loaded-count)
           indexed-messages    (index-messages new-messages)
-          referenced-messages (index-messages
+          referenced-messages (into empty-message-map
-                               (get-referenced-messages (get-referenced-ids indexed-messages)))
+                                    (get-referenced-messages (get-referenced-ids indexed-messages)))
           new-message-ids     (keys indexed-messages)
           new-statuses        (get-stored-user-statuses current-chat-id new-message-ids)
           public-key          (accounts.db/current-public-key cofx)

src/status_im/chat/models/message.cljs

@@ -163,10 +163,20 @@
                                               :to    chat-id
                                               :from  from}}))))
 
+(defn check-response-to
+  [{{:keys [response-to response-to-v2]} :content :as message}
+   old-id->message]
+  (if (and response-to (not response-to-v2))
+    (let [response-to-v2
+          (or (get-in old-id->message [response-to :message-id])
+              (messages-store/get-message-id-by-old response-to))]
+      (assoc-in message [:content :response-to-v2] response-to-v2))
+    message))
+
 (fx/defn add-received-message
-  [{:keys [db now] :as cofx}
+  [{:keys [db] :as cofx}
-   batch?
+   old-id->message
-   {:keys [from message-id chat-id js-obj] :as raw-message}]
+   {:keys [from message-id chat-id js-obj content] :as raw-message}]
   (let [{:keys [web3 current-chat-id view-id]} db
         current-public-key            (accounts.db/current-public-key cofx)
         current-chat?                 (and (or (= :chat view-id)
@@ -176,12 +186,13 @@
         message                       (-> raw-message
                                           (commands-receiving/enhance-receive-parameters cofx)
                                           (ensure-clock-value chat)
+                                          (check-response-to old-id->message)
                                           ;; TODO (cammellos): Refactor so it's not computed twice
                                           (add-outgoing-status current-public-key))]
     (fx/merge cofx
               {:transport/confirm-messages-processed [{:web3   web3
                                                        :js-obj js-obj}]}
-              (add-message batch? message current-chat?)
+              (add-message true message current-chat?)
               ;; Checking :outgoing here only works for now as we don't have a :seen
               ;; status for public chats, if we add processing of our own messages
               ;; for 1-to-1 care needs to be taken not to override the :seen status
@@ -209,17 +220,24 @@
              (messages-store/message-exists? message-id)))))
 
 (defn- filter-messages [cofx messages]
-  (:accumulated (reduce (fn [{:keys [seen-ids] :as acc}
+  (:accumulated
-                             {:keys [message-id] :as message}]
+   (reduce (fn [{:keys [seen-ids] :as acc}
-                          (if (and (add-to-chat? cofx message)
+                {:keys [message-id old-message-id] :as message}]
-                                   (not (seen-ids message-id)))
+             (if (and (add-to-chat? cofx message)
-                            (-> acc
+                      (not (seen-ids message-id)))
-                                (update :seen-ids conj message-id)
+               (-> acc
-                                (update :accumulated conj message))
+                   (update :seen-ids conj message-id)
-                            acc))
+                   (update :accumulated
-                        {:seen-ids    #{}
+                           (fn [acc]
-                         :accumulated []}
+                             (-> acc
-                        messages)))
+                                 (update :messages conj message)
+                                 (assoc-in [:by-old-message-id old-message-id]
+                                           message)))))
+               acc))
+           {:seen-ids    #{}
+            :accumulated {:messages     []
+                          :by-old-message-id {}}}
+           messages)))
 
 (defn extract-chat-id [cofx {:keys [chat-id from message-type]}]
   "Validate and return a valid chat-id"
@@ -249,8 +267,11 @@
 
 (fx/defn receive-many
   [{:keys [now] :as cofx} messages]
-  (let [valid-messages   (keep #(when-let [chat-id (extract-chat-id cofx %)] (assoc % :chat-id chat-id)) messages)
+  (let [valid-messages   (keep #(when-let [chat-id (extract-chat-id cofx %)]
-        deduped-messages (filter-messages cofx valid-messages)
+                                  (assoc % :chat-id chat-id)) messages)
+        filtered-messages (filter-messages cofx valid-messages)
+        deduped-messages (:messages filtered-messages)
+        old-id->message  (:by-old-message-id filtered-messages)
         chat->message    (group-by :chat-id deduped-messages)
         chat-ids         (keys chat->message)
         chats-fx-fns     (map (fn [chat-id]
@@ -265,7 +286,7 @@
                                     :timestamp               now
                                     :unviewed-messages-count unviewed-messages-count})))
                               chat-ids)
-        messages-fx-fns (map #(add-received-message true %) deduped-messages)
+        messages-fx-fns (map #(add-received-message old-id->message %) deduped-messages)
         groups-fx-fns   (map #(update-group-messages chat->message %) chat-ids)]
     (apply fx/merge cofx (concat chats-fx-fns
                                  messages-fx-fns
@@ -283,7 +304,9 @@
                  :content      content
                  :message-type :system-message
                  :content-type constants/content-type-status}]
-    (assoc message :message-id (transport.utils/message-id message))))
+    (assoc message
+           :message-id (transport.utils/message-id message)
+           :old-message-id "system")))
 
 (defn group-message? [{:keys [message-type]}]
   (#{:group-user-message :public-group-user-message} message-type))
@@ -314,8 +337,11 @@
 
 (fx/defn upsert-and-send [{:keys [now] :as cofx} {:keys [chat-id] :as message}]
   (let [send-record     (protocol/map->Message (select-keys message transport-keys))
+        old-message-id  (transport.utils/old-message-id send-record)
         message-id      (transport.utils/message-id message)
-        message-with-id (assoc message :message-id message-id)]
+        message-with-id (assoc message
+                               :message-id message-id
+                               :old-message-id old-message-id)]
 
     (fx/merge cofx
               (chat-model/upsert-chat {:chat-id chat-id

src/status_im/chat/subs.cljs

@@ -263,4 +263,4 @@
  :chats/reply-message
  :<- [:chats/current-chat]
  (fn [{:keys [metadata messages]}]
-   (get messages (:responding-to-message metadata))))
+   (get messages (get-in metadata [:responding-to-message :message-id]))))

src/status_im/data_store/messages.cljs

@@ -21,18 +21,32 @@
                       (core/all-clj :message))]
      (map transform-message messages))))
 
-(defn- get-by-messages-ids
+(defn get-message-id-by-old [old-message-id]
+  (when-let
+   [js-message (core/single
+                (core/get-by-field
+                 @core/account-realm
+                 :message :old-message-id old-message-id))]
+    (aget js-message "message-id")))
+
+(defn- get-references-by-ids
   [message-ids]
   (when (seq message-ids)
-    (keep (fn [message-id]
+    (keep (fn [{:keys [response-to response-to-v2]}]
-            (when-let [js-message (.objectForPrimaryKey @core/account-realm "message" message-id)]
+            (when-let [js-message
-              (-> js-message
+                       (if response-to-v2
-                  (core/realm-obj->clj :message)
+                         (.objectForPrimaryKey @core/account-realm "message" response-to-v2)
-                  transform-message)))
+                         (core/single (core/get-by-field
+                                       @core/account-realm
+                                       :message :old-message-id response-to)))]
+              [(or response-to-v2 response-to)
+               (-> js-message
+                   (core/realm-obj->clj :message)
+                   transform-message)]))
           message-ids)))
 
 (def default-values
-  {:to             nil})
+  {:to nil})
 
 (re-frame/reg-cofx
  :data-store/get-messages
@@ -45,7 +59,7 @@
 (re-frame/reg-cofx
  :data-store/get-referenced-messages
  (fn [cofx _]
-   (assoc cofx :get-referenced-messages get-by-messages-ids)))
+   (assoc cofx :get-referenced-messages get-references-by-ids)))
 
 (defn- prepare-content [content]
   (if (string? content)

src/status_im/data_store/realm/schemas/account/core.cljs

@@ -278,6 +278,19 @@
           browser/v8
           dapp-permissions/v9])
 
+(def v27 [chat/v9
+          transport/v7
+          contact/v3
+          message/v8
+          mailserver/v11
+          mailserver-topic/v1
+          user-status/v2
+          membership-update/v1
+          installation/v2
+          local-storage/v1
+          browser/v8
+          dapp-permissions/v9])
+
 ;; put schemas ordered by version
 (def schemas [{:schema        v1
                :schemaVersion 1
@@ -356,4 +369,7 @@
                :migration     migrations/v25}
               {:schema        v26
                :schemaVersion 26
-               :migration     migrations/v26}])
+               :migration     migrations/v26}
+              {:schema        v27
+               :schemaVersion 27
+               :migration     migrations/v27}])

src/status_im/data_store/realm/schemas/account/message.cljs

@@ -51,3 +51,9 @@
                                          :default 0}
                       :show?            {:type    :bool
                                          :default true}}})
+
+(def v8
+  (-> v7
+      (assoc-in [:properties :old-message-id]
+                {:type    :string
+                 :indexed true})))

src/status_im/data_store/realm/schemas/account/migrations.cljs

@@ -3,6 +3,9 @@
             [cljs.reader :as reader]
             [status-im.chat.models.message-content :as message-content]
             [status-im.transport.utils :as transport.utils]
+            [cljs.tools.reader.edn :as edn]
+            [status-im.js-dependencies :as dependencies]
+            [clojure.string :as string]
             [cljs.tools.reader.edn :as edn]))
 
 (defn v1 [old-realm new-realm]
@@ -172,7 +175,7 @@
       (let [message (aget new-messages i)
             message-id (aget message "message-id")
             from (aget message "from")
-            chat-id (aget message "chat-id")
+            chat-id (:chat-id (edn/read-string (aget message "content")))
             clock-value (aget message "clock-value")
             new-message-id (transport.utils/message-id
                             {:from        from
@@ -240,3 +243,33 @@
                                                     "status = \"received\""))
                                     (.-length))]
         (aset chat "unviewed-messages-count" user-statuses-count)))))
+
+(defrecord Message [content content-type message-type clock-value timestamp])
+
+(defn sha3 [s]
+  (.sha3 dependencies/Web3.prototype s))
+
+(defn replace-ns [str-message]
+  (string/replace-first
+   str-message
+   "status-im.data-store.realm.schemas.account.migrations"
+   "status-im.transport.message.protocol"))
+
+(defn old-message-id
+  [message]
+  (sha3 (replace-ns (pr-str message))))
+
+(defn v27 [old-realm new-realm]
+  (let [messages (.objects new-realm "message")]
+    (dotimes [i (.-length messages)]
+      (let [js-message     (aget messages i)
+            message        {:content      (edn/read-string
+                                           (aget js-message "content"))
+                            :content-type (aget js-message "content-type")
+                            :message-type (keyword
+                                           (aget js-message "message-type"))
+                            :clock-value  (aget js-message "clock-value")
+                            :timestamp    (aget js-message "timestamp")}
+            message-record (map->Message message)
+            old-message-id (old-message-id message-record)]
+        (aset js-message "old-message-id" old-message-id)))))

src/status_im/events.cljs

@@ -692,8 +692,8 @@
 
 (handlers/register-handler-fx
  :chat.ui/reply-to-message
- (fn [cofx [_ message-id]]
+ (fn [cofx [_ message-id old-message-id]]
-   (chat.input/reply-to-message cofx message-id)))
+   (chat.input/reply-to-message cofx message-id old-message-id)))
 
 (handlers/register-handler-fx
  :chat.ui/send-current-message

src/status_im/transport/db.cljs

@@ -58,6 +58,7 @@
 
 (spec/def :message.content/text (spec/and string? (complement s/blank?)))
 (spec/def :message.content/response-to string?)
+(spec/def :message.content/response-to-v2 string?)
 (spec/def :message.content/command-path (spec/tuple string? (spec/coll-of (spec/or :scope keyword? :chat-id string?) :kind set? :min-count 1)))
 (spec/def :message.content/params (spec/map-of keyword? any?))
 

src/status_im/transport/message/protocol.cljs

@@ -116,8 +116,9 @@
   (receive [this chat-id signature _ cofx]
     {:chat-received-message/add-fx
      [(assoc (into {} this)
+             :old-message-id (transport.utils/old-message-id this)
              :message-id (transport.utils/message-id
-                          {:chat-id     chat-id
+                          {:chat-id     (:chat-id content)
                            :from        signature
                            :clock-value clock-value})
              :chat-id chat-id

src/status_im/transport/utils.cljs

@@ -17,11 +17,13 @@
 (defn sha3 [s]
   (.sha3 dependencies/Web3.prototype s))
 
+(defn old-message-id
+  [message]
+  (sha3 (pr-str message)))
+
 (defn message-id
   "Get a message-id"
   [{:keys [from chat-id clock-value] :as m}]
-  {:pre [(not (nil? from))
-         (not (nil? chat-id))]}
   (sha3 (str from chat-id clock-value)))
 
 (defn get-topic

src/status_im/ui/components/list_selection.cljs

@@ -12,9 +12,9 @@
             (:url content))
     (.share react/sharing (clj->js content))))
 
-(defn- message-options [message-id text]
+(defn- message-options [message-id old-message-id text]
   [{:label  (i18n/label :t/message-reply)
-    :action #(re-frame/dispatch [:chat.ui/reply-to-message message-id])}
+    :action #(re-frame/dispatch [:chat.ui/reply-to-message message-id old-message-id])}
    {:label  (i18n/label :t/sharing-copy-to-clipboard)
     :action #(react/copy-to-clipboard text)}
    {:label  (i18n/label :t/sharing-share)
@@ -25,9 +25,9 @@
     (action-sheet/show options)
     (dialog/show options)))
 
-(defn chat-message [message-id text dialog-title]
+(defn chat-message [message-id old-message-id text dialog-title]
   (show {:title       dialog-title
-         :options     (message-options message-id text)
+         :options     (message-options message-id old-message-id text)
          :cancel-text (i18n/label :t/message-options-cancel)}))
 
 (defn browse [link]

src/status_im/ui/screens/chat/message/message.cljs

@@ -253,13 +253,13 @@
    [react/view (style/delivery-status outgoing)
     [message-delivery-status message]]])
 
-(defn chat-message [{:keys [message-id outgoing group-chat modal? current-public-key content-type content] :as message}]
+(defn chat-message [{:keys [message-id old-message-id outgoing group-chat modal? current-public-key content-type content] :as message}]
   [react/view
    [react/touchable-highlight {:on-press      (fn [_]
                                                 (re-frame/dispatch [:chat.ui/set-chat-ui-props {:messages-focused? true}])
                                                 (react/dismiss-keyboard!))
                                :on-long-press #(when (= content-type constants/content-type-text)
-                                                 (list-selection/chat-message message-id (:text content) (i18n/label :t/message)))}
+                                                 (list-selection/chat-message message-id old-message-id (:text content) (i18n/label :t/message)))}
     [react/view {:accessibility-label :chat-item}
      (let [incoming-group (and group-chat (not outgoing))]
        [message-content message-body (merge message

src/status_im/ui/screens/desktop/main/chat/views.cljs

@@ -98,7 +98,7 @@
   (not= (get-in user-statuses [current-public-key :status]) :not-sent))
 
 (views/defview message-without-timestamp
-  [text {:keys [message-id content current-public-key user-statuses] :as message} style]
+  [text {:keys [message-id old-message-id content current-public-key user-statuses] :as message} style]
   [react/view {:flex 1 :margin-vertical 5}
    [react/touchable-highlight {:on-press (fn [arg]
                                            (when (= "right" (.-button (.-nativeEvent arg)))
@@ -107,7 +107,7 @@
                                                 :on-select #(do (utils/show-popup "" "Message copied to clipboard") (react/copy-to-clipboard text))}
                                                {:text (i18n/label :t/message-reply)
                                                 :on-select #(when (message-sent? user-statuses current-public-key)
-                                                              (re-frame/dispatch [:chat.ui/reply-to-message message-id]))}])))}
+                                                              (re-frame/dispatch [:chat.ui/reply-to-message message-id old-message-id]))}])))}
     [react/view {:style styles/message-container}
      (when (:response-to content)
        [quoted-message (:response-to content) false current-public-key])