Use dedup-id instead of raw-payload

This is a backward/forward compatible change with status-go.
We are changing the way messages are confirmed from passing the
raw-object to status-go to a dedup-id instead, which needs to be sent
back.
Based on the response from status-go we detect whether they are ids or
object and act accordingly.

Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>

mobile_files/package.json.orig

@@ -66,7 +66,7 @@
     "string_decoder": "0.10.31",
     "text-encoding": "^0.6.4",
     "url": "0.10.3",
-    "web3": "git+https://github.com/status-im/web3.js.git#0.20.1-status",
+    "web3": "git+https://github.com/status-im/web3.js.git#0.20.2-status",
     "web3-utils": "1.0.0-beta.36",
     "hi-base32": "0.5.0"
   },

mobile_files/yarn.lock

@@ -6014,7 +6014,6 @@ react-native-splash-screen@3.1.1:
   version "2.3.4"
   resolved "git+https://github.com/status-im/react-native-status-keycard.git#04b0fb6fcbe588bd7fa076e1b0379e661bf1bddc"
 
-
 react-native-svg@6.5.2:
   version "6.5.2"
   resolved "https://registry.yarnpkg.com/react-native-svg/-/react-native-svg-6.5.2.tgz#1105896b8873b0856821b18daa0c6898cea6c00c"
@@ -7423,9 +7422,9 @@ web3-utils@1.0.0-beta.36:
     underscore "1.8.3"
     utf8 "2.1.1"
 
-"web3@git+https://github.com/status-im/web3.js.git#0.20.1-status":
+"web3@git+https://github.com/status-im/web3.js.git#0.20.2-status":
   version "0.20.1"
-  resolved "git+https://github.com/status-im/web3.js.git#dad342dc95f7d5a888411ed57c9618937e2b6ebd"
+  resolved "git+https://github.com/status-im/web3.js.git#958dbabff2c77615e23f5de678a6fae0b0d70147"
   dependencies:
     bignumber.js "git+https://github.com/status-im/bignumber.js.git#v4.0.2-status"
     crypto-js "^3.1.4"

src/status_im/chat/core.cljs

@@ -4,7 +4,7 @@
 
 ;; Seen messages
 (fx/defn receive-seen
-  [{:keys [db js-obj]} chat-id sender {:keys [message-ids]}]
+  [{:keys [db js-obj dedup-id]} chat-id sender {:keys [message-ids]}]
   (when-let [seen-messages-ids (-> (get-in db [:chats chat-id :messages])
                                    (select-keys message-ids)
                                    keys)]
@@ -21,4 +21,4 @@
                               db
                               statuses)
        :data-store/tx [{:transaction (user-statuses-store/save-statuses-tx statuses)
-                        :success-event [:message/messages-persisted [js-obj]]}]})))
+                        :success-event [:message/messages-persisted [(or dedup-id js-obj)]]}]})))

src/status_im/chat/models/message.cljs

@@ -111,7 +111,7 @@
 (fx/defn add-message
   [{:keys [db] :as cofx}
    {{:keys [chat-id message-id clock-value timestamp from] :as message} :message
-    :keys [current-chat? batch? last-clock-value raw-message]}]
+    :keys [current-chat? batch? last-clock-value dedup-id raw-message]}]
   (let [current-public-key (accounts.db/current-public-key cofx)
         prepared-message (-> message
                              (prepare-message chat-id current-chat?)
@@ -138,9 +138,9 @@
                                            (fnil conj #{}) message-id))
                :data-store/tx [(merge
                                 {:transaction (messages-store/save-message-tx prepared-message)}
-                                (when raw-message
+                                (when (or dedup-id raw-message)
                                   {:success-event
-                                   [:message/messages-persisted [raw-message]]}))]}
+                                   [:message/messages-persisted [(or dedup-id raw-message)]]}))]}
               (when (and platform/desktop?
                          (not batch?)
                          (not (system-message? prepared-message)))
@@ -173,7 +173,7 @@
 (fx/defn add-received-message
   [{:keys [db] :as cofx}
    old-id->message
-   {:keys [from message-id chat-id js-obj content] :as raw-message}]
+   {:keys [from message-id chat-id js-obj content dedup-id] :as raw-message}]
   (let [{:keys [web3 current-chat-id view-id]} db
         current-public-key            (accounts.db/current-public-key cofx)
         current-chat?                 (and (or (= :chat view-id)
@@ -189,6 +189,7 @@
     (fx/merge cofx
               (add-message {:batch?       true
                             :message      message
+                            :dedup-id     dedup-id
                             :current-chat current-chat?
                             :raw-message  js-obj})
               ;; Checking :outgoing here only works for now as we don't have a :seen

src/status_im/events.cljs

@@ -1463,8 +1463,8 @@
 
 (handlers/register-handler-fx
  :group-chats.callback/extract-signature-success
- (fn [cofx [_ group-update raw-payload sender-signature]]
+ (fn [cofx [_ group-update message-info sender-signature]]
-   (group-chats/handle-membership-update cofx group-update raw-payload sender-signature)))
+   (group-chats/handle-membership-update cofx group-update message-info sender-signature)))
 
 ;; profile module
 

src/status_im/group_chats/core.cljs

@@ -152,8 +152,8 @@
 
 (fx/defn handle-membership-update-received
   "Extract signatures in status-go and act if successful"
-  [cofx membership-update signature raw-payload]
+  [cofx membership-update signature message-info]
-  {:group-chats/extract-membership-signature [[membership-update raw-payload signature]]})
+  {:group-chats/extract-membership-signature [[membership-update message-info signature]]})
 
 (defn chat->group-update
   "Transform a chat in a GroupMembershipUpdate"
@@ -180,21 +180,21 @@
 
 (defn handle-extract-signature-response
   "Callback to dispatch on extract signature response"
-  [payload raw-payload sender-signature response-js]
+  [payload message-info sender-signature response-js]
   (let [{:keys [error identities]} (parse-response response-js)]
     (if error
       (re-frame/dispatch [:group-chats.callback/extract-signature-failed  error])
       (re-frame/dispatch [:group-chats.callback/extract-signature-success
-                          (add-identities payload identities) raw-payload sender-signature]))))
+                          (add-identities payload identities) message-info sender-signature]))))
 
 (defn sign-membership [{:keys [chat-id events] :as payload}]
   (native-module/sign-group-membership (signature-material chat-id events)
                                        (partial handle-sign-response payload)))
 
-(defn extract-membership-signature [payload raw-payload sender]
+(defn extract-membership-signature [payload message-info sender]
   (native-module/extract-group-membership-signatures
    (signature-pairs payload)
-   (partial handle-extract-signature-response payload raw-payload sender)))
+   (partial handle-extract-signature-response payload message-info sender)))
 
 (defn- members-added-event [last-clock-value members]
   {:type "members-added"
@@ -484,7 +484,7 @@
   [cofx {:keys [chat-id
                 message
                 membership-updates] :as membership-update}
-   raw-payload
+   {:keys [raw-payload dedup-id]}
    sender-signature]
   (let [dev-mode? (get-in cofx [:db :account/account :dev-mode?])]
     (when (valid-chat-id? chat-id (extract-creator membership-update))
@@ -515,7 +515,9 @@
                               (instance? protocol/Message message)
                               (= :group-user-message (:message-type message)))
                      (protocol/receive message chat-id sender-signature nil
-                                       (assoc % :js-obj #js {:payload raw-payload}))))))))
+                                       (assoc %
+                                              :dedup-id dedup-id
+                                              :js-obj #js {:payload raw-payload}))))))))
 
 (defn handle-sign-success
   "Upsert chat and send signed payload to group members"

src/status_im/transport/impl/receive.cljs

@@ -9,8 +9,9 @@
 
 (extend-type transport.group-chat/GroupMembershipUpdate
   protocol/StatusMessage
-  (receive [this _ signature _ {:keys [js-obj] :as cofx}]
+  (receive [this _ signature _ {:keys [dedup-id js-obj] :as cofx}]
-    (group-chats/handle-membership-update-received cofx this signature (.-payload js-obj))))
+    (group-chats/handle-membership-update-received cofx this signature {:dedup-id dedup-id
+                                                                        :raw-payload (.-payload js-obj)})))
 
 (extend-type transport.contact/ContactRequest
   protocol/StatusMessage

src/status_im/transport/message/core.cljs

@@ -1,6 +1,7 @@
 (ns ^{:doc "Definition of the StatusMessage protocol"}
  status-im.transport.message.core
   (:require [re-frame.core :as re-frame]
+            [goog.object :as o]
             [status-im.chat.models.message :as models.message]
             [status-im.utils.config :as config]
             [status-im.data-store.transport :as transport-store]
@@ -12,10 +13,28 @@
             [status-im.utils.fx :as fx]
             [taoensso.timbre :as log]))
 
+(defn unwrap-message
+  "Extract message from new payload {:id some-id :message some-message}
+  or old (just plain message)"
+  [js-message]
+  (let [clj-message (js->clj js-message :keywordize-keys true)
+        {:keys [message id]} clj-message]
+    {:message (or message clj-message)
+     :raw-payload (if message
+                    (o/get js-message "message")
+                    js-message)
+     :id id}))
+
 (fx/defn receive-message
+  "Receive message handles a new status-message.
+  dedup-id is passed by status-go and is used to deduplicate messages at that layer.
+  Once a message has been successfuly processed, that id needs to be sent back
+  in order to stop receiving that message"
   [cofx now-in-s filter-chat-id js-message]
   (let [blocked-contacts (get-in cofx [:db :contacts/blocked] #{})
-        {:keys [payload sig timestamp ttl]} (js->clj js-message :keywordize-keys true)
+        {{:keys [payload sig timestamp ttl]} :message
+         dedup-id :id
+         raw-payload :raw-payload} (unwrap-message js-message)
         status-message (-> payload
                            transport.utils/to-utf8
                            transit/deserialize)]
@@ -24,7 +43,7 @@
                (not (blocked-contacts sig)))
       (try
         (when-let [valid-message (protocol/validate status-message)]
-          (fx/merge (assoc cofx :js-obj js-message)
+          (fx/merge (assoc cofx :js-obj raw-payload :dedup-id dedup-id)
                     #(protocol/receive valid-message
                                        (or
                                         filter-chat-id
@@ -173,8 +192,14 @@
                           (keep :js-obj)
                           (apply array))]
      (when (pos? (.-length js-messages))
-       (.confirmMessagesProcessed (transport.utils/shh web3)
+       (if (string? (first js-messages))
-                                  js-messages
+         (.confirmMessagesProcessedByID (transport.utils/shh web3)
-                                  (fn [err resp]
+                                        js-messages
-                                    (when err
+                                        (fn [err resp]
-                                      (log/info "Confirming messages processed failed"))))))))
+                                          (when err
+                                            (log/warn "Confirming messages processed failed" err))))
+         (.confirmMessagesProcessed (transport.utils/shh web3)
+                                    js-messages
+                                    (fn [err resp]
+                                      (when err
+                                        (log/warn "Confirming messages processed failed" err)))))))))

src/status_im/transport/message/protocol.cljs

@@ -118,6 +118,7 @@
              :raw-payload-hash (transport.utils/sha3
                                 (.-payload (:js-obj cofx)))
              :from signature
+             :dedup-id (:dedup-id cofx)
              :js-obj (:js-obj cofx))]})
   (validate [this]
     (if (spec/valid? :message/message this)