Use payload & avoid transforming in clojure objects.

This commit includes a few performance fixes: 1) Pass a string payload instead of an hex encoded string, to avoid unecessary conversion 2) Don't js->clj on messages, as that's fairly expensive and we can get away without 3) Don't use `pr-str` `read-string`, rather convert to json Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-23 09:09:11 +02:00 · 2019-10-23 09:09:11 +02:00 · 9cd891365a
parent 57b1722863
commit 9cd891365a
9 changed files with 654 additions and 626 deletions
--- a/externs.js
+++ b/externs.js
--- a/src/status_im/data_store/messages.cljs
+++ b/src/status_im/data_store/messages.cljs
@ -5,6 +5,7 @@
            [clojure.string :as string]
            [taoensso.timbre :as log]
            [re-frame.core :as re-frame]
+            [status-im.utils.types :as utils.types]
            [status-im.ethereum.json-rpc :as json-rpc]
            [status-im.constants :as constants]
            [status-im.utils.core :as utils]))
@ -12,7 +13,7 @@
 (defn prepare-content [content]
  (if (string? content)
    content
-    (pr-str content)))
+    (utils.types/clj->json content)))

 (defn ->rpc [message]
  (-> message
--- a/src/status_im/signals/core.cljs
+++ b/src/status_im/signals/core.cljs
@ -2,6 +2,7 @@
  (:require [status-im.ethereum.subscriptions :as ethereum.subscriptions]
            [status-im.i18n :as i18n]
            [status-im.mailserver.core :as mailserver]
+            [clojure.string :as string]
            [status-im.multiaccounts.login.core :as login]
            [status-im.multiaccounts.model :as multiaccounts.model]
            [status-im.pairing.core :as pairing]
@ -42,19 +43,23 @@

 (fx/defn process
  [cofx event-str]
-  (let [{:keys [type event]} (types/json->clj event-str)]
+  ;; We only convert to clojure when strictly necessary or we know it
+  ;; won't impact performance, as it is a fairly costly operation on large-ish
+  ;; data structures
+  (let [data (.parse js/JSON event-str)
+        event-js (.-event data)
+        type (.-type data)]
    (case type
-      "node.login"         (status-node-started cofx event)
-      "envelope.sent"      (transport.message/update-envelopes-status cofx (:ids event) :sent)
-      "envelope.expired"   (transport.message/update-envelopes-status cofx (:ids event) :not-sent)
-      "bundles.added"      (pairing/handle-bundles-added cofx event)
-      "mailserver.request.completed" (mailserver/handle-request-completed cofx event)
+      "node.login"         (status-node-started cofx (js->clj event-js :keywordize-keys true))
+      "envelope.sent"      (transport.message/update-envelopes-status cofx (:ids (js->clj event-js :keywordize-keys true)) :sent)
+      "envelope.expired"   (transport.message/update-envelopes-status cofx (:ids (js->clj event-js :keywordize-keys true)) :not-sent)
+      "mailserver.request.completed" (mailserver/handle-request-completed cofx (js->clj event-js :keywordize-keys true))
      "mailserver.request.expired"   (when (multiaccounts.model/logged-in? cofx)
-                                       (mailserver/resend-request cofx {:request-id (:hash event)}))
-      "discovery.summary"  (summary cofx event)
-      "subscriptions.data" (ethereum.subscriptions/handle-signal cofx event)
-      "subscriptions.error" (ethereum.subscriptions/handle-error cofx event)
-      "whisper.filter.added" (transport.filters/handle-negotiated-filter cofx event)
-      "messages.new" (transport.message/receive-messages cofx event)
-      "wallet" (ethereum.subscriptions/new-wallet-event cofx event)
-      (log/debug "Event " type " not handled" event))))
+                                       (mailserver/resend-request cofx {:request-id (.-hash event-js)}))
+      "discovery.summary"  (summary cofx (js->clj event-js :keywordize-keys true))
+      "subscriptions.data" (ethereum.subscriptions/handle-signal cofx (js->clj event-js :keywordize-keys true))
+      "subscriptions.error" (ethereum.subscriptions/handle-error cofx (js->clj event-js :keywordize-keys true))
+      "whisper.filter.added" (transport.filters/handle-negotiated-filter cofx (js->clj event-js :keywordize-keys true))
+      "messages.new" (transport.message/receive-messages cofx event-js)
+      "wallet" (ethereum.subscriptions/new-wallet-event cofx (js->clj event-js :keywordize-keys true))
+      (log/debug "Event " type " not handled"))))
--- a/src/status_im/transport/message/core.cljs
+++ b/src/status_im/transport/message/core.cljs
@ -26,13 +26,19 @@
  dedup-id is passed by status-go and is used to deduplicate messages at that layer.
  Once a message has been successfuly processed, that id needs to be sent back
  in order to stop receiving that message"
-  [cofx now-in-s filter-chat-id message]
+  [cofx now-in-s filter-chat-id message-js]
  (let [blocked-contacts (get-in cofx [:db :contacts/blocked] #{})
-        {{:keys [payload timestamp]} :message
-         metadata :metadata
-         raw-payload :raw-payload} (add-raw-payload message)
+        payload (.-payload message-js)
+        timestamp (.-timestamp (.-message message-js))
+        metadata-js (.-metadata message-js)
+        metadata {:author {:publicKey (.-publicKey (.-author metadata-js))
+                           :alias (.-alias (.-author metadata-js))
+                           :identicon (.-identicon (.-author metadata-js))}
+                  :dedupId (.-dedupId metadata-js)
+                  :encryptionId (.-encryptionId metadata-js)
+                  :messageId (.-messageId metadata-js)}
+        raw-payload  {:raw-payload message-js}
        status-message (-> payload
-                           ethereum/hex-to-utf8
                           transit/deserialize)
        sig (-> metadata :author :publicKey)]
    (when (and sig
@ -70,21 +76,24 @@
      (apply fx/merge cofx receive-message-fxs))
    (log/error "Something went wrong" error messages)))

-(fx/defn receive-messages [cofx event]
+(fx/defn receive-messages [cofx event-js]
  (let [fxs (keep
-             (fn [{:keys [chat messages error]}]
-               (when (seq messages)
-                 (receive-whisper-messages
-                  error
-                  messages
+             (fn [message-specs]
+               (let [chat (.-chat message-specs)
+                     messages (.-messages message-specs)
+                     error (.-error message-specs)]
+                 (when (seq messages)
+                   (receive-whisper-messages
+                    error
+                    messages
                  ;; For discovery and negotiated filters we don't
                  ;; set a chatID, and we use the signature of the message
                  ;; to indicate which chat it is for
-                  (if (or (:discovery chat)
-                          (:negotiated chat))
-                    nil
-                    (:chatId chat)))))
-             (:messages event))]
+                    (if (or (.-discovery chat)
+                            (.-negotiated chat))
+                      nil
+                      (.-chatId chat))))))
+             (.-messages event-js))]
    (apply fx/merge cofx fxs)))

 (fx/defn remove-hash
--- a/src/status_im/ui/screens/navigation.cljs
+++ b/src/status_im/ui/screens/navigation.cljs
@ -122,4 +122,4 @@
             {:db (-> db
                      (assoc :prev-tab-view-id (:view-id db))
                      (assoc :prev-view-id (:view-id db)))}
-             (navigate-to-cofx view-id {}))))
+             (navigate-to-cofx view-id {}))))
--- a/src/status_im/utils/core.cljc
+++ b/src/status_im/utils/core.cljc
@ -1,7 +1,5 @@
 (ns status-im.utils.core
  (:require [clojure.string :as str]
-            #?(:cljs [cljs.tools.reader.edn :as edn]
-               :clj [clojure.tools.reader.edn :as edn])
            [taoensso.timbre :as log]))

 (defn truncate-str
@ -70,6 +68,6 @@
 #?(:cljs
   (defn safe-read-message-content [content]
     (try
-       (edn/read-string content)
+        (js->clj (.parse js/JSON content) :keywordize-keys true)
       (catch :default e
         (log/warn "failed to transform message with " e)))))
--- a/status-go-version.json
+++ b/status-go-version.json
@ -2,7 +2,7 @@
    "_comment": "DO NOT EDIT THIS FILE BY HAND. USE 'scripts/update-status-go.sh <tag>' instead",
    "owner": "status-im",
    "repo": "status-go",
-    "version": "v0.34.0-beta.0",
-    "commit-sha1": "94228bce2ee6a7a5f62917e28247a22c37b6699b",
-    "src-sha256": "0f62yh94vn6k4gqa50r3gxzscqabmgc047sng291g5wp6d755fmr"
+    "version": "5f2544f1e8f2ee832bbab95c8d74e26ba0ab2481",
+    "commit-sha1": "5f2544f1e8f2ee832bbab95c8d74e26ba0ab2481",
+    "src-sha256": "0dlkr6056mhyny9k895zs5012ywiz75qm6y7a6hypb3pfxr35931"
 }
--- a/test/cljs/status_im/test/data_store/messages.cljs
+++ b/test/cljs/status_im/test/data_store/messages.cljs
@ -27,7 +27,7 @@
                    :from from
                    :chatId chat-id
                    :replyTo "id-2"
-                    :content "{:chat-id \"chat-id\", :response-to-v2 \"id-2\", :text \"hta\"}"
+                    :content "{\"chat-id\":\"chat-id\",\"response-to-v2\":\"id-2\",\"text\":\"hta\"}"
                    :contentType "text/plain"
                    :messageType "public-group-user-message"
                    :clockValue 2
@ -55,12 +55,12 @@
                   :whisperTimestamp 1
                   :from from
                   :chatId chat-id
-                   :content "{:chat-id \"chat-id\", :text \"hta\"}"
+                   :content "{\"chat-id\":\"chat-id\",\"text\":\"hta\"}"
                   :contentType "text/plain"
                   :messageType "public-group-user-message"
                   :clockValue 2
                   :quotedMessage {:from "from"
-                                   :content "{:chat-id \"chat-id\", :text \"reply\"}"}
+                                   :content "{\"chat-id\":\"chat-id\",\"text\":\"reply\"}"}
                   :timestamp 3
                   :outgoingStatus "sending"}]
      (is (= expected (m/<-rpc message))))))
--- a/test/cljs/status_im/test/transport/core.cljs
+++ b/test/cljs/status_im/test/transport/core.cljs
@ -46,16 +46,17 @@

 (def sig "0x04325367620ae20dd878dbb39f69f02c567d789dd21af8a88623dc5b529827c2812571c380a2cd8236a2851b8843d6486481166c39debf60a5d30b9099c66213e4")

-(def messages [{:id "someid"
-                :metadata {:author {:publicKey sig}}
-                :message {:sig       sig
-                          :ttl       10
-                          :timestamp 1527692015
-                          :topic     "0x9c22ff5f"
-                          :payload   "0x5b227e236334222c5b2246222c22746578742f706c61696e222c227e3a7075626c69632d67726f75702d757365722d6d657373616765222c3135323736393230313433383130312c313532373639323031343337375d5d"
-                          :padding   "0xbf06347cc7f9aa18b4a846032264a88f559d9b14079975d14b10648847c0543a77a80624e101c082d19b502ae3b4f97958d18abf59eb0a82afc1301aa22470495fac739a30c2f563599fa8d8e09363a43d39311596b7f119dee7b046989c08224f1ef5cdc385"
-                          :pow       0.002631578947368421
-                          :hash      "0x220ef9994a4fae64c112b27ed07ef910918159cbe6fcf8ac515ee2bf9a6711a0"}}])
+(def messages [(clj->js {:id "someid"
+                         :metadata {:author {:publicKey sig}}
+                         :payload "[\"~#c4\",[\"F\",\"text/plain\",\"~:public-group-user-message\",152769201438101,1527692014377]]"
+                         :message {:sig       sig
+                                   :ttl       10
+                                   :timestamp 1527692015
+                                   :topic     "0x9c22ff5f"
+                                   :payload   "0x5b227e236334222c5b2246222c22746578742f706c61696e222c227e3a7075626c69632d67726f75702d757365722d6d657373616765222c3135323736393230313433383130312c313532373639323031343337375d5d"
+                                   :padding   "0xbf06347cc7f9aa18b4a846032264a88f559d9b14079975d14b10648847c0543a77a80624e101c082d19b502ae3b4f97958d18abf59eb0a82afc1301aa22470495fac739a30c2f563599fa8d8e09363a43d39311596b7f119dee7b046989c08224f1ef5cdc385"
+                                   :pow       0.002631578947368421
+                                   :hash      "0x220ef9994a4fae64c112b27ed07ef910918159cbe6fcf8ac515ee2bf9a6711a0"}})])

 (deftest receive-whisper-messages-test
  (testing "an error is reported"