[slow sign in] Adding symkeys and filters in batch

This commit changes the way how keys are restored:
1. batch of `ssh.addSymKey` requests for all sym keys is sent at once
2. `:status-im.transport.core/sym-keys-added` event is dispatched with
   results of all successful `ssh.addSymKey` calls
3. filter is created via `ssh.newMessageFilter`
4. `:shh.callback/filters-added` event is dispatched with all added
   filters as a parameter
5. profit

In ideal case only 2 `re-frame` events are dispatched.

src/status_im/accounts/login/core.cljs

@@ -101,7 +101,7 @@
           :web3/fetch-node-version  [web3
                                      #(re-frame/dispatch
                                        [:web3/fetch-node-version-callback %])]}
-         (protocol/initialize-protocol address)
+         (protocol/initialize-protocol)
          #(when-not platform/desktop?
             (initialize-wallet %)))
         (account-and-db-password-do-not-match cofx error)))))

src/status_im/protocol/core.cljs

@@ -47,7 +47,7 @@
 
 (fx/defn initialize-protocol
   [{:data-store/keys [transport mailserver-topics mailservers]
-    :keys [db web3] :as cofx} address]
+    :keys [db web3] :as cofx}]
   (let [network (get-in db [:account/account :network])
         network-id (str (get-in db [:account/account :networks network :config :NetworkId]))]
     (fx/merge cofx
@@ -59,7 +59,7 @@
                                                  :network-id network-id}}
               (start-check-sync-state)
               (mailserver/initialize-mailserver mailservers)
-              (transport/init-whisper address))))
+              (transport/init-whisper))))
 
 (fx/defn handle-close-app-confirmed
   [_]

src/status_im/transport/core.cljs

@@ -17,22 +17,23 @@
   - adding fixed shh discovery filter
   - restoring existing symetric keys along with their unique filters
   - (optionally) initializing mailserver"
-  [{:keys [db web3] :as cofx} current-account-id]
+  [{:keys [db web3] :as cofx}]
   (log/debug :init-whisper)
   (when-let [public-key (get-in db [:account/account :public-key])]
-
+    (let [topic (transport.utils/get-topic constants/contact-discovery)]
-    (let [sym-key-added-callback (fn [chat-id sym-key sym-key-id]
-                                   (re-frame/dispatch [::sym-key-added {:chat-id    chat-id
-                                                                        :sym-key    sym-key
-                                                                        :sym-key-id sym-key-id}]))
-          topic (transport.utils/get-topic constants/contact-discovery)]
       (fx/merge cofx
-                {:shh/add-discovery-filter {:web3           web3
+                {:shh/add-discovery-filter
-                                            :private-key-id public-key
+                 {:web3           web3
-                                            :topic topic}
+                  :private-key-id public-key
-                 :shh/restore-sym-keys {:web3       web3
+                  :topic          topic}
-                                        :transport  (filter (comp :topic second) (:transport/chats db))
+                 :shh/restore-sym-keys-batch
-                                        :on-success sym-key-added-callback}}
+                 {:web3       web3
+                  :transport  (keep (fn [[chat-id {:keys [topic sym-key]
+                                                   :as   chat}]]
+                                      (when (and topic sym-key)
+                                        (assoc chat :chat-id chat-id)))
+                                    (:transport/chats db))
+                  :on-success #(re-frame/dispatch [::sym-keys-added %])}}
                 (mailserver/connect-to-mailserver)
                 (message/resend-contact-messages [])))))
 
@@ -42,17 +43,27 @@
 ;;it saves the sym-key-id in app-db to send messages later
 ;;and starts a filter to receive messages
 (handlers/register-handler-fx
- ::sym-key-added
+ ::sym-keys-added
- (fn [{:keys [db]} [_ {:keys [chat-id sym-key sym-key-id]}]]
+ (fn [{:keys [db]} [_ keys]]
+   (log/debug "PERF" ::sym-keys-added (count keys))
    (let [web3 (:web3 db)
-         {:keys [topic] :as chat} (get-in db [:transport/chats chat-id])]
+         chats (:transport/chats db)
-     {:db (assoc-in db [:transport/chats chat-id :sym-key-id] sym-key-id)
+         {:keys [updated-chats filters]}
-      :data-store/tx   [(transport-store/save-transport-tx {:chat-id chat-id
+         (reduce
-                                                            :chat    (assoc chat :sym-key-id sym-key-id)})]
+          (fn [{:keys [updated-chats filters]} chat]
-      :shh/add-filter {:web3       web3
+            (let [{:keys [chat-id sym-key-id]} chat
-                       :sym-key-id sym-key-id
+                  {:keys [topic]} (get updated-chats chat-id)]
-                       :topic      topic
+              {:updated-chats (assoc-in updated-chats
-                       :chat-id    chat-id}})))
+                                        [chat-id :sym-key-id] sym-key-id)
+               :filters       (conj filters {:sym-key-id sym-key-id
+                                             :topic      topic
+                                             :chat-id    chat-id})}))
+          {:updated-chats chats
+           :filters       []}
+          keys)]
+     {:db              (assoc db :transport/chats updated-chats)
+      :shh/add-filters {:web3    web3
+                        :filters filters}})))
 
 ;;TODO (yenda) uncomment and rework once go implements persistence
 #_(doseq [[chat-id {:keys [sym-key-id topic] :as chat}] transport]

src/status_im/transport/filters.cljs

@@ -18,7 +18,7 @@
   (log/debug :stop-watching filter))
 
 (defn add-filter!
-  [web3 {:keys [topics to] :as options} callback chat-id]
+  [web3 {:keys [topics] :as options} callback chat-id]
   (let [options  (assoc options :allowP2P true)]
     (log/debug :add-filter options)
     (when-let [filter (.newMessageFilter (utils/shh web3)
@@ -27,6 +27,24 @@
                                          #(log/warn :add-filter-error (.stringify js/JSON (clj->js options)) %))]
       (re-frame/dispatch [:shh.callback/filter-added (first topics) chat-id filter]))))
 
+(defn add-filters!
+  [web3 filters]
+  (log/debug "PERF" :add-filters (first filters))
+  (re-frame/dispatch
+   [:shh.callback/filters-added
+    (keep
+     (fn [{:keys [options callback chat-id]}]
+       (when-let [filter (.newMessageFilter
+                          (utils/shh web3)
+                          (clj->js (assoc options :allowP2P true))
+                          callback
+                          #(log/warn :add-filter-error
+                                     (.stringify js/JSON (clj->js options)) %))]
+         {:topic   (first (:topics options))
+          :chat-id chat-id
+          :filter  filter}))
+     filters)]))
+
 (re-frame/reg-fx
  :shh/add-filter
  (fn [{:keys [web3 sym-key-id topic chat-id]}]
@@ -36,6 +54,24 @@
                     (re-frame/dispatch [:transport/messages-received js-error js-message chat-id]))]
      (add-filter! web3 params callback chat-id))))
 
+(re-frame/reg-fx
+ :shh/add-filters
+ (fn [{:keys [web3 filters]}]
+   (log/debug "PERF" :shh/add-filters)
+   (let [filters
+         (reduce
+          (fn [acc {:keys [sym-key-id topic chat-id]}]
+            (conj acc
+                  {:options  {:topics   [topic]
+                              :symKeyID sym-key-id}
+                   :callback (fn [js-error js-message]
+                               (re-frame/dispatch [:transport/messages-received
+                                                   js-error js-message chat-id]))
+                   :chat-id  chat-id}))
+          []
+          filters)]
+     (add-filters! web3 filters))))
+
 (re-frame/reg-fx
  :shh/add-discovery-filter
  (fn [{:keys [web3 private-key-id topic]}]
@@ -55,6 +91,7 @@
 (handlers/register-handler-fx
  :shh.callback/filter-added
  (fn [{:keys [db] :as cofx} [_ topic chat-id filter]]
+   (log/debug "PERF" :shh.callback/filter-added)
    (fx/merge cofx
              {:db (assoc-in db [:transport/filters chat-id] filter)}
              (mailserver/reset-request-to)
@@ -62,6 +99,27 @@
                                                   :chat-id chat-id})
              (mailserver/process-next-messages-request))))
 
+(fx/defn add-filter
+  [{:keys [db]} chat-id filter]
+  {:db (assoc-in db [:transport/filters chat-id] filter)})
+
+(handlers/register-handler-fx
+ :shh.callback/filters-added
+ (fn [cofx [_ filters]]
+   (log/debug "PERF" :shh.callback/filters-added)
+   (let [filters-fx-fns
+         (mapcat
+          (fn [{:keys [topic chat-id filter]}]
+            [(add-filter chat-id filter)
+             (mailserver/upsert-mailserver-topic {:topic   topic
+                                                  :chat-id chat-id})])
+          filters)]
+     (apply fx/merge cofx
+            (mailserver/reset-request-to)
+            (concat
+             filters-fx-fns
+             [(mailserver/process-next-messages-request)])))))
+
 (handlers/register-handler-fx
  :shh.callback/filter-removed
  (fn [{:keys [db]} [_ chat-id]]

src/status_im/transport/shh.cljs

@@ -151,6 +151,31 @@
                              (on-success resp)
                              (on-error err))))))
 
+(defn add-sym-keys-batch
+  [{:keys [web3 keys on-success on-error]}]
+  (let [batch    (.createBatch web3)
+        results  (atom [])
+        total    (count keys)
+        counter  (atom 0)
+        callback (fn [chat-id sym-key err resp]
+                   (swap! counter inc)
+                   (if err
+                     (on-error err)
+                     (swap! results conj {:chat-id    chat-id
+                                          :sym-key    sym-key
+                                          :sym-key-id resp}))
+                   (when (= @counter total)
+                     (on-success @results)))]
+    (log/debug "PERF" :add-sym-key-batch total)
+    (doseq [{:keys [chat-id sym-key]} keys]
+      (let [request (.. web3
+                        -shh
+                        -addSymKey
+                        (request sym-key
+                                 (partial callback chat-id sym-key)))]
+        (.add batch request)))
+    (.execute batch)))
+
 (defn get-sym-key
   [{:keys [web3 sym-key-id on-success on-error]}]
   (.. web3
@@ -174,15 +199,13 @@
 
 ;;TODO (yenda) remove once go implements persistence
 (re-frame/reg-fx
- :shh/restore-sym-keys
+ :shh/restore-sym-keys-batch
  (fn [{:keys [web3 transport on-success]}]
-   (doseq [[chat-id {:keys [sym-key]}] transport]
+   (log/debug "PERF" :shh/restore-sym-keys-batch (.now js/Date))
-     (when sym-key
+   (add-sym-keys-batch {:web3       web3
-       (add-sym-key {:web3       web3
+                        :keys       transport
-                     :sym-key    sym-key
+                        :on-success on-success
-                     :on-success (fn [sym-key-id]
+                        :on-error   log-error})))
-                                   (on-success chat-id sym-key sym-key-id))
-                     :on-error   log-error})))))
 
 (defn add-new-sym-key [{:keys [web3 sym-key on-success]}]
   (add-sym-key {:web3       web3

test/cljs/status_im/test/transport/core.cljs

@@ -6,16 +6,19 @@
 
 (deftest init-whisper
   (let [cofx {:db {:account/account {:public-key "1"}
-                   :transport/chats {"1" {:topic "topic-1"}
+                   :transport/chats {"1" {:topic   "topic-1"
+                                          :sym-key "sk1"}
                                      "2" {}
-                                     "3" {:topic "topic-3"}}
+                                     "3" {:topic   "topic-3"
-                   :semaphores #{}}}]
+                                          :sym-key "sk3"}}
+                   :semaphores      #{}}}]
     (testing "it adds the discover filter"
       (is (= {:web3 nil :private-key-id "1" :topic "0xf8946aac"}
-             (:shh/add-discovery-filter (transport/init-whisper cofx "user-address")))))
+             (:shh/add-discovery-filter (transport/init-whisper cofx)))))
     (testing "it restores the sym-keys"
-      (is (= [["1" {:topic "topic-1"}] ["3" {:topic "topic-3"}]]
+      (is (= [{:topic "topic-1", :sym-key "sk1", :chat-id "1"}
-             (-> (transport/init-whisper cofx  "user-address") :shh/restore-sym-keys :transport))))
+              {:topic "topic-3", :sym-key "sk3", :chat-id "3"}]
+             (-> (transport/init-whisper cofx) :shh/restore-sym-keys-batch :transport))))
     (testing "custom mailservers"
       (let [ms-1            {:id "1"
                              :fleet :eth.beta
@@ -48,7 +51,7 @@
                                     ms-3])]
         (is (= expected-mailservers
                (-> (get-in
-                    (protocol/initialize-protocol cofx-with-ms "user-address")
+                    (protocol/initialize-protocol cofx-with-ms)
                     [:db :mailserver/mailservers])
                    (update-in [:eth.beta "1"] dissoc :generating-sym-key?)
                    (update-in [:eth.beta "2"] dissoc :generating-sym-key?)