From e3cc50c7d76723840bf669970a24026248bcf5a8 Mon Sep 17 00:00:00 2001 From: Roman Volosovskyi Date: Fri, 14 Dec 2018 11:18:53 +0200 Subject: [PATCH] [slow sign in] Adding symkeys and filters in batch This commit changes the way how keys are restored: 1. batch of `ssh.addSymKey` requests for all sym keys is sent at once 2. `:status-im.transport.core/sym-keys-added` event is dispatched with results of all successful `ssh.addSymKey` calls 3. filter is created via `ssh.newMessageFilter` 4. `:shh.callback/filters-added` event is dispatched with all added filters as a parameter 5. profit In ideal case only 2 `re-frame` events are dispatched. --- src/status_im/accounts/login/core.cljs | 2 +- src/status_im/protocol/core.cljs | 4 +- src/status_im/transport/core.cljs | 57 +++++++++++-------- src/status_im/transport/filters.cljs | 60 +++++++++++++++++++- src/status_im/transport/shh.cljs | 39 ++++++++++--- test/cljs/status_im/test/transport/core.cljs | 17 +++--- 6 files changed, 137 insertions(+), 42 deletions(-) diff --git a/src/status_im/accounts/login/core.cljs b/src/status_im/accounts/login/core.cljs index f6e7630919..f22f4e616e 100644 --- a/src/status_im/accounts/login/core.cljs +++ b/src/status_im/accounts/login/core.cljs @@ -101,7 +101,7 @@ :web3/fetch-node-version [web3 #(re-frame/dispatch [:web3/fetch-node-version-callback %])]} - (protocol/initialize-protocol address) + (protocol/initialize-protocol) #(when-not platform/desktop? (initialize-wallet %))) (account-and-db-password-do-not-match cofx error))))) diff --git a/src/status_im/protocol/core.cljs b/src/status_im/protocol/core.cljs index 7f92cbf048..dbb56d48d6 100644 --- a/src/status_im/protocol/core.cljs +++ b/src/status_im/protocol/core.cljs @@ -47,7 +47,7 @@ (fx/defn initialize-protocol [{:data-store/keys [transport mailserver-topics mailservers] - :keys [db web3] :as cofx} address] + :keys [db web3] :as cofx}] (let [network (get-in db [:account/account :network]) network-id (str (get-in db [:account/account :networks network :config :NetworkId]))] (fx/merge cofx @@ -59,7 +59,7 @@ :network-id network-id}} (start-check-sync-state) (mailserver/initialize-mailserver mailservers) - (transport/init-whisper address)))) + (transport/init-whisper)))) (fx/defn handle-close-app-confirmed [_] diff --git a/src/status_im/transport/core.cljs b/src/status_im/transport/core.cljs index f4f1b4318c..7dd0a237d9 100644 --- a/src/status_im/transport/core.cljs +++ b/src/status_im/transport/core.cljs @@ -17,22 +17,23 @@ - adding fixed shh discovery filter - restoring existing symetric keys along with their unique filters - (optionally) initializing mailserver" - [{:keys [db web3] :as cofx} current-account-id] + [{:keys [db web3] :as cofx}] (log/debug :init-whisper) (when-let [public-key (get-in db [:account/account :public-key])] - - (let [sym-key-added-callback (fn [chat-id sym-key sym-key-id] - (re-frame/dispatch [::sym-key-added {:chat-id chat-id - :sym-key sym-key - :sym-key-id sym-key-id}])) - topic (transport.utils/get-topic constants/contact-discovery)] + (let [topic (transport.utils/get-topic constants/contact-discovery)] (fx/merge cofx - {:shh/add-discovery-filter {:web3 web3 - :private-key-id public-key - :topic topic} - :shh/restore-sym-keys {:web3 web3 - :transport (filter (comp :topic second) (:transport/chats db)) - :on-success sym-key-added-callback}} + {:shh/add-discovery-filter + {:web3 web3 + :private-key-id public-key + :topic topic} + :shh/restore-sym-keys-batch + {:web3 web3 + :transport (keep (fn [[chat-id {:keys [topic sym-key] + :as chat}]] + (when (and topic sym-key) + (assoc chat :chat-id chat-id))) + (:transport/chats db)) + :on-success #(re-frame/dispatch [::sym-keys-added %])}} (mailserver/connect-to-mailserver) (message/resend-contact-messages []))))) @@ -42,17 +43,27 @@ ;;it saves the sym-key-id in app-db to send messages later ;;and starts a filter to receive messages (handlers/register-handler-fx - ::sym-key-added - (fn [{:keys [db]} [_ {:keys [chat-id sym-key sym-key-id]}]] + ::sym-keys-added + (fn [{:keys [db]} [_ keys]] + (log/debug "PERF" ::sym-keys-added (count keys)) (let [web3 (:web3 db) - {:keys [topic] :as chat} (get-in db [:transport/chats chat-id])] - {:db (assoc-in db [:transport/chats chat-id :sym-key-id] sym-key-id) - :data-store/tx [(transport-store/save-transport-tx {:chat-id chat-id - :chat (assoc chat :sym-key-id sym-key-id)})] - :shh/add-filter {:web3 web3 - :sym-key-id sym-key-id - :topic topic - :chat-id chat-id}}))) + chats (:transport/chats db) + {:keys [updated-chats filters]} + (reduce + (fn [{:keys [updated-chats filters]} chat] + (let [{:keys [chat-id sym-key-id]} chat + {:keys [topic]} (get updated-chats chat-id)] + {:updated-chats (assoc-in updated-chats + [chat-id :sym-key-id] sym-key-id) + :filters (conj filters {:sym-key-id sym-key-id + :topic topic + :chat-id chat-id})})) + {:updated-chats chats + :filters []} + keys)] + {:db (assoc db :transport/chats updated-chats) + :shh/add-filters {:web3 web3 + :filters filters}}))) ;;TODO (yenda) uncomment and rework once go implements persistence #_(doseq [[chat-id {:keys [sym-key-id topic] :as chat}] transport] diff --git a/src/status_im/transport/filters.cljs b/src/status_im/transport/filters.cljs index 9094d35072..5185df762e 100644 --- a/src/status_im/transport/filters.cljs +++ b/src/status_im/transport/filters.cljs @@ -18,7 +18,7 @@ (log/debug :stop-watching filter)) (defn add-filter! - [web3 {:keys [topics to] :as options} callback chat-id] + [web3 {:keys [topics] :as options} callback chat-id] (let [options (assoc options :allowP2P true)] (log/debug :add-filter options) (when-let [filter (.newMessageFilter (utils/shh web3) @@ -27,6 +27,24 @@ #(log/warn :add-filter-error (.stringify js/JSON (clj->js options)) %))] (re-frame/dispatch [:shh.callback/filter-added (first topics) chat-id filter])))) +(defn add-filters! + [web3 filters] + (log/debug "PERF" :add-filters (first filters)) + (re-frame/dispatch + [:shh.callback/filters-added + (keep + (fn [{:keys [options callback chat-id]}] + (when-let [filter (.newMessageFilter + (utils/shh web3) + (clj->js (assoc options :allowP2P true)) + callback + #(log/warn :add-filter-error + (.stringify js/JSON (clj->js options)) %))] + {:topic (first (:topics options)) + :chat-id chat-id + :filter filter})) + filters)])) + (re-frame/reg-fx :shh/add-filter (fn [{:keys [web3 sym-key-id topic chat-id]}] @@ -36,6 +54,24 @@ (re-frame/dispatch [:transport/messages-received js-error js-message chat-id]))] (add-filter! web3 params callback chat-id)))) +(re-frame/reg-fx + :shh/add-filters + (fn [{:keys [web3 filters]}] + (log/debug "PERF" :shh/add-filters) + (let [filters + (reduce + (fn [acc {:keys [sym-key-id topic chat-id]}] + (conj acc + {:options {:topics [topic] + :symKeyID sym-key-id} + :callback (fn [js-error js-message] + (re-frame/dispatch [:transport/messages-received + js-error js-message chat-id])) + :chat-id chat-id})) + [] + filters)] + (add-filters! web3 filters)))) + (re-frame/reg-fx :shh/add-discovery-filter (fn [{:keys [web3 private-key-id topic]}] @@ -55,6 +91,7 @@ (handlers/register-handler-fx :shh.callback/filter-added (fn [{:keys [db] :as cofx} [_ topic chat-id filter]] + (log/debug "PERF" :shh.callback/filter-added) (fx/merge cofx {:db (assoc-in db [:transport/filters chat-id] filter)} (mailserver/reset-request-to) @@ -62,6 +99,27 @@ :chat-id chat-id}) (mailserver/process-next-messages-request)))) +(fx/defn add-filter + [{:keys [db]} chat-id filter] + {:db (assoc-in db [:transport/filters chat-id] filter)}) + +(handlers/register-handler-fx + :shh.callback/filters-added + (fn [cofx [_ filters]] + (log/debug "PERF" :shh.callback/filters-added) + (let [filters-fx-fns + (mapcat + (fn [{:keys [topic chat-id filter]}] + [(add-filter chat-id filter) + (mailserver/upsert-mailserver-topic {:topic topic + :chat-id chat-id})]) + filters)] + (apply fx/merge cofx + (mailserver/reset-request-to) + (concat + filters-fx-fns + [(mailserver/process-next-messages-request)]))))) + (handlers/register-handler-fx :shh.callback/filter-removed (fn [{:keys [db]} [_ chat-id]] diff --git a/src/status_im/transport/shh.cljs b/src/status_im/transport/shh.cljs index ef74eeec83..1b026795cb 100644 --- a/src/status_im/transport/shh.cljs +++ b/src/status_im/transport/shh.cljs @@ -151,6 +151,31 @@ (on-success resp) (on-error err)))))) +(defn add-sym-keys-batch + [{:keys [web3 keys on-success on-error]}] + (let [batch (.createBatch web3) + results (atom []) + total (count keys) + counter (atom 0) + callback (fn [chat-id sym-key err resp] + (swap! counter inc) + (if err + (on-error err) + (swap! results conj {:chat-id chat-id + :sym-key sym-key + :sym-key-id resp})) + (when (= @counter total) + (on-success @results)))] + (log/debug "PERF" :add-sym-key-batch total) + (doseq [{:keys [chat-id sym-key]} keys] + (let [request (.. web3 + -shh + -addSymKey + (request sym-key + (partial callback chat-id sym-key)))] + (.add batch request))) + (.execute batch))) + (defn get-sym-key [{:keys [web3 sym-key-id on-success on-error]}] (.. web3 @@ -174,15 +199,13 @@ ;;TODO (yenda) remove once go implements persistence (re-frame/reg-fx - :shh/restore-sym-keys + :shh/restore-sym-keys-batch (fn [{:keys [web3 transport on-success]}] - (doseq [[chat-id {:keys [sym-key]}] transport] - (when sym-key - (add-sym-key {:web3 web3 - :sym-key sym-key - :on-success (fn [sym-key-id] - (on-success chat-id sym-key sym-key-id)) - :on-error log-error}))))) + (log/debug "PERF" :shh/restore-sym-keys-batch (.now js/Date)) + (add-sym-keys-batch {:web3 web3 + :keys transport + :on-success on-success + :on-error log-error}))) (defn add-new-sym-key [{:keys [web3 sym-key on-success]}] (add-sym-key {:web3 web3 diff --git a/test/cljs/status_im/test/transport/core.cljs b/test/cljs/status_im/test/transport/core.cljs index 66203c848a..ebeaabb519 100644 --- a/test/cljs/status_im/test/transport/core.cljs +++ b/test/cljs/status_im/test/transport/core.cljs @@ -6,16 +6,19 @@ (deftest init-whisper (let [cofx {:db {:account/account {:public-key "1"} - :transport/chats {"1" {:topic "topic-1"} + :transport/chats {"1" {:topic "topic-1" + :sym-key "sk1"} "2" {} - "3" {:topic "topic-3"}} - :semaphores #{}}}] + "3" {:topic "topic-3" + :sym-key "sk3"}} + :semaphores #{}}}] (testing "it adds the discover filter" (is (= {:web3 nil :private-key-id "1" :topic "0xf8946aac"} - (:shh/add-discovery-filter (transport/init-whisper cofx "user-address"))))) + (:shh/add-discovery-filter (transport/init-whisper cofx))))) (testing "it restores the sym-keys" - (is (= [["1" {:topic "topic-1"}] ["3" {:topic "topic-3"}]] - (-> (transport/init-whisper cofx "user-address") :shh/restore-sym-keys :transport)))) + (is (= [{:topic "topic-1", :sym-key "sk1", :chat-id "1"} + {:topic "topic-3", :sym-key "sk3", :chat-id "3"}] + (-> (transport/init-whisper cofx) :shh/restore-sym-keys-batch :transport)))) (testing "custom mailservers" (let [ms-1 {:id "1" :fleet :eth.beta @@ -48,7 +51,7 @@ ms-3])] (is (= expected-mailservers (-> (get-in - (protocol/initialize-protocol cofx-with-ms "user-address") + (protocol/initialize-protocol cofx-with-ms) [:db :mailserver/mailservers]) (update-in [:eth.beta "1"] dissoc :generating-sym-key?) (update-in [:eth.beta "2"] dissoc :generating-sym-key?)