From fe7c7088db52993162a0514bdf29706e9d65bcae Mon Sep 17 00:00:00 2001 From: Roman Volosovskyi Date: Wed, 23 Jan 2019 20:15:07 +0200 Subject: [PATCH] Use partitioned topic for discovery Currently we use a single topic for discovery. This provides the best obscurity at the cost of bandwidth, as a message sent on the discovery topic will be received by any peer. This PR changes this behavior and start listening on a partitioned topic. Each pk will be hashed to a limited number of topics. Everytime someone is in a conversation with someone from another topic they will have to listen as well to avoid loosing obscurity, because we only forward messages that we also advertise in the bloom filter. The choice for the number of partitions depends on 2 factors: 1) The expected number of users using the network 2) The average number of contacts each user Any change to the discovery topic will need to be split across 3 releases, to avoid breaking compatibility: 1) Listen to the new and old topic, publish to the old topic 2) Listen to the new and old topic, publish to the new topic 3) Listen to the new topic, publish to the new topic This is step 1. --- src/status_im/contact/core.cljs | 26 ++++++- .../realm/schemas/account/core.cljs | 19 ++++- .../realm/schemas/account/migrations.cljs | 14 ++++ .../realm/schemas/account/transport.cljs | 4 + src/status_im/events.cljs | 12 ++- src/status_im/group_chats/core.cljs | 3 +- src/status_im/mailserver/core.cljs | 8 +- src/status_im/transport/chat/core.cljs | 10 ++- src/status_im/transport/core.cljs | 59 ++++++++++----- src/status_im/transport/db.cljs | 25 ++++--- src/status_im/transport/filters.cljs | 74 ++++++++----------- src/status_im/transport/impl/send.cljs | 8 +- src/status_im/transport/message/contact.cljs | 6 +- src/status_im/transport/message/protocol.cljs | 16 ++-- .../transport/message/public_chat.cljs | 24 +++--- .../transport/partitioned_topic.cljs | 54 ++++++++++++++ src/status_im/transport/shh.cljs | 11 ++- src/status_im/utils/config.cljs | 1 + .../cljs/status_im/test/group_chats/core.cljs | 2 +- test/cljs/status_im/test/transport/core.cljs | 10 ++- .../test/transport/partitioned_topic.cljs | 38 ++++++++++ 21 files changed, 303 insertions(+), 121 deletions(-) create mode 100644 src/status_im/transport/partitioned_topic.cljs create mode 100644 test/cljs/status_im/test/transport/partitioned_topic.cljs diff --git a/src/status_im/contact/core.cljs b/src/status_im/contact/core.cljs index 2b5b0b267d..beeb3dbd76 100644 --- a/src/status_im/contact/core.cljs +++ b/src/status_im/contact/core.cljs @@ -10,7 +10,9 @@ [status-im.ui.screens.add-new.new-chat.db :as new-chat.db] [status-im.ui.screens.navigation :as navigation] [status-im.utils.fx :as fx] - [status-im.utils.utils :as utils])) + [status-im.utils.utils :as utils] + [status-im.transport.partitioned-topic :as transport.topic] + [status-im.utils.config :as config])) (fx/defn load-contacts [{:keys [db all-contacts]}] @@ -65,6 +67,24 @@ (add-new-contact contact) (send-contact-request contact))))) +(fx/defn add-contacts-filter [{:keys [db]} public-key action] + (when (not= (get-in db [:account/account :public-key]) public-key) + (let [current-public-key (get-in db [:account/account :public-key])] + {:db + (cond-> db + config/partitioned-topic-enabled? + (assoc :filters/after-adding-discovery-filter + {:action action + :public-key public-key})) + + :shh/add-discovery-filters + {:web3 (:web3 db) + :private-key-id current-public-key + :topics [{:topic (transport.topic/partitioned-topic-hash public-key) + :chat-id public-key + :minPow 1 + :callback (constantly nil)}]}}))) + (fx/defn add-tag "add a tag to the contact" [{:keys [db] :as cofx}] @@ -148,7 +168,9 @@ :on-dismiss #(re-frame/dispatch [:navigate-to-clean :home])}} (fx/merge cofx fx - (add-contact-and-open-chat contact-identity))))) + (if config/partitioned-topic-enabled? + (add-contacts-filter contact-identity :add-contact-and-open-chat) + (add-contact-and-open-chat contact-identity)))))) (fx/defn open-contact-toggle-list [{:keys [db :as cofx]}] diff --git a/src/status_im/data_store/realm/schemas/account/core.cljs b/src/status_im/data_store/realm/schemas/account/core.cljs index bf35ecda20..6cb761e197 100644 --- a/src/status_im/data_store/realm/schemas/account/core.cljs +++ b/src/status_im/data_store/realm/schemas/account/core.cljs @@ -385,6 +385,20 @@ dapp-permissions/v9 contact-recovery/v1]) +(def v35 [chat/v14 + transport/v8 + contact/v3 + message/v9 + mailserver/v11 + mailserver-topic/v1 + user-status/v2 + membership-update/v1 + installation/v3 + local-storage/v1 + browser/v8 + dapp-permissions/v9 + contact-recovery/v1]) + ;; put schemas ordered by version (def schemas [{:schema v1 :schemaVersion 1 @@ -487,4 +501,7 @@ :migration (constantly nil)} {:schema v34 :schemaVersion 34 - :migration migrations/v34}]) + :migration migrations/v34} + {:schema v35 + :schemaVersion 35 + :migration migrations/v35}]) diff --git a/src/status_im/data_store/realm/schemas/account/migrations.cljs b/src/status_im/data_store/realm/schemas/account/migrations.cljs index ff8cf804d2..8635b24830 100644 --- a/src/status_im/data_store/realm/schemas/account/migrations.cljs +++ b/src/status_im/data_store/realm/schemas/account/migrations.cljs @@ -346,3 +346,17 @@ (let [chat (aget chats i) chat-id (aget chat "chat-id")] (aset chat "group-chat-local-version" 0))))) + +(defn one-to-one? [chat-id] + (re-matches #"^0x[0-9a-fA-F]+$" chat-id)) + +(defn v35 [old-realm new-realm] + (log/debug "migrating transport chats") + (let [old-chats (.objects old-realm "transport") + new-chats (.objects new-realm "transport")] + (dotimes [i (.-length old-chats)] + (let [old-chat (aget old-chats i) + new-chat (aget new-chats i) + chat-id (aget old-chat "chat-id")] + (when (one-to-one? chat-id) + (aset new-chat "one-to-one" true)))))) diff --git a/src/status_im/data_store/realm/schemas/account/transport.cljs b/src/status_im/data_store/realm/schemas/account/transport.cljs index 763837b2b1..8def1864ed 100644 --- a/src/status_im/data_store/realm/schemas/account/transport.cljs +++ b/src/status_im/data_store/realm/schemas/account/transport.cljs @@ -64,3 +64,7 @@ ;;TODO (yenda) remove once go implements persistence :sym-key {:type :string :optional true}}}) + +(def v8 (assoc-in v7 [:properties :one-to-one] + {:type :bool + :optional true})) diff --git a/src/status_im/events.cljs b/src/status_im/events.cljs index 8672763d73..a2b0c12968 100644 --- a/src/status_im/events.cljs +++ b/src/status_im/events.cljs @@ -50,7 +50,8 @@ [status-im.chat.models.loading :as chat-loading] [status-im.node.core :as node] [cljs.reader :as edn] - [status-im.stickers.core :as stickers])) + [status-im.stickers.core :as stickers] + [status-im.utils.config :as config])) ;; init module @@ -1463,7 +1464,9 @@ :contact.ui/add-to-contact-pressed [(re-frame/inject-cofx :random-id-generator)] (fn [cofx [_ public-key]] - (contact/add-contact cofx public-key))) + (if config/partitioned-topic-enabled? + (contact/add-contacts-filter cofx public-key :add-contact) + (contact/add-contact cofx public-key)))) (handlers/register-handler-fx :contact.ui/close-contact-pressed @@ -1476,6 +1479,11 @@ (fn [cofx [_ _ contact-identity]] (contact/handle-qr-code cofx contact-identity))) +(handlers/register-handler-fx + :contact/filters-added + (fn [cofx [_ contact-identity]] + (contact/add-contact-and-open-chat cofx contact-identity))) + (handlers/register-handler-fx :contact.ui/start-group-chat-pressed (fn [{:keys [db] :as cofx} _] diff --git a/src/status_im/group_chats/core.cljs b/src/status_im/group_chats/core.cljs index d5dc7d0ed6..94eb526cf6 100644 --- a/src/status_im/group_chats/core.cljs +++ b/src/status_im/group_chats/core.cljs @@ -17,6 +17,7 @@ [status-im.transport.message.protocol :as protocol] [status-im.transport.message.group-chat :as message.group-chat] [status-im.transport.message.public-chat :as transport.public-chat] + [status-im.transport.partitioned-topic :as transport.topic] [status-im.transport.chat.core :as transport.chat] [status-im.utils.fx :as fx] [status-im.chat.models :as models.chat] @@ -149,7 +150,7 @@ {:public-key member :chat chat-id} {:public-key member - :chat constants/contact-discovery})) + :chat (transport.topic/public-key->discovery-topic member)})) members)] (fx/merge cofx diff --git a/src/status_im/mailserver/core.cljs b/src/status_im/mailserver/core.cljs index 05a886f5a3..cefa0f7042 100644 --- a/src/status_im/mailserver/core.cljs +++ b/src/status_im/mailserver/core.cljs @@ -17,7 +17,8 @@ [status-im.i18n :as i18n] [status-im.utils.handlers :as handlers] [status-im.accounts.update.core :as accounts.update] - [status-im.ui.screens.navigation :as navigation])) + [status-im.ui.screens.navigation :as navigation] + [status-im.transport.partitioned-topic :as transport.topic])) ;; How do mailserver work ? ;; @@ -515,8 +516,9 @@ :mailserver-topic mailserver-topic})]})))) (fx/defn fetch-history [{:keys [db] :as cofx} chat-id] - (let [topic (or (get-in db [:transport/chats chat-id :topic]) - (transport.utils/get-topic constants/contact-discovery)) + (let [public-key (accounts.db/current-public-key cofx) + topic (or (get-in db [:transport/chats chat-id :topic]) + (transport.topic/public-key->discovery-topic-hash public-key)) {:keys [chat-ids last-request] :as current-mailserver-topic} (get-in db [:mailserver/topics topic] {:chat-ids #{}})] (let [mailserver-topic (-> current-mailserver-topic diff --git a/src/status_im/transport/chat/core.cljs b/src/status_im/transport/chat/core.cljs index d690672092..7dc92f6680 100644 --- a/src/status_im/transport/chat/core.cljs +++ b/src/status_im/transport/chat/core.cljs @@ -5,10 +5,12 @@ (fx/defn remove-transport-chat [{:keys [db]} chat-id] - {:db (update db :transport/chats dissoc chat-id) - :data-store/tx [(transport-store/delete-transport-tx chat-id)] - :shh/remove-filter {:chat-id chat-id - :filter (get-in db [:transport/filters chat-id])}}) + {:db (update db :transport/chats dissoc chat-id) + :data-store/tx [(transport-store/delete-transport-tx chat-id)] + :shh/remove-filters {:filters (map + (fn [filter] + [chat-id filter]) + (get-in db [:transport/filters chat-id]))}}) (fx/defn unsubscribe-from-chat "Unsubscribe from chat on transport layer" diff --git a/src/status_im/transport/core.cljs b/src/status_im/transport/core.cljs index f2c4e7b19b..a5e449cbcb 100644 --- a/src/status_im/transport/core.cljs +++ b/src/status_im/transport/core.cljs @@ -2,15 +2,29 @@ status-im.transport.core (:require status-im.transport.filters [re-frame.core :as re-frame] - [status-im.constants :as constants] - [status-im.data-store.transport :as transport-store] [status-im.mailserver.core :as mailserver] [status-im.transport.message.core :as message] - [status-im.transport.shh :as shh] - [status-im.transport.utils :as transport.utils] + [status-im.transport.partitioned-topic :as transport.topic] [status-im.utils.fx :as fx] [status-im.utils.handlers :as handlers] - [taoensso.timbre :as log])) + [taoensso.timbre :as log] + status-im.transport.shh + [status-im.utils.config :as config])) + +(defn get-public-key-topics [chats] + (keep (fn [[chat-id {:keys [topic sym-key one-to-one]}]] + (cond (and (not sym-key) topic) + {:topic topic + :chat-id chat-id} + + ;; we have to listen the topic to which we are going to send + ;; a message, otherwise the message will not match bloom + (and config/partitioned-topic-enabled? one-to-one) + {:topic (transport.topic/partitioned-topic-hash chat-id) + :chat-id chat-id + :minPow 1 + :callback (constantly nil)})) + chats)) (fx/defn init-whisper "Initialises whisper protocol by: @@ -19,19 +33,18 @@ - (optionally) initializing mailserver" [{:keys [db web3] :as cofx}] (when-let [public-key (get-in db [:account/account :public-key])] - (let [public-key-topics (keep (fn [[chat-id {:keys [topic sym-key]}]] - (when (and (not sym-key) - topic) - {:topic topic - :chat-id chat-id})) - (:transport/chats db)) - discovery-topic (transport.utils/get-topic constants/contact-discovery)] + (let [public-key-topics (get-public-key-topics (:transport/chats db)) + discovery-topics (transport.topic/discovery-topics public-key)] (fx/merge cofx - {:shh/add-discovery-filters {:web3 web3 - :private-key-id public-key - :topics (conj public-key-topics - {:topic discovery-topic - :chat-id :discovery-topic})} + {:shh/add-discovery-filters + {:web3 web3 + :private-key-id public-key + :topics (concat public-key-topics + (map + (fn [discovery-topic] + {:topic discovery-topic + :chat-id :discovery-topic}) + discovery-topics))} :shh/restore-sym-keys-batch {:web3 web3 @@ -59,12 +72,13 @@ (reduce (fn [{:keys [updated-chats filters]} chat] (let [{:keys [chat-id sym-key-id]} chat - {:keys [topic]} (get updated-chats chat-id)] + {:keys [topic one-to-one]} (get updated-chats chat-id)] {:updated-chats (assoc-in updated-chats [chat-id :sym-key-id] sym-key-id) :filters (conj filters {:sym-key-id sym-key-id :topic topic - :chat-id chat-id})})) + :chat-id chat-id + :one-to-one one-to-one})})) {:updated-chats chats :filters []} keys)] @@ -88,4 +102,9 @@ account A messages without this." [{:keys [db]} callback] (let [{:transport/keys [filters]} db] - {:shh/remove-filters [filters callback]})) + {:shh/remove-filters {:filter (mapcat (fn [[chat-id chat-filters]] + (map (fn [filter] + [chat-id filter]) + chat-filters)) + filters) + :callback callback}})) diff --git a/src/status_im/transport/db.cljs b/src/status_im/transport/db.cljs index 70cde0f136..45a849beb5 100644 --- a/src/status_im/transport/db.cljs +++ b/src/status_im/transport/db.cljs @@ -29,18 +29,19 @@ (spec/def :transport/chat (spec/keys :req-un [::ack ::seen ::pending-ack ::pending-send ::topic] :opt-un [::sym-key-id ::sym-key ::resend?])) (spec/def :transport/chats (spec/map-of :global/not-empty-string :transport/chat)) -(spec/def :transport/filters (spec/map-of :transport/filter-id :transport/filter)) +(spec/def :transport/filters (spec/map-of :transport/filter-id (spec/coll-of :transport/filter))) (defn create-chat "Initialize datastructure for chat representation at the transport level Currently only :topic is actually used" - [{:keys [topic resend? now]}] - {:ack [] - :seen [] - :pending-ack [] - :pending-send [] - :resend? resend? - :topic topic}) + [{:keys [topic resend? one-to-one now]}] + {:ack [] + :seen [] + :pending-ack [] + :pending-send [] + :one-to-one (boolean one-to-one) + :resend? resend? + :topic topic}) (spec/def ::profile-image :contact/photo-path) @@ -123,6 +124,10 @@ (defn all-filters-added? [{:keys [db]}] (let [filters (set (keys (get db :transport/filters))) - chats (into #{:discovery-topic} - (keys (filter #(:topic (val %)) (get db :transport/chats))))] + chats (into #{:discovery-topic} + (keys (filter (fn [[chat-id {:keys [topic one-to-one]}]] + (if one-to-one + chat-id + topic)) + (get db :transport/chats))))] (= chats filters))) diff --git a/src/status_im/transport/filters.cljs b/src/status_im/transport/filters.cljs index 964a9244a7..8248b0f72c 100644 --- a/src/status_im/transport/filters.cljs +++ b/src/status_im/transport/filters.cljs @@ -5,7 +5,12 @@ [status-im.transport.utils :as utils] [status-im.utils.fx :as fx] [status-im.utils.handlers :as handlers] - [taoensso.timbre :as log])) + [status-im.transport.partitioned-topic :as transport.topic] + [taoensso.timbre :as log] + [status-im.contact.core :as contact])) + +(defn- receive-message [chat-id js-error js-message] + (re-frame/dispatch [:transport/messages-received js-error js-message chat-id])) (defn remove-filter! [{:keys [chat-id filter success-callback?] :or {success-callback? true}}] @@ -17,16 +22,6 @@ (re-frame/dispatch [:shh.callback/filter-removed chat-id]))))) (log/debug :stop-watching filter)) -(defn add-filter! - [web3 {:keys [topics] :as options} callback chat-id] - (let [options (assoc options :allowP2P true)] - (log/debug :add-filter options) - (when-let [filter (.newMessageFilter (utils/shh web3) - (clj->js options) - callback - #(log/warn :add-filter-error (.stringify js/JSON (clj->js options)) %))] - (re-frame/dispatch [:shh.callback/filter-added (first topics) chat-id filter])))) - (defn add-filters! [web3 filters] (log/debug "PERF" :add-filters (first filters)) @@ -45,15 +40,6 @@ :filter filter})) filters)])) -(re-frame/reg-fx - :shh/add-filter - (fn [{:keys [web3 sym-key-id topic chat-id]}] - (let [params {:topics [topic] - :symKeyID sym-key-id} - callback (fn [js-error js-message] - (re-frame/dispatch [:transport/messages-received js-error js-message chat-id]))] - (add-filter! web3 params callback chat-id)))) - (re-frame/reg-fx :shh/add-filters (fn [{:keys [web3 filters]}] @@ -64,9 +50,7 @@ (conj acc {:options {:topics [topic] :symKeyID sym-key-id} - :callback (fn [js-error js-message] - (re-frame/dispatch [:transport/messages-received - js-error js-message chat-id])) + :callback (partial receive-message chat-id) :chat-id chat-id})) [] filters)] @@ -75,32 +59,26 @@ (re-frame/reg-fx :shh/add-discovery-filters (fn [{:keys [web3 private-key-id topics]}] - (let [params {:topics (mapv :topic topics) - :privateKeyID private-key-id} - callback (fn [js-error js-message] - (re-frame/dispatch [:transport/messages-received js-error js-message]))] - (doseq [{:keys [chat-id topic]} topics] - (add-filter! web3 params callback chat-id))))) - -(handlers/register-handler-fx - :shh.callback/filter-added - (fn [{:keys [db] :as cofx} [_ topic chat-id filter]] - (fx/merge cofx - {:db (assoc-in db [:transport/filters chat-id] filter)} - (mailserver/reset-request-to) - (mailserver/upsert-mailserver-topic {:topic topic - :chat-id chat-id}) - (mailserver/process-next-messages-request)))) + (let [params {:privateKeyID private-key-id}] + (add-filters! + web3 + (map (fn [{:keys [chat-id topic callback minPow]}] + {:options (cond-> (assoc params :topics [topic]) + minPow + (assoc :minPow minPow)) + :callback (or callback (partial receive-message chat-id)) + :chat-id chat-id}) topics))))) (fx/defn add-filter [{:keys [db]} chat-id filter] - {:db (assoc-in db [:transport/filters chat-id] filter)}) + {:db (update-in db [:transport/filters chat-id] conj filter)}) (handlers/register-handler-fx :shh.callback/filters-added - (fn [cofx [_ filters]] + (fn [{:keys [db] :as cofx} [_ filters]] (log/debug "PERF" :shh.callback/filters-added) - (let [filters-fx-fns + (let [{:keys [action public-key]} (:filters/after-adding-discovery-filter db) + filters-fx-fns (mapcat (fn [{:keys [topic chat-id filter]}] [(add-filter chat-id filter) @@ -108,8 +86,16 @@ :chat-id chat-id})]) filters)] (apply fx/merge cofx + {:db (dissoc db :filters/after-adding-discovery-filter)} (mailserver/reset-request-to) (concat + [(when action + (case action + :add-contact + (contact/add-contact public-key) + + :add-contact-and-open-chat + (contact/add-contact-and-open-chat public-key)))] filters-fx-fns [(mailserver/process-next-messages-request)]))))) @@ -125,10 +111,10 @@ (re-frame/reg-fx :shh/remove-filters - (fn [[filters callback]] + (fn [{:keys [filters callback]}] (doseq [[chat-id filter] filters] (when filter (remove-filter! {:chat-id chat-id :filter filter :success-callback false}))) - (callback))) + (when callback (callback)))) diff --git a/src/status_im/transport/impl/send.cljs b/src/status_im/transport/impl/send.cljs index 48e73168bb..6e694c55b2 100644 --- a/src/status_im/transport/impl/send.cljs +++ b/src/status_im/transport/impl/send.cljs @@ -34,8 +34,9 @@ nil nil)] (fx/merge cofx - (protocol/init-chat {:chat-id chat-id - :resend? "contact-request"}) + (protocol/init-chat {:chat-id chat-id + :one-to-one true + :resend? "contact-request"}) (protocol/send-with-pubkey {:chat-id chat-id :payload this :success-event [:transport/contact-message-sent chat-id]}) @@ -54,7 +55,8 @@ chat (get-in db [:transport/chats chat-id]) updated-chat (if chat (assoc chat :resend? "contact-request-confirmation") - (transport.db/create-chat {:resend? "contact-request-confirmation"}))] + (transport.db/create-chat {:resend? "contact-request-confirmation" + :one-to-one true}))] (fx/merge cofx {:db (assoc-in db [:transport/chats chat-id] updated-chat) diff --git a/src/status_im/transport/message/contact.cljs b/src/status_im/transport/message/contact.cljs index 7874500c53..14307da8e3 100644 --- a/src/status_im/transport/message/contact.cljs +++ b/src/status_im/transport/message/contact.cljs @@ -25,7 +25,7 @@ (fx/defn remove-chat-filter "Stops the filter for the given chat-id" [{:keys [db]} chat-id] - (when-let [filter (get-in db [:transport/filters chat-id])] - {:shh/remove-filter {:chat-id chat-id - :filter filter}})) + (when-let [filters (get-in db [:transport/filters chat-id])] + {:shh/remove-filters + {:filters (map (fn [filter] [chat-id filter]) filters)}})) diff --git a/src/status_im/transport/message/protocol.cljs b/src/status_im/transport/message/protocol.cljs index b2ab6d3149..020ffbd30a 100644 --- a/src/status_im/transport/message/protocol.cljs +++ b/src/status_im/transport/message/protocol.cljs @@ -3,9 +3,9 @@ (:require [cljs.spec.alpha :as spec] [status-im.accounts.db :as accounts.db] [status-im.chat.core :as chat] - [status-im.constants :as constants] [status-im.transport.db :as transport.db] [status-im.transport.utils :as transport.utils] + [status-im.transport.partitioned-topic :as transport.topic] [status-im.utils.config :as config] [status-im.utils.fx :as fx] [taoensso.timbre :as log])) @@ -28,12 +28,13 @@ "Initialises chat on protocol layer. If topic is not passed as argument it is derived from `chat-id`" [{:keys [db now]} - {:keys [chat-id topic resend?]}] + {:keys [chat-id topic one-to-one resend?]}] {:db (assoc-in db [:transport/chats chat-id] - (transport.db/create-chat {:topic topic - :resend? resend? - :now now}))}) + (transport.db/create-chat {:topic topic + :one-to-one one-to-one + :resend? resend? + :now now}))}) (defn send-public-message "Sends the payload to topic" @@ -56,7 +57,8 @@ :message (merge {:sig (accounts.db/current-public-key cofx) :symKeyID sym-key-id :payload payload - :topic topic} + :topic (or topic + (transport.topic/public-key->discovery-topic-hash chat-id))} whisper-opts)}]})) (fx/defn send-direct-message @@ -84,7 +86,7 @@ :message (merge {:sig (accounts.db/current-public-key cofx) :pubKey chat-id :payload payload - :topic (transport.utils/get-topic constants/contact-discovery)} + :topic (transport.topic/public-key->discovery-topic-hash chat-id)} whisper-opts)}]})))) (defrecord Message [content content-type message-type clock-value timestamp] diff --git a/src/status_im/transport/message/public_chat.cljs b/src/status_im/transport/message/public_chat.cljs index 5d6b48318f..4bc119c51a 100644 --- a/src/status_im/transport/message/public_chat.cljs +++ b/src/status_im/transport/message/public_chat.cljs @@ -44,17 +44,17 @@ (handlers/register-handler-fx ::add-new-sym-key - (fn [{:keys [db] :as cofx} [_ {:keys [sym-key-id sym-key chat-id]}]] + (fn [{:keys [db]} [_ {:keys [sym-key-id sym-key chat-id]}]] (let [{:keys [web3]} db topic (transport.utils/get-topic chat-id)] - {:db (assoc-in db [:transport/chats chat-id :sym-key-id] sym-key-id) - :shh/add-filter {:web3 web3 - :sym-key-id sym-key-id - :topic topic - :chat-id chat-id} - :data-store/tx [(transport-store/save-transport-tx - {:chat-id chat-id - :chat (-> (get-in db [:transport/chats chat-id]) - (assoc :sym-key-id sym-key-id) - ;;TODO (yenda) remove once go implements persistence - (assoc :sym-key sym-key))})]}))) + {:db (assoc-in db [:transport/chats chat-id :sym-key-id] sym-key-id) + :shh/add-filters {:web3 web3 + :filters [{:sym-key-id sym-key-id + :topic topic + :chat-id chat-id}]} + :data-store/tx [(transport-store/save-transport-tx + {:chat-id chat-id + :chat (-> (get-in db [:transport/chats chat-id]) + (assoc :sym-key-id sym-key-id) + ;;TODO (yenda) remove once go implements persistence + (assoc :sym-key sym-key))})]}))) diff --git a/src/status_im/transport/partitioned_topic.cljs b/src/status_im/transport/partitioned_topic.cljs new file mode 100644 index 0000000000..82d2384fd0 --- /dev/null +++ b/src/status_im/transport/partitioned_topic.cljs @@ -0,0 +1,54 @@ +(ns status-im.transport.partitioned-topic + (:require [status-im.utils.random :as random] + [status-im.transport.utils :as utils] + [status-im.constants :as constants] + [status-im.utils.config :as config])) + +;; Number of different personal topics +(def n-partitions 5000) + +(defn expected-number-of-collisions + "Expected number of topic collision given the number of expected users, + we want this value to be greater than a threshold to avoid positive + identification given the attacker has a topic & public key. + Used only for safety-checking n-partitions. + + https://en.wikipedia.org/wiki/Birthday_problem#Collision_counting" + [total-users] + (+ + (- total-users n-partitions) + (* n-partitions + (js/Math.pow + (/ + (- n-partitions 1) + n-partitions) + total-users)))) + +(defn- partitioned-topic + [public-key] + (let [gen (random/rand-gen public-key)] + (-> (random/seeded-rand-int gen n-partitions) + (str "-discovery")))) + +(defn partitioned-topic-hash + "Given a public key return a partitioned topic between 0 and n" + [public-key] + (-> public-key + partitioned-topic + utils/get-topic)) + +(def discovery-topic-hash (utils/get-topic constants/contact-discovery)) + +(defn public-key->discovery-topic + [public-key] + (if config/partitioned-topic-enabled? + (partitioned-topic public-key) + constants/contact-discovery)) + +(defn public-key->discovery-topic-hash [public-key] + (if config/partitioned-topic-enabled? + (partitioned-topic-hash public-key) + discovery-topic-hash)) + +(defn discovery-topics [public-key] + [(partitioned-topic-hash public-key) discovery-topic-hash]) diff --git a/src/status_im/transport/shh.cljs b/src/status_im/transport/shh.cljs index b6a5b3c885..2b7f0900a5 100644 --- a/src/status_im/transport/shh.cljs +++ b/src/status_im/transport/shh.cljs @@ -4,7 +4,9 @@ [status-im.constants :as constants] [status-im.transport.message.transit :as transit] [status-im.transport.utils :as transport.utils] - [taoensso.timbre :as log])) + [taoensso.timbre :as log] + [status-im.transport.partitioned-topic :as transport.topic] + [status-im.utils.config :as config])) (defn get-new-key-pair [{:keys [web3 on-success on-error]}] (if web3 @@ -72,9 +74,10 @@ (fn [post-calls] (doseq [{:keys [web3 payload src dst success-event error-event] :or {error-event :transport/send-status-message-error}} post-calls] - (let [direct-message (clj->js {:pubKey dst + (let [chat (transport.topic/public-key->discovery-topic dst) + direct-message (clj->js {:pubKey dst :sig src - :chat constants/contact-discovery + :chat chat :payload (-> payload transit/serialize transport.utils/from-utf8)})] @@ -90,7 +93,7 @@ (let [{:keys [web3 payload src success-event error-event] :or {error-event :protocol/send-status-message-error}} params message (clj->js {:sig src - :chat constants/contact-discovery + :chat (transport.topic/public-key->discovery-topic src) :payload (-> payload transit/serialize transport.utils/from-utf8)})] diff --git a/src/status_im/utils/config.cljs b/src/status_im/utils/config.cljs index 2bb67af14f..645e9ca70a 100644 --- a/src/status_im/utils/config.cljs +++ b/src/status_im/utils/config.cljs @@ -31,6 +31,7 @@ (def hardwallet-enabled? (enabled? (get-config :HARDWALLET_ENABLED 0))) (def dev-build? (enabled? (get-config :DEV_BUILD 0))) (def erc20-contract-warnings-enabled? (enabled? (get-config :ERC20_CONTRACT_WARNINGS))) +(def partitioned-topic-enabled? (enabled? (get-config :PARTITIONED_TOPIC "0"))) ;; CONFIG VALUES (def log-level diff --git a/test/cljs/status_im/test/group_chats/core.cljs b/test/cljs/status_im/test/group_chats/core.cljs index cb357c9a68..1fca590064 100644 --- a/test/cljs/status_im/test/group_chats/core.cljs +++ b/test/cljs/status_im/test/group_chats/core.cljs @@ -196,7 +196,7 @@ :member member-1}]}]} actual (group-chats/handle-membership-update cofx new-message "payload" admin)] (testing "it removes the topic" - (is (:shh/remove-filter actual))))))) + (is (:shh/remove-filters actual))))))) (deftest build-group-test (testing "only adds" diff --git a/test/cljs/status_im/test/transport/core.cljs b/test/cljs/status_im/test/transport/core.cljs index 8ea9754450..136623d34e 100644 --- a/test/cljs/status_im/test/transport/core.cljs +++ b/test/cljs/status_im/test/transport/core.cljs @@ -15,10 +15,12 @@ "4" {:topic "topic-4"}} :semaphores #{}}}] (testing "it adds the discover filters" - (is (= {:web3 nil :private-key-id "1" :topics [{:chat-id :discovery-topic - :topic "0xf8946aac"} - {:chat-id "4" - :topic "topic-4"}]} + (is (= {:web3 nil :private-key-id "1" :topics '({:topic "topic-4" + :chat-id "4"} + {:topic "0x2af2e6e7" + :chat-id :discovery-topic} + {:topic "0xf8946aac" + :chat-id :discovery-topic})} (:shh/add-discovery-filters (transport/init-whisper cofx))))) (testing "it restores the sym-keys" diff --git a/test/cljs/status_im/test/transport/partitioned_topic.cljs b/test/cljs/status_im/test/transport/partitioned_topic.cljs new file mode 100644 index 0000000000..4334e832bb --- /dev/null +++ b/test/cljs/status_im/test/transport/partitioned_topic.cljs @@ -0,0 +1,38 @@ +(ns status-im.test.transport.filters + (:require [cljs.test :refer-macros [deftest is testing]] + [status-im.transport.partitioned-topic :as t])) + +(def pk "0x04985040682b77a32bb4bb58268a0719bd24ca4d07c255153fe1eb2ccd5883669627bd1a092d7cc76e8e4b9104327667b19dcda3ac469f572efabe588c38c1985f") + +(def expected-users 40000) +(def average-number-of-contacts 100) +;; Minimum number of identical personal topics +(def min-number-of-clashes 1000) +;; Maximum share of traffic that each user will be receiving +(def max-share-of-traffic (/ 1 250)) +(def partitioned-topic "0xed2fdbad") +(def discovery-topic "0xf8946aac") + +(deftest partition-topic + (testing "it returns a seeded topic based on an input string" + (is (= partitioned-topic (t/partitioned-topic-hash pk)))) + (testing "same input same output" + (is (= (t/partitioned-topic-hash "a") (t/partitioned-topic-hash "a"))))) + +(deftest discovery-topics + (testing "it returns a partition topic & the discovery topic" + (is (= [partitioned-topic discovery-topic] + (t/discovery-topics pk))))) + +(deftest minimum-number-of-clashes-test + (testing (str "it needs to be greater than " min-number-of-clashes) + (is (<= min-number-of-clashes (t/expected-number-of-collisions expected-users))))) + +(deftest max-avg-share-of-traffic-test + (testing (str "it needs to be less than " max-share-of-traffic) + (is (>= max-share-of-traffic + ;; we always listen to our personal topic + + ;; we listen to contact topics - collisions + (/ (+ 1 (- average-number-of-contacts + (t/expected-number-of-collisions average-number-of-contacts))) + expected-users)))))