status-im
/
status-mobile
mirror of https://github.com/status-im/status-mobile.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

[slow sign in] 

fix iterating over all messages from realm db (was done for deduplication)
async loading of chats (:init-chats event)
This commit is contained in:
Roman Volosovskyi 2018-11-14 21:49:18 +01:00
parent 205c9aaecc
commit d66198a420
No known key found for this signature in database
GPG Key ID: 0238A4B5ECEE70DE
17 changed files with 250 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/status_im/accounts/login/core.cljs
View File

@ -33,7 +33,7 @@
(catch (fn [error]
(log/warn "Could not change account" error)
;; If all else fails we fallback to showing initial error
(re-frame/dispatch [:init.callback/account-change-error])))))
(re-frame/dispatch [:init.callback/account-change-error (str error)])))))
;;;; Handlers
(fx/defn login [cofx]

57
src/status_im/chat/models/loading.cljs
View File

@ -52,35 +52,48 @@
(filter #(not (contains? message-id->messages %))))
(vals message-id->messages)))
(fx/defn load-chats-messages
[{:keys [db get-stored-messages get-stored-user-statuses
get-referenced-messages get-stored-unviewed-messages]
:as cofx}]
(let [chats (:chats db)]
(fx/merge
cofx
{:db (assoc
db :chats
(reduce
(fn [chats chat-id]
(let [stored-unviewed-messages (get-stored-unviewed-messages (accounts.db/current-public-key cofx))
chat-messages (index-messages (get-stored-messages chat-id))
message-ids (keys chat-messages)]
(update
chats
chat-id
assoc
:messages chat-messages
:message-statuses (get-stored-user-statuses chat-id message-ids)
:unviewed-messages (get stored-unviewed-messages chat-id)
:referenced-messages (into {}
(map (juxt :message-id identity)
(get-referenced-messages
(get-referenced-ids chat-messages)))))))
chats
(keys chats)))}
(group-messages))))
(fx/defn initialize-chats
"Initialize all persisted chats on startup"
[{:keys [db default-dapps all-stored-chats get-stored-messages get-stored-user-statuses
get-stored-unviewed-messages get-referenced-messages stored-message-ids
stored-deduplication-ids] :as cofx}]
(let [stored-unviewed-messages (get-stored-unviewed-messages (accounts.db/current-public-key cofx))
chats (reduce (fn [acc {:keys [chat-id] :as chat}]
(let [chat-messages (index-messages (get-stored-messages chat-id))
message-ids (keys chat-messages)
unviewed-ids (get stored-unviewed-messages chat-id)]
(assoc acc chat-id
(assoc chat
:unviewed-messages unviewed-ids
:messages chat-messages
:message-statuses (get-stored-user-statuses chat-id message-ids)
:deduplication-ids (get stored-deduplication-ids chat-id)
:not-loaded-message-ids (set/difference (get stored-message-ids chat-id)
(set message-ids))
:referenced-messages (into {}
(map (juxt :message-id identity)
(get-referenced-messages
(get-referenced-ids chat-messages))))))))
[{:keys [db default-dapps all-stored-chats] :as cofx}]
(let [chats (reduce (fn [acc {:keys [chat-id] :as chat}]
(assoc acc chat-id
(assoc chat :not-loaded-message-ids #{})))
{}
all-stored-chats)]
(fx/merge cofx
{:db (assoc db
:chats chats
:chats chats
:contacts/dapps default-dapps)}
(group-messages)
(commands/load-commands commands/register))))
(fx/defn initialize-pending-messages

15
src/status_im/chat/models/message.cljs
View File

@ -27,7 +27,8 @@
[status-im.utils.platform :as platform]
[status-im.ui.components.react :as react]
[status-im.utils.fx :as fx]
[taoensso.timbre :as log]))
[taoensso.timbre :as log]
[status-im.data-store.messages :as messages-store]))
(defn- wrap-group-message
"Wrap a group message in a membership update"
@ -198,14 +199,13 @@
(chat-loading/group-chat-messages chat-id (get chat->message chat-id))))
(defn- add-to-chat?
[{:keys [db]} {:keys [chat-id clock-value message-id from] :as message}]
(let [deduplication-id (messages-store/deduplication-id from chat-id clock-value)
{:keys [deleted-at-clock-value messages not-loaded-message-ids deduplication-ids]}
[{:keys [db]} {:keys [chat-id clock-value message-id from]}]
(let [{:keys [deleted-at-clock-value messages not-loaded-message-ids]}
(get-in db [:chats chat-id])]
(not (or (get messages message-id)
(get not-loaded-message-ids message-id)
(get deduplication-ids deduplication-id)
(>= deleted-at-clock-value clock-value)))))
(>= deleted-at-clock-value clock-value)
(messages-store/message-exists? message-id)))))
(defn- filter-messages [cofx messages]
(:accumulated (reduce (fn [{:keys [seen-ids] :as acc}
@ -290,8 +290,9 @@
(fx/defn upsert-and-send [{:keys [now] :as cofx} {:keys [chat-id] :as message}]
(let [send-record (protocol/map->Message (select-keys message transport-keys))
message-id (transport.utils/message-id send-record)
message-id (transport.utils/message-id message)
message-with-id (assoc message :message-id message-id)]
(fx/merge cofx
(chat-model/upsert-chat {:chat-id chat-id
:timestamp now})

2
src/status_im/chat/specs.cljs
View File

@ -18,7 +18,6 @@
(s/def :chat/message-statuses (s/nilable map?)) ; message/user statuses indexed by two level index
(s/def :chat/not-loaded-message-ids (s/nilable set?)) ; set of message-ids not yet fully loaded from persisted state
(s/def :chat/referenced-messages (s/nilable map?)) ; map of messages indexed by message-id which are not displayed directly, but referenced by other messages
(s/def :chat/deduplication-ids (s/nilable set?)) ; set of helper deduplication ids
(s/def :chat/last-clock-value (s/nilable number?)) ; last logical clock value of messages in chat
(s/def :chat/loaded-chats (s/nilable seq?))
(s/def :chat/bot-db (s/nilable map?))
@ -26,3 +25,4 @@
(s/def :chat/cooldown-enabled? (s/nilable boolean?))
(s/def :chat/last-outgoing-message-sent-at (s/nilable number?))
(s/def :chat/spam-messages-frequency (s/nilable number?)) ; number of consecutive spam messages sent
(s/def :chats/loading? (s/nilable boolean?))

43
src/status_im/data_store/messages.cljs
View File

@ -39,47 +39,9 @@
(fn [cofx _]
(assoc cofx :get-stored-messages get-by-chat-id)))
(re-frame/reg-cofx
:data-store/message-ids
(fn [cofx _]
(assoc cofx :stored-message-ids (let [chat-id->message-id (volatile! {})]
(-> @core/account-realm
(.objects "message")
(.map (fn [msg _ _]
(vswap! chat-id->message-id
#(update %
(aget msg "chat-id")
(fnil conj #{})
(aget msg "message-id"))))))
@chat-id->message-id))))
(defn- sha3 [s]
(.sha3 dependencies/Web3.prototype s))
(defn deduplication-id
"Computes deduplication id from message sender-pk, chat-id and clock-value"
[sender-pk chat-id clock-value]
(sha3 (str sender-pk chat-id clock-value)))
(re-frame/reg-cofx
:data-store/deduplication-ids
(fn [cofx _]
(assoc cofx :stored-deduplication-ids (let [chat-id->message-id (volatile! {})]
(-> @core/account-realm
(.objects "message")
(.map (fn [msg _ _]
(let [chat-id (aget msg "chat-id")
sender-pk (aget msg "from")
clock-value (aget msg "clock-value")]
(vswap! chat-id->message-id
#(update %
(aget msg "chat-id")
(fnil conj #{})
(deduplication-id sender-pk
chat-id
clock-value)))))))
@chat-id->message-id))))
(defn- get-unviewed-messages
[public-key]
(-> @core/account-realm
@ -136,3 +98,8 @@
(fn [realm]
(core/delete realm (core/get-by-field realm :message :chat-id chat-id))
(core/delete realm (core/get-by-field realm :user-status :chat-id chat-id))))
(defn message-exists? [message-id]
(if @core/account-realm
(not (nil? (.objectForPrimaryKey @core/account-realm "message" message-id)))
false))

18
src/status_im/data_store/realm/schemas/account/core.cljs
View File

@ -252,6 +252,19 @@
browser/v8
dapp-permissions/v9])
(def v25 [chat/v8
transport/v7
contact/v3
message/v7
mailserver/v11
mailserver-topic/v1
user-status/v2
membership-update/v1
installation/v2
local-storage/v1
browser/v8
dapp-permissions/v9])
;; put schemas ordered by version
(def schemas [{:schema v1
:schemaVersion 1
@ -324,4 +337,7 @@
:migration migrations/v23}
{:schema v24
:schemaVersion 24
:migration migrations/v24}])
:migration migrations/v24}
{:schema v25
:schemaVersion 25
:migration migrations/v25}])

60
src/status_im/data_store/realm/schemas/account/migrations.cljs
View File

@ -1,7 +1,9 @@
(ns status-im.data-store.realm.schemas.account.migrations
(:require [taoensso.timbre :as log]
[cljs.reader :as reader]
[status-im.chat.models.message-content :as message-content]))
[status-im.chat.models.message-content :as message-content]
[status-im.transport.utils :as transport.utils]
[cljs.tools.reader.edn :as edn]))
(defn v1 [old-realm new-realm]
(log/debug "migrating v1 account database: " old-realm new-realm))
@ -156,3 +158,59 @@
(defn v24 [old-realm new-realm]
(log/debug "migrating v24 account database"))
(defn v25 [old-realm new-realm]
(log/debug "migrating v25 account database")
(let [new-messages (.objects new-realm "message")
user-statuses (.objects new-realm "user-status")
old-ids->new-ids (volatile! {})
updated-messages-ids (volatile! #{})
updated-message-statuses-ids (volatile! #{})
messages-to-be-deleted (volatile! [])
statuses-to-be-deleted (volatile! [])]
(dotimes [i (.-length new-messages)]
(let [message (aget new-messages i)
message-id (aget message "message-id")
from (aget message "from")
chat-id (aget message "chat-id")
clock-value (aget message "clock-value")
new-message-id (transport.utils/message-id
{:from from
:chat-id chat-id
:clock-value clock-value})]
(vswap! old-ids->new-ids assoc message-id new-message-id)))
(dotimes [i (.-length new-messages)]
(let [message (aget new-messages i)
old-message-id (aget message "message-id")
content (edn/read-string (aget message "content"))
response-to (:response-to content)
new-message-id (get @old-ids->new-ids old-message-id)]
(if (contains? @updated-messages-ids new-message-id)
(vswap! messages-to-be-deleted conj message)
(do
(vswap! updated-messages-ids conj new-message-id)
(aset message "message-id" new-message-id)
(when (and response-to (get @old-ids->new-ids response-to))
(let [new-content (assoc content :response-to
(get @old-ids->new-ids response-to))]
(aset message "content" (prn-str new-content))))))))
(doseq [message @messages-to-be-deleted]
(.delete new-realm message))
(dotimes [i (.-length user-statuses)]
(let [user-status (aget user-statuses i)
message-id (aget user-status "message-id")
new-message-id (get @old-ids->new-ids message-id)
whisper-id (aget user-status "whisper-identity")
new-status-id (str new-message-id "-" whisper-id)]
(if (contains? @updated-message-statuses-ids new-status-id)
(vswap! statuses-to-be-deleted conj user-status)
(do
(vswap! updated-message-statuses-ids conj new-status-id)
(aset user-status "status-id" new-status-id)
(aset user-status "message-id" new-message-id)))))
(doseq [status @statuses-to-be-deleted]
(.delete new-realm status))))

42
src/status_im/events.cljs
View File

@ -42,7 +42,9 @@
[status-im.utils.fx :as fx]
[status-im.utils.handlers :as handlers]
[status-im.utils.utils :as utils]
[taoensso.timbre :as log]))
[taoensso.timbre :as log]
[status-im.utils.datetime :as time]
[status-im.chat.models.loading :as chat-loading]))
;; init module
@ -83,22 +85,34 @@
(init/handle-init-store-error cofx encryption-key)))
(handlers/register-handler-fx
:init.callback/account-change-success
:load-chats-messages
[(re-frame/inject-cofx :data-store/get-unviewed-messages)
(re-frame/inject-cofx :data-store/get-messages)
(re-frame/inject-cofx :data-store/get-referenced-messages)
(re-frame/inject-cofx :data-store/get-user-statuses)]
(fn [cofx]
(chat-loading/load-chats-messages cofx)))
(handlers/register-handler-fx
:init-chats
[(re-frame/inject-cofx :web3/get-web3)
(re-frame/inject-cofx :get-default-dapps)
(re-frame/inject-cofx :data-store/all-chats)
(re-frame/inject-cofx :data-store/get-messages)
(re-frame/inject-cofx :data-store/get-user-statuses)
(re-frame/inject-cofx :data-store/get-unviewed-messages)
(re-frame/inject-cofx :data-store/get-referenced-messages)
(re-frame/inject-cofx :data-store/message-ids)
(re-frame/inject-cofx :data-store/deduplication-ids)
(re-frame/inject-cofx :data-store/get-local-storage-data)
(re-frame/inject-cofx :data-store/get-all-contacts)
(re-frame/inject-cofx :data-store/get-all-installations)
(re-frame/inject-cofx :data-store/get-all-mailservers)
(re-frame/inject-cofx :data-store/transport)
(re-frame/inject-cofx :data-store/mailserver-topics)
(re-frame/inject-cofx :data-store/mailserver-topics)]
(fn [{:keys [db] :as cofx} [_ address]]
(fx/merge cofx
{:db (assoc db :chats/loading? false)}
(chat-loading/initialize-chats)
(protocol/initialize-protocol address)
(chat-loading/initialize-pending-messages))))
(handlers/register-handler-fx
:init.callback/account-change-success
[(re-frame/inject-cofx :web3/get-web3)
(re-frame/inject-cofx :data-store/get-all-contacts)
(re-frame/inject-cofx :data-store/get-all-installations)
(re-frame/inject-cofx :data-store/all-browsers)
(re-frame/inject-cofx :data-store/all-dapp-permissions)]
(fn [cofx [_ address]]
@ -106,8 +120,8 @@
(handlers/register-handler-fx
:init.callback/account-change-error
(fn [cofx _]
(init/handle-change-account-error cofx)))
(fn [cofx [_ error]]
(init/handle-change-account-error cofx error)))
(handlers/register-handler-fx
:init.callback/keychain-reset

6
src/status_im/group_chats/core.cljs
View File

@ -115,7 +115,11 @@
:dsts members
:success-event [:transport/message-sent
chat-id
(transport.utils/message-id (:message payload))
(transport.utils/message-id
;; NOTE: There is no clock-value here.
;; Will we have collision?
{:from current-public-key
:chat-id chat-id})
:group-user-message]
:payload payload}}))))

7
src/status_im/init/core.cljs
View File

@ -98,10 +98,10 @@
{:db (assoc db :device-UUID device-uuid)})
(fx/defn handle-change-account-error
[cofx]
[cofx error]
{:ui/show-confirmation
{:title (i18n/label :invalid-key-title)
:content (i18n/label :invalid-key-content)
:content (str error "\n" (i18n/label :invalid-key-content))
:confirm-button-text (i18n/label :invalid-key-confirm)
;; On cancel we initialize the app with the invalid key, to allow the user
;; to recover the seed phrase
@ -209,13 +209,10 @@
[:web3/fetch-node-version-callback %])]
:notifications/get-fcm-token nil}
(initialize-account-db address)
(protocol/initialize-protocol address)
(contact/load-contacts)
(pairing/load-installations)
#(when (dev-mode? %)
(models.dev-server/start))
(chat-loading/initialize-chats)
(chat-loading/initialize-pending-messages)
(browser/initialize-browsers)
(browser/initialize-dapp-permissions)
(extensions.registry/initialize)

10
src/status_im/transport/message/protocol.cljs
View File

@ -87,7 +87,10 @@
:payload this
:success-event [:transport/message-sent
chat-id
(transport.utils/message-id this)
(transport.utils/message-id
{:from current-public-key
:chat-id chat-id
:clock-value clock-value})
message-type]}]
(case message-type
:public-group-user-message
@ -113,7 +116,10 @@
(receive [this chat-id signature _ cofx]
{:chat-received-message/add-fx
[(assoc (into {} this)
:message-id (transport.utils/message-id this)
:message-id (transport.utils/message-id
{:chat-id chat-id
:from signature
:clock-value clock-value})
:chat-id chat-id
:from signature
:js-obj (:js-obj cofx))]})

6
src/status_im/transport/utils.cljs
View File

@ -19,8 +19,10 @@
(defn message-id
"Get a message-id"
[message]
(sha3 (pr-str message)))
[{:keys [from chat-id clock-value] :as m}]
{:pre [(not (nil? from))
(not (nil? chat-id))]}
(sha3 (str from chat-id clock-value)))
(defn get-topic
"Get the topic of a group chat or public chat from the chat-id"

7
src/status_im/ui/screens/db.cljs
View File

@ -65,7 +65,8 @@
:nfc-enabled? false
:pin {:original []
:confirmation []
:enter-step :original}}})
:enter-step :original}}
:chats/loading? true})
;;;;GLOBAL
@ -247,7 +248,8 @@
:wallet/all-tokens
:ui/contact
:ui/search
:ui/chat]
:ui/chat
:chats/loading?]
:opt-un [::modal
::was-modal?
::rpc-url
@ -294,7 +296,6 @@
:chat/message-groups
:chat/message-statuses
:chat/not-loaded-message-ids
:chat/deduplication-ids
:chat/referenced-messages
:chat/last-clock-value
:chat/loaded-chats

37
src/status_im/ui/screens/desktop/main/tabs/home/views.cljs
View File

@ -10,7 +10,9 @@
[taoensso.timbre :as log]
[status-im.ui.components.icons.vector-icons :as icons]
[status-im.ui.components.react :as react]
[status-im.constants :as constants]))
[status-im.constants :as constants]
[status-im.utils.utils :as utils]
[status-im.ui.components.react :as components]))
(views/defview unviewed-indicator [chat-id]
(let [unviewed-messages-count (re-frame/subscribe [:chats/unviewed-messages-count chat-id])]
@ -99,9 +101,21 @@
text (.-text native-event)]
(re-frame/dispatch [:search/filter-changed text])))}]])
(views/defview chat-list-view []
(views/defview chat-list-view [loading?]
(views/letsubs [search-filter [:search/filter]
filtered-home-items [:search/filtered-home-items]]
{:component-did-mount
(fn [this]
(let [[_ loading?] (.. this -props -argv)]
(when loading?
(re-frame/dispatch [:init-chats]))))
:component-did-update
(fn [this [_ old-loading?]]
(let [[_ loading?] (.. this -props -argv)]
(when (and (false? loading?)
(true? old-loading?))
(re-frame/dispatch [:load-chats-messages]))))}
[react/view {:style styles/chat-list-view}
[react/view {:style styles/chat-list-header}
[search-input search-filter]
@ -109,8 +123,17 @@
[react/view {:style styles/add-new}
[icons/icon :icons/add {:style {:tint-color :white}}]]]]
[react/view {:style styles/chat-list-separator}]
[react/scroll-view {:enableArrayScrollingOptimization true}
[react/view
(for [[index chat] (map-indexed vector filtered-home-items)]
^{:key (first chat)}
[chat-list-item chat])]]]))
(if loading?
[react/view {:style {:flex 1
:justify-content :center
:align-items :center}}
[components/activity-indicator {:animating true}]]
[react/scroll-view {:enableArrayScrollingOptimization true}
[react/view
(for [[index chat] (map-indexed vector filtered-home-items)]
^{:key (first chat)}
[chat-list-item chat])]])]))
(views/defview chat-list-view-wrapper []
(views/letsubs [loading? [:get :chats/loading?]]
[chat-list-view loading?]))

2
src/status_im/ui/screens/desktop/main/views.cljs
View File

@ -17,7 +17,7 @@
(views/letsubs [tab [:get-in [:desktop/desktop :tab-view-id]]]
(let [component (case tab
:profile profile.views/profile-data
:home home.views/chat-list-view
:home home.views/chat-list-view-wrapper
react/view)]
[react/view {:style {:flex 1}}
[component]])))

51
src/status_im/ui/screens/home/views.cljs
View File

@ -11,7 +11,10 @@
[status-im.utils.platform :as platform]
[status-im.react-native.resources :as resources]
[status-im.ui.components.common.common :as components.common]
[status-im.ui.components.icons.vector-icons :as icons]))
[status-im.ui.components.icons.vector-icons :as icons]
[status-im.utils.datetime :as time]
[status-im.ui.components.react :as components]
[status-im.utils.utils :as utils]))
(defn- toolbar [show-welcome? show-sync-state sync-state latest-block-number]
(when-not (and show-welcome?
@ -74,26 +77,52 @@
[react/i18n-text {:style styles/welcome-text-description
:key :welcome-to-status-description}]]]))
(views/defview home []
(views/letsubs [home-items [:home-items]
show-welcome? [:get-in [:accounts/create :show-welcome?]]
(views/defview chats-list []
(views/letsubs [home-items [:home-items]]
(if (empty? home-items)
[react/view styles/no-chats
[react/i18n-text {:style styles/no-chats-text :key :no-recent-chats}]]
[list/flat-list {:data home-items
:key-fn first
:render-fn (fn [home-item]
[home-list-item home-item])}])))
(views/defview home [loading?]
(views/letsubs [show-welcome? [:get-in [:accounts/create :show-welcome?]]
view-id [:get :view-id]
sync-state [:chain-sync-state]
latest-block-number [:latest-block-number]
rpc-network? [:current-network-uses-rpc?]]
{:component-did-mount
(fn [this]
(let [[_ loading?] (.. this -props -argv)]
(when loading?
(utils/set-timeout
#(re-frame/dispatch [:init-chats])
100))))
:component-did-update
(fn [this [_ old-loading?]]
(let [[_ loading?] (.. this -props -argv)]
(when (and (false? loading?)
(true? old-loading?))
(re-frame/dispatch [:load-chats-messages]))))}
[react/view styles/container
[toolbar show-welcome? (not rpc-network?) sync-state latest-block-number]
(cond show-welcome?
[welcome view-id]
(empty? home-items)
[react/view styles/no-chats
[react/i18n-text {:style styles/no-chats-text :key :no-recent-chats}]]
loading?
[react/view {:style {:flex 1
:justify-content :center
:align-items :center}}
[components/activity-indicator {:animating true}]]
:else
[list/flat-list {:data home-items
:key-fn first
:render-fn (fn [home-item]
[home-list-item home-item])}])
[chats-list])
(when platform/android?
[home-action-button])
(when-not show-welcome?
[connectivity/error-view])]))
(views/defview home-wrapper []
(views/letsubs [loading? [:get :chats/loading?]]
[home loading?]))

2
src/status_im/ui/screens/main_tabs/views.cljs
View File

@ -76,7 +76,7 @@
(contains? #{:home :wallet :my-profile} new-view-id))}
[react/view common.styles/main-container
(case view-id
:home [home/home]
:home [home/home-wrapper]
:wallet [wallet.main/wallet]
:my-profile [profile.user/my-profile]
nil)