2020-04-30 14:44:17 +00:00
|
|
|
(ns status-im.chat.models.message-test
|
2018-04-19 08:48:04 +00:00
|
|
|
(:require [cljs.test :refer-macros [deftest is testing]]
|
2020-04-03 11:32:42 +00:00
|
|
|
[status-im.chat.models.loading :as chat-loading]
|
2020-05-05 14:18:23 +00:00
|
|
|
[status-im.chat.models.message :as message]
|
2019-11-12 06:07:17 +00:00
|
|
|
[status-im.chat.models.message-list :as models.message-list]
|
2020-05-05 14:18:23 +00:00
|
|
|
[status-im.constants :as constants]
|
2020-04-03 11:32:42 +00:00
|
|
|
[status-im.ui.screens.chat.state :as view.state]
|
2020-05-05 14:18:23 +00:00
|
|
|
[status-im.utils.datetime :as time]
|
|
|
|
[status-im.utils.gfycat.core :as gfycat]
|
|
|
|
[status-im.utils.identicon :as identicon]))
|
2018-04-19 08:48:04 +00:00
|
|
|
|
2020-04-03 11:32:42 +00:00
|
|
|
(deftest add-received-message-test
|
|
|
|
(with-redefs [message/add-message (constantly :added)]
|
|
|
|
(let [chat-id "chat-id"
|
|
|
|
clock-value 10
|
|
|
|
cursor-clock-value (dec clock-value)
|
|
|
|
cursor (chat-loading/clock-value->cursor cursor-clock-value)
|
|
|
|
cofx {:now 0
|
|
|
|
:db {:loaded-chat-id chat-id
|
|
|
|
:current-chat-id chat-id
|
|
|
|
:all-loaded? true
|
|
|
|
:chats {chat-id {:cursor cursor
|
|
|
|
:cursor-clock-value cursor-clock-value}}}}
|
2020-08-31 09:52:17 +00:00
|
|
|
message {:chat-id chat-id
|
|
|
|
:clock-value clock-value
|
|
|
|
:alias "alias"
|
|
|
|
:name "name"
|
|
|
|
:identicon "identicon"
|
|
|
|
:from "from"}]
|
2020-04-03 11:32:42 +00:00
|
|
|
(testing "not current-chat"
|
|
|
|
(is (nil? (message/add-received-message
|
|
|
|
(update cofx :db dissoc :loaded-chat-id)
|
|
|
|
message))))
|
|
|
|
;; <- cursor
|
|
|
|
;; <- message
|
|
|
|
;; <- top of the chat
|
|
|
|
(testing "there's no hidden item"
|
|
|
|
(with-redefs [view.state/first-not-visible-item (atom nil)]
|
2020-08-31 09:52:17 +00:00
|
|
|
(is (= {:db {:loaded-chat-id "chat-id",
|
|
|
|
:current-chat-id "chat-id",
|
|
|
|
:all-loaded? true,
|
|
|
|
:chats
|
|
|
|
{"chat-id"
|
|
|
|
{:cursor
|
|
|
|
"00000000000000000000000000000000000000000000000000090x0000000000000000000000000000000000000000000000000000000000000000",
|
|
|
|
:cursor-clock-value 9,
|
|
|
|
:users
|
2020-09-17 12:00:46 +00:00
|
|
|
{"from" {:alias "alias",
|
|
|
|
:name "name",
|
|
|
|
:identicon "identicon",
|
|
|
|
:public-key "from"}}}}}}
|
2020-08-31 09:52:17 +00:00
|
|
|
(message/add-received-message
|
|
|
|
cofx
|
|
|
|
message)))))
|
2020-04-03 11:32:42 +00:00
|
|
|
;; <- cursor
|
|
|
|
;; <- first-hidden-item
|
|
|
|
;; <- message
|
|
|
|
;; <- top of the chat
|
|
|
|
(testing "the hidden item has a clock value less than the current"
|
|
|
|
(with-redefs [view.state/first-not-visible-item (atom {:clock-value (dec clock-value)})]
|
2020-08-31 09:52:17 +00:00
|
|
|
(is (= {:db {:loaded-chat-id "chat-id",
|
|
|
|
:current-chat-id "chat-id",
|
|
|
|
:all-loaded? true,
|
|
|
|
:chats
|
|
|
|
{"chat-id"
|
|
|
|
{:cursor
|
|
|
|
"00000000000000000000000000000000000000000000000000090x0000000000000000000000000000000000000000000000000000000000000000",
|
|
|
|
:cursor-clock-value 9,
|
|
|
|
:users
|
2020-09-17 12:00:46 +00:00
|
|
|
{"from" {:alias "alias",
|
|
|
|
:name "name",
|
|
|
|
:identicon "identicon",
|
|
|
|
:public-key "from"}}}}}}
|
2020-08-31 09:52:17 +00:00
|
|
|
(message/add-received-message
|
|
|
|
cofx
|
|
|
|
message)))))
|
2020-04-03 11:32:42 +00:00
|
|
|
;; <- cursor
|
|
|
|
;; <- message
|
|
|
|
;; <- first-hidden-item
|
|
|
|
;; <- top of the chat
|
|
|
|
(testing "the message falls between the first-hidden-item and cursor"
|
|
|
|
(with-redefs [view.state/first-not-visible-item (atom {:clock-value (inc clock-value)})]
|
|
|
|
(let [result (message/add-received-message
|
|
|
|
cofx
|
|
|
|
message)]
|
|
|
|
(testing "it sets all-loaded? to false"
|
|
|
|
(is (not (get-in result [:db :chats chat-id :all-loaded?]))))
|
|
|
|
(testing "it updates cursor-clock-value & cursor"
|
|
|
|
(is (= clock-value (get-in result [:db :chats chat-id :cursor-clock-value])))
|
|
|
|
(is (= (chat-loading/clock-value->cursor clock-value) (get-in result [:db :chats chat-id :cursor])))))))
|
|
|
|
;; <- message
|
|
|
|
;; <- first-hidden-item
|
|
|
|
;; <- top of the chat
|
|
|
|
(testing "the message falls between the first-hidden-item and cursor is nil"
|
|
|
|
(with-redefs [view.state/first-not-visible-item (atom {:clock-value (inc clock-value)})]
|
|
|
|
(let [result (message/add-received-message
|
|
|
|
(update-in cofx [:db :chats chat-id] dissoc :cursor :cursor-clock-value)
|
|
|
|
message)]
|
|
|
|
(testing "it sets all-loaded? to false"
|
|
|
|
(is (not (get-in result [:db :chats chat-id :all-loaded?]))))
|
|
|
|
(testing "it updates cursor-clock-value & cursor"
|
|
|
|
(is (= clock-value (get-in result [:db :chats chat-id :cursor-clock-value])))
|
|
|
|
(is (= (chat-loading/clock-value->cursor clock-value) (get-in result [:db :chats chat-id :cursor])))))))
|
|
|
|
;; <- message
|
|
|
|
;; <- cursor
|
|
|
|
;; <- first-hidden-item
|
|
|
|
;; <- top of the chat
|
|
|
|
(testing "the message falls before both the first-hidden-item and cursor"
|
|
|
|
(with-redefs [view.state/first-not-visible-item (atom {:clock-value (inc clock-value)})]
|
|
|
|
(let [result (message/add-received-message
|
|
|
|
cofx
|
|
|
|
(update message :clock-value #(- % 2)))]
|
|
|
|
(testing "it sets all-loaded? to false"
|
|
|
|
(is (not (get-in result [:db :chats chat-id :all-loaded?]))))
|
|
|
|
(testing "it does not update cursor-clock-value & cursor"
|
|
|
|
(is (= cursor-clock-value (get-in result [:db :chats chat-id :cursor-clock-value])))
|
|
|
|
(is (= cursor (get-in result [:db :chats chat-id :cursor]))))))))))
|
|
|
|
|
2020-04-08 16:01:11 +00:00
|
|
|
(deftest message-loaded?
|
|
|
|
(testing "it returns false when it's not in loaded message"
|
2020-05-05 14:18:23 +00:00
|
|
|
(is (not (#'status-im.chat.models.message/message-loaded? {:db {:chats {"a" {}}}}
|
|
|
|
{:message-id "message-id"
|
|
|
|
:from "a"
|
|
|
|
:clock-value 1
|
|
|
|
:chat-id "a"}))))
|
2020-04-08 16:01:11 +00:00
|
|
|
(testing "it returns true when it's already in the loaded message"
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(is (#'status-im.chat.models.message/message-loaded? {:db
|
|
|
|
{:messages {"a" {"message-id" {}}}}}
|
2020-05-05 14:18:23 +00:00
|
|
|
{:message-id "message-id"
|
|
|
|
:from "a"
|
|
|
|
:clock-value 1
|
|
|
|
:chat-id "a"}))))
|
2020-04-08 16:01:11 +00:00
|
|
|
(deftest earlier-than-deleted-at?
|
|
|
|
(testing "it returns true when the clock-value is the same as the deleted-clock-value in chat"
|
2020-05-05 14:18:23 +00:00
|
|
|
(is (#'status-im.chat.models.message/earlier-than-deleted-at? {:db {:chats {"a" {:deleted-at-clock-value 1}}}}
|
|
|
|
{:message-id "message-id"
|
|
|
|
:from "a"
|
|
|
|
:clock-value 1
|
|
|
|
:chat-id "a"})))
|
2020-04-08 16:01:11 +00:00
|
|
|
(testing "it returns false when the clock-value is greater than the deleted-clock-value in chat"
|
2020-05-05 14:18:23 +00:00
|
|
|
(is (not (#'status-im.chat.models.message/earlier-than-deleted-at? {:db {:chats {"a" {:deleted-at-clock-value 1}}}}
|
|
|
|
{:message-id "message-id"
|
|
|
|
:from "a"
|
|
|
|
:clock-value 2
|
|
|
|
:chat-id "a"}))))
|
2020-04-08 16:01:11 +00:00
|
|
|
(testing "it returns true when the clock-value is less than the deleted-clock-value in chat"
|
2020-05-05 14:18:23 +00:00
|
|
|
(is (#'status-im.chat.models.message/earlier-than-deleted-at? {:db {:chats {"a" {:deleted-at-clock-value 1}}}}
|
|
|
|
{:message-id "message-id"
|
|
|
|
:from "a"
|
|
|
|
:clock-value 0
|
|
|
|
:chat-id "a"}))))
|
2018-05-03 18:20:56 +00:00
|
|
|
|
[fixes #4745] Don't query mailserver on restored account
When creating a new account / recovery we don't poll the mailserver anymore for historic messages, which solves the immediate issue of fetching only received messages
Handle messages sent from a different device in public chat / restore history. The message will be added, shown correctly as sent by the user, and the status will be set as sent ( need to check for seen race condition, as messages will now be added twice). This means that multidevice should now work for public chats.
Move contact updates to discovery topic. This is necessary as there is a pre-existing bug whereby contact updates would not work anymore after wallet recovery, as the code relies on the initial contact request being stored on the mailserver, which we cannot guarantee (we only pull 7 days of data). Not pulling history anymore exacerbate the problems but does not introduce it.
To make sure that contact updates will work after wallet recovery, we also need to consider a ContactUpdate in the same way we consider a ContactRequest (the other peer has no idea that the user has recovered the wallet). This does not change any behaviour in terms of obscurity/security as ContactRequest are automatically processed (in both case the contact will be set as pending?, not as accepted)
At this stage ContactRequest, ContactRequestConfirmed, ContactUpdate have all the same logic, i.e. update the contact information, leave the pending flag alone.
Only 1 day of history is fetched for newly joined chats, if catching up 7 days is the cap as before.
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2018-06-19 11:43:47 +00:00
|
|
|
(deftest add-own-received-message
|
2019-07-03 14:29:01 +00:00
|
|
|
(let [db {:multiaccount {:public-key "me"}
|
[fixes #4745] Don't query mailserver on restored account
When creating a new account / recovery we don't poll the mailserver anymore for historic messages, which solves the immediate issue of fetching only received messages
Handle messages sent from a different device in public chat / restore history. The message will be added, shown correctly as sent by the user, and the status will be set as sent ( need to check for seen race condition, as messages will now be added twice). This means that multidevice should now work for public chats.
Move contact updates to discovery topic. This is necessary as there is a pre-existing bug whereby contact updates would not work anymore after wallet recovery, as the code relies on the initial contact request being stored on the mailserver, which we cannot guarantee (we only pull 7 days of data). Not pulling history anymore exacerbate the problems but does not introduce it.
To make sure that contact updates will work after wallet recovery, we also need to consider a ContactUpdate in the same way we consider a ContactRequest (the other peer has no idea that the user has recovered the wallet). This does not change any behaviour in terms of obscurity/security as ContactRequest are automatically processed (in both case the contact will be set as pending?, not as accepted)
At this stage ContactRequest, ContactRequestConfirmed, ContactUpdate have all the same logic, i.e. update the contact information, leave the pending flag alone.
Only 1 day of history is fetched for newly joined chats, if catching up 7 days is the cap as before.
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2018-06-19 11:43:47 +00:00
|
|
|
:view-id :chat
|
2020-03-16 08:46:58 +00:00
|
|
|
:loaded-chat-id "chat-id"
|
[fixes #4745] Don't query mailserver on restored account
When creating a new account / recovery we don't poll the mailserver anymore for historic messages, which solves the immediate issue of fetching only received messages
Handle messages sent from a different device in public chat / restore history. The message will be added, shown correctly as sent by the user, and the status will be set as sent ( need to check for seen race condition, as messages will now be added twice). This means that multidevice should now work for public chats.
Move contact updates to discovery topic. This is necessary as there is a pre-existing bug whereby contact updates would not work anymore after wallet recovery, as the code relies on the initial contact request being stored on the mailserver, which we cannot guarantee (we only pull 7 days of data). Not pulling history anymore exacerbate the problems but does not introduce it.
To make sure that contact updates will work after wallet recovery, we also need to consider a ContactUpdate in the same way we consider a ContactRequest (the other peer has no idea that the user has recovered the wallet). This does not change any behaviour in terms of obscurity/security as ContactRequest are automatically processed (in both case the contact will be set as pending?, not as accepted)
At this stage ContactRequest, ContactRequestConfirmed, ContactUpdate have all the same logic, i.e. update the contact information, leave the pending flag alone.
Only 1 day of history is fetched for newly joined chats, if catching up 7 days is the cap as before.
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2018-06-19 11:43:47 +00:00
|
|
|
:current-chat-id "chat-id"
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
:messages {"chat-id" {}}
|
|
|
|
:chats {"chat-id" {}}}]
|
[fixes #4745] Don't query mailserver on restored account
When creating a new account / recovery we don't poll the mailserver anymore for historic messages, which solves the immediate issue of fetching only received messages
Handle messages sent from a different device in public chat / restore history. The message will be added, shown correctly as sent by the user, and the status will be set as sent ( need to check for seen race condition, as messages will now be added twice). This means that multidevice should now work for public chats.
Move contact updates to discovery topic. This is necessary as there is a pre-existing bug whereby contact updates would not work anymore after wallet recovery, as the code relies on the initial contact request being stored on the mailserver, which we cannot guarantee (we only pull 7 days of data). Not pulling history anymore exacerbate the problems but does not introduce it.
To make sure that contact updates will work after wallet recovery, we also need to consider a ContactUpdate in the same way we consider a ContactRequest (the other peer has no idea that the user has recovered the wallet). This does not change any behaviour in terms of obscurity/security as ContactRequest are automatically processed (in both case the contact will be set as pending?, not as accepted)
At this stage ContactRequest, ContactRequestConfirmed, ContactUpdate have all the same logic, i.e. update the contact information, leave the pending flag alone.
Only 1 day of history is fetched for newly joined chats, if catching up 7 days is the cap as before.
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2018-06-19 11:43:47 +00:00
|
|
|
(testing "a message coming from you!"
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
(let [actual (message/receive-one {:db db}
|
|
|
|
{:from "me"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-one-to-one
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:timestamp 0
|
|
|
|
:whisper-timestamp 0
|
|
|
|
:message-id "id"
|
|
|
|
:chat-id "chat-id"
|
2019-11-26 13:15:19 +00:00
|
|
|
:outgoing true
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:content "b"
|
|
|
|
:clock-value 1})
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
message (get-in actual [:db :messages "chat-id" "id"])]
|
[fixes #4745] Don't query mailserver on restored account
When creating a new account / recovery we don't poll the mailserver anymore for historic messages, which solves the immediate issue of fetching only received messages
Handle messages sent from a different device in public chat / restore history. The message will be added, shown correctly as sent by the user, and the status will be set as sent ( need to check for seen race condition, as messages will now be added twice). This means that multidevice should now work for public chats.
Move contact updates to discovery topic. This is necessary as there is a pre-existing bug whereby contact updates would not work anymore after wallet recovery, as the code relies on the initial contact request being stored on the mailserver, which we cannot guarantee (we only pull 7 days of data). Not pulling history anymore exacerbate the problems but does not introduce it.
To make sure that contact updates will work after wallet recovery, we also need to consider a ContactUpdate in the same way we consider a ContactRequest (the other peer has no idea that the user has recovered the wallet). This does not change any behaviour in terms of obscurity/security as ContactRequest are automatically processed (in both case the contact will be set as pending?, not as accepted)
At this stage ContactRequest, ContactRequestConfirmed, ContactUpdate have all the same logic, i.e. update the contact information, leave the pending flag alone.
Only 1 day of history is fetched for newly joined chats, if catching up 7 days is the cap as before.
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2018-06-19 11:43:47 +00:00
|
|
|
(testing "it adds the message"
|
2019-11-26 13:15:19 +00:00
|
|
|
(is message))))))
|
2019-02-20 13:47:04 +00:00
|
|
|
|
2018-07-19 15:51:06 +00:00
|
|
|
(deftest receive-group-chats
|
2018-12-17 14:59:04 +00:00
|
|
|
(let [cofx {:db {:chats {"chat-id" {:contacts #{"present"}
|
|
|
|
:members-joined #{"a"}}}
|
2019-07-03 14:29:01 +00:00
|
|
|
:multiaccount {:public-key "a"}
|
2020-03-16 08:46:58 +00:00
|
|
|
:loaded-chat-id "chat-id"
|
2018-07-19 15:51:06 +00:00
|
|
|
:current-chat-id "chat-id"
|
|
|
|
:view-id :chat}}
|
2018-12-17 14:59:04 +00:00
|
|
|
cofx-without-member (update-in cofx [:db :chats "chat-id" :members-joined] disj "a")
|
2018-07-19 15:51:06 +00:00
|
|
|
valid-message {:chat-id "chat-id"
|
|
|
|
:from "present"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-private-group
|
2018-07-19 15:51:06 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2018-07-19 15:51:06 +00:00
|
|
|
:timestamp 0}
|
|
|
|
bad-chat-id-message {:chat-id "bad-chat-id"
|
|
|
|
:from "present"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-private-group
|
2018-07-19 15:51:06 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2018-07-19 15:51:06 +00:00
|
|
|
:timestamp 0}
|
|
|
|
bad-from-message {:chat-id "chat-id"
|
|
|
|
:from "not-present"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-private-group
|
2018-07-19 15:51:06 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2018-07-19 15:51:06 +00:00
|
|
|
:timestamp 0}]
|
|
|
|
(testing "a valid message"
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(is (get-in (message/receive-one cofx valid-message) [:db :messages "chat-id" "1"])))
|
2018-07-19 15:51:06 +00:00
|
|
|
(testing "a message from someone not in the list of participants"
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
(is (not (message/receive-one cofx bad-from-message))))
|
2018-07-19 15:51:06 +00:00
|
|
|
(testing "a message with non existing chat-id"
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
(is (not (message/receive-one cofx bad-chat-id-message))))
|
2018-11-29 10:06:06 +00:00
|
|
|
(testing "a message from a delete chat"
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
(is (not (message/receive-one cofx-without-member valid-message))))))
|
2018-07-19 15:51:06 +00:00
|
|
|
|
|
|
|
(deftest receive-public-chats
|
|
|
|
(let [cofx {:db {:chats {"chat-id" {:public? true}}
|
2019-07-03 14:29:01 +00:00
|
|
|
:multiaccount {:public-key "a"}
|
2020-03-16 08:46:58 +00:00
|
|
|
:loaded-chat-id "chat-id"
|
2018-07-19 15:51:06 +00:00
|
|
|
:current-chat-id "chat-id"
|
|
|
|
:view-id :chat}}
|
|
|
|
valid-message {:chat-id "chat-id"
|
|
|
|
:from "anyone"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-public-group
|
2018-07-19 15:51:06 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2018-07-19 15:51:06 +00:00
|
|
|
:timestamp 0}
|
|
|
|
bad-chat-id-message {:chat-id "bad-chat-id"
|
|
|
|
:from "present"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-public-group
|
2018-07-19 15:51:06 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2018-07-19 15:51:06 +00:00
|
|
|
:timestamp 0}]
|
|
|
|
(testing "a valid message"
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(is (get-in (message/receive-one cofx valid-message) [:db :messages "chat-id" "1"])))
|
2018-07-19 15:51:06 +00:00
|
|
|
(testing "a message with non existing chat-id"
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
(is (not (message/receive-one cofx bad-chat-id-message))))))
|
2018-07-19 15:51:06 +00:00
|
|
|
|
|
|
|
(deftest receive-one-to-one
|
2019-09-12 09:41:25 +00:00
|
|
|
(with-redefs [gfycat/generate-gfy (constantly "generated")
|
|
|
|
identicon/identicon (constantly "generated")]
|
2018-07-19 15:51:06 +00:00
|
|
|
|
2019-09-12 09:41:25 +00:00
|
|
|
(let [cofx {:db {:chats {"matching" {}}
|
|
|
|
:multiaccount {:public-key "me"}
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:current-chat-id "matching"
|
2020-03-16 08:46:58 +00:00
|
|
|
:loaded-chat-id "matching"
|
2019-09-12 09:41:25 +00:00
|
|
|
:view-id :chat}}
|
|
|
|
valid-message {:chat-id "matching"
|
|
|
|
:from "matching"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-one-to-one
|
2019-09-12 09:41:25 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2019-09-12 09:41:25 +00:00
|
|
|
:timestamp 0}
|
|
|
|
own-message {:chat-id "matching"
|
|
|
|
:from "me"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-one-to-one
|
2019-09-12 09:41:25 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2019-09-12 09:41:25 +00:00
|
|
|
:timestamp 0}
|
|
|
|
|
|
|
|
bad-chat-id-message {:chat-id "bad-chat-id"
|
|
|
|
:from "not-matching"
|
2019-11-26 13:15:19 +00:00
|
|
|
:message-type constants/message-type-one-to-one
|
2019-09-12 09:41:25 +00:00
|
|
|
:message-id "1"
|
|
|
|
:clock-value 1
|
Fix message ordering and improve performance rec. messages
This commit does a few things:
==== Ordering of messages ====
Change the ordering of messages from a mixture of timestamp/clock-value to use
only clock-value.
Datemarks are now not used for sorting anymore, which means that the
order of messages is always causally related (not the case before, as we
were breaking this property by sorting by datemark), but datemark
calculation is unreliable (a reply to a message might have a timestamp <
then the message that is replied to).
So for timestamp calculation we
naively group them ignoring "out-of-order timestamp" messages, although
there's much to improve.
It fixes an issue whereby the user would change their time and the
message will be displayed in the past, although it is still possible to
craft a message with a lower clock value and order it in the past
(there's no way we can prevent this to some extent, but there are ways
to mitigate, but outside the scope of this PR).
==== Performance of receiving messages ====
The app would freeze on pulling messages from a mailserver (100 or so).
This is due to the JS Thread being hogged by CPU calculation, coupled
with the fact that we always tried to process messages all in one go.
This strategy can't scale, and given x is big enough (200,300,1000) the
UI will freeze.
Instead, each message is now processed separately, and we leave a gap
between processing each message for the UI to respond to user input
(otherwise the app freezes again).
Pulling messages will be longer overall, but the app will be usuable
while this happen (albeit it might slow down).
Other strategies are possible (calculate off-db and do a big swap,
avoiding many re-renders etc), but this is the reccommended strategy by
re-frame author (Solving the CPU Hog problem), so sounds like a safe
base point.
The underlying data structure for holding messages was also changed, we
used an immutable Red and Black Tree, same as a sorted map for clojure, but we use
a js library as is twice as performing then clojure sorted map.
We also don't sort messages again each time we receive them O(nlogn), but we
insert them in order O(logn).
Other data structures considered but discarded:
1) Plain vector, but performance prepending/insertion in the middle
(both O(n)) were not great, as not really suited for these operations.
2) Linked list, appealing as append/prepend is O(1), while insertion is
O(n). This is probably acceptable as messages tend to come in order
(from the db, so adding N messages is O(n)), or the network (most of
them prepends, or close to the head), while mailserver would not follow this path.
An implementation of a linked list was built, which performed roughtly the
same as a clojure sorted-map (although faster append/prepend), but not
worth the complexity of having our own implementation.
3) Clojure sorted-map, probably the most versatile, performance were
acceptable, but nowhere near the javascript implementation we decided on
4) Priority map, much slower than a sorted map (twice as slow)
5) Mutable sorted map, js implementation, (bintrees), not explored this very much, but from
just a quick benchmark, performance were much worse that clojure
immutable sorted map
Given that each message is now processed separately, saving the chat /
messages is also debounced to avoid spamming status-go with network
requests. This is a temporary measure for now until that's done directly
in status-go, without having to ping-pong with status-react.
Next steps performance wise is to move stuff to status-go, parsing of
transit, validation, which is heavy, at which point we can re-consider
performance and how to handle messages.
Fixes also an issue with the last message in the chat, we were using the
last message in the chat list, which might not necessarely be the last
message the chat has seen, in case messages were not loaded and a more
recent message is the database (say you fetch historical messages for
1-to-1 A, you don't have any messages in 1-to-1 chat B loaded, you receive an
historical message for chat B, it sets it as last message).
Also use clj beans instead of js->clj for type conversion
Signed-off-by: Andrea Maria Piana <andrea.maria.piana@gmail.com>
2019-10-24 14:23:20 +00:00
|
|
|
:whisper-timestamp 0
|
2019-09-12 09:41:25 +00:00
|
|
|
:timestamp 0}]
|
|
|
|
(testing "a valid message"
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(is (get-in (message/receive-one cofx valid-message) [:db :messages "matching" "1"])))
|
2019-09-12 09:41:25 +00:00
|
|
|
(testing "our own message"
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(is (get-in (message/receive-one cofx own-message) [:db :messages "matching" "1"])))
|
2019-09-12 09:41:25 +00:00
|
|
|
(testing "a message with non matching chat-id"
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(is (not (get-in (message/receive-one cofx bad-chat-id-message) [:db :messages "not-matching" "1"])))))))
|
2018-07-19 15:51:06 +00:00
|
|
|
|
2018-05-10 15:19:51 +00:00
|
|
|
(deftest delete-message
|
2019-11-12 06:07:17 +00:00
|
|
|
(with-redefs [time/day-relative (constantly "day-relative")
|
|
|
|
time/timestamp->time (constantly "timestamp")]
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(let [cofx1 {:db {:messages {"chat-id" {0 {:message-id 0
|
|
|
|
:content "a"
|
|
|
|
:clock-value 0
|
|
|
|
:whisper-timestamp 0
|
|
|
|
:timestamp 0}
|
|
|
|
1 {:message-id 1
|
|
|
|
:content "b"
|
|
|
|
:clock-value 1
|
|
|
|
:whisper-timestamp 1
|
|
|
|
:timestamp 1}}}
|
|
|
|
:message-lists {"chat-id" [{:something :something}]}
|
|
|
|
:chats {"chat-id" {}}}}
|
|
|
|
cofx2 {:db {:messages {"chat-id" {0 {:message-id 0
|
|
|
|
:content "a"
|
|
|
|
:clock-value 0
|
|
|
|
:whisper-timestamp 1
|
|
|
|
:timestamp 1}}}
|
|
|
|
:message-list {"chat-id" [{:something :something}]}
|
|
|
|
:chats {"chat-id" {}}}}
|
2019-11-12 06:07:17 +00:00
|
|
|
fx1 (message/delete-message cofx1 "chat-id" 1)
|
|
|
|
fx2 (message/delete-message cofx2 "chat-id" 0)]
|
|
|
|
(testing "Deleting message deletes it along with all references"
|
|
|
|
(is (= '(0)
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(keys (get-in fx1 [:db :messages "chat-id"]))))
|
2019-11-12 06:07:17 +00:00
|
|
|
(is (= [{:one-to-one? false
|
|
|
|
:message-id 0
|
|
|
|
:whisper-timestamp 0
|
|
|
|
:type :message
|
|
|
|
:display-photo? true
|
|
|
|
:system-message? false
|
|
|
|
:last-in-group? true
|
|
|
|
:datemark "day-relative"
|
|
|
|
:clock-value 0
|
|
|
|
:first-in-group? true
|
|
|
|
:from nil
|
2019-11-26 13:15:19 +00:00
|
|
|
:first-outgoing? false
|
|
|
|
:outgoing-seen? false
|
2019-11-12 06:07:17 +00:00
|
|
|
:timestamp-str "timestamp"
|
|
|
|
:first? true
|
|
|
|
:display-username? true
|
2019-11-26 13:15:19 +00:00
|
|
|
:outgoing false}]
|
2019-11-12 06:07:17 +00:00
|
|
|
(models.message-list/->seq
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(get-in fx1 [:db :message-lists "chat-id"]))))
|
2019-11-12 06:07:17 +00:00
|
|
|
(is (= {}
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(get-in fx2 [:db :messages "chat-id"])))
|
2019-11-12 06:07:17 +00:00
|
|
|
(is (= nil
|
Improve chat loading performance
This commit does a few things:
Move collections top level
Move `messages`,`message-lists`,`pagination-info` from nested in
`chats` to top level at the db.
The reason for this change is that if any of the `messages` fields
change, any `sub` that relies on `chat` will be recomputed, which is
unnecessary.
Move chat-name to events
`chat-name` was computed dynamically, while it is now only calculated
when loading chat the first time around.
Remove `enrich-chats`
Enrich chats was doing a lot of work, and many subscriptions were
relying on it.
Not all the computations were necessary, for example it would always
calculate the name of who invited the user to a group chat, regardless
of whether it was actually used in the view.
This commit changes that behavior so that we use smaller subscriptions
to calculate such fields.
In general we should move computations to events, if that's not
desirable (there are some cases where we might not want to do that), we
should have "bottom/leaf heavy" subscriptions as opposed to "top heavy",
especially if they are to be shared, so only when (and if) we load that
particular view, the subscription is triggered, while others can be
re-used.
I have compared performance with current release, and there's a
noticeable difference. Opening a chat is faster (messages are loaded
faster), and clicking on the home view on a chat is more responsing
(the animation on-press is much quicker).
2020-05-27 14:29:33 +00:00
|
|
|
(get-in fx2 [:db :message-lists "chat-id"])))))))
|