Add replies to messages

Currently replies to messages are handled in status-react.
This causes some issues with the fact that sometimes replies might come
out of order, they might be offloaded to the database etc.

This commit changes the behavior so that status-go always returns the
replies, and in case a reply comes out of order (first the reply, later
the message being replied to), it will include in the messages the
updated message.

It also adds some fields (RTL,Replace,LineCount) to the database which
were not previously saved, resulting in some potential bugs.

The method that we use to pull replies is currently a bit naive, we just
pull all the message again from the database, but has the advantage of
being simple. It will go through performance testing to make sure
performnace are acceptable, if so I think it's reasonable to avoid some
complexity.
This commit is contained in:
Andrea Maria Piana 2020-04-08 15:42:02 +02:00
parent 4d2fb67add
commit 3a84afd0f1
11 changed files with 178 additions and 14 deletions

View File

@ -1 +1 @@
0.52.1
0.52.2

View File

@ -296,7 +296,7 @@ func _0006_appearanceUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "0006_appearance.up.sql", size: 67, mode: os.FileMode(0644), modTime: time.Unix(1585125858, 0)}
info := bindataFileInfo{name: "0006_appearance.up.sql", size: 67, mode: os.FileMode(0644), modTime: time.Unix(1585895847, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xae, 0x6, 0x25, 0x6c, 0xe4, 0x9d, 0xa7, 0x72, 0xe8, 0xbc, 0xe4, 0x1f, 0x1e, 0x2d, 0x7c, 0xb7, 0xf6, 0xa3, 0xec, 0x3b, 0x4e, 0x93, 0x2e, 0xa4, 0xec, 0x6f, 0xe5, 0x95, 0x94, 0xe8, 0x4, 0xfb}}
return a, nil
}
@ -316,7 +316,7 @@ func _0007_enable_waku_defaultUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "0007_enable_waku_default.up.sql", size: 38, mode: os.FileMode(0644), modTime: time.Unix(1585302708, 0)}
info := bindataFileInfo{name: "0007_enable_waku_default.up.sql", size: 38, mode: os.FileMode(0644), modTime: time.Unix(1585895900, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd4, 0x42, 0xb6, 0xe5, 0x48, 0x41, 0xeb, 0xc0, 0x7e, 0x3b, 0xe6, 0x8e, 0x96, 0x33, 0x20, 0x92, 0x24, 0x5a, 0x60, 0xfa, 0xa0, 0x3, 0x5e, 0x76, 0x4b, 0x89, 0xaa, 0x37, 0x66, 0xbc, 0x26, 0x11}}
return a, nil
}

View File

@ -87,9 +87,12 @@ type Message struct {
CommandParameters *CommandParameters `json:"commandParameters"`
// Computed fields
RTL bool `json:"rtl"`
// RTL is whether this is a right-to-left message (arabic/hebrew script etc)
RTL bool `json:"rtl"`
// ParsedText is the parsed markdown for displaying
ParsedText []byte `json:"parsedText"`
LineCount int `json:"lineCount"`
// LineCount is the count of newlines in the message
LineCount int `json:"lineCount"`
// Replace indicates that this is a replacement of a message
// that has been updated

View File

@ -1453,8 +1453,12 @@ func (m *Messenger) SendChatMessage(ctx context.Context, message *Message) (*Mes
return nil, err
}
response.Messages, err = m.pullMessagesAndResponsesFromDB([]*Message{message})
if err != nil {
return nil, err
}
response.Chats = []*Chat{chat}
response.Messages = []*Message{message}
return &response, m.saveChat(chat)
}
@ -2012,6 +2016,12 @@ func (m *Messenger) handleRetrievedMessages(chatWithMessages map[transport.Filte
}
}
messagesWithResponses, err := m.pullMessagesAndResponsesFromDB(messageState.Response.Messages)
if err != nil {
return nil, err
}
messageState.Response.Messages = messagesWithResponses
// Reset installations
m.modifiedInstallations = make(map[string]bool)
@ -2883,6 +2893,22 @@ func (m *Messenger) ValidateTransactions(ctx context.Context, addresses []types.
return &response, nil
}
// pullMessagesAndResponsesFromDB pulls all the messages and the one that have
// been replied to from the database
func (m *Messenger) pullMessagesAndResponsesFromDB(messages []*Message) ([]*Message, error) {
var messageIDs []string
for _, message := range messages {
messageIDs = append(messageIDs, message.ID)
if len(message.ResponseTo) != 0 {
messageIDs = append(messageIDs, message.ResponseTo)
}
}
// We pull from the database all the messages & replies involved,
// so we let the db build the correct messages
return m.persistence.MessagesByIDs(messageIDs)
}
func (m *Messenger) getTimesource() TimeSource {
return m.transport
}

View File

@ -4,6 +4,8 @@
// 000001_init.up.db.sql (2.719kB)
// 000002_add_last_ens_clock_value.down.sql (0)
// 000002_add_last_ens_clock_value.up.sql (77B)
// 1586358095_add_replace.down.sql (0)
// 1586358095_add_replace.up.sql (224B)
// doc.go (377B)
package migrations
@ -88,7 +90,7 @@ func _000001_initDownDbSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "000001_init.down.db.sql", size: 65, mode: os.FileMode(0644), modTime: time.Unix(1581443166, 0)}
info := bindataFileInfo{name: "000001_init.down.db.sql", size: 65, mode: os.FileMode(0644), modTime: time.Unix(1578682784, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x5e, 0xbb, 0x3f, 0x1, 0x75, 0x19, 0x70, 0x86, 0xa7, 0x34, 0x40, 0x17, 0x34, 0x3e, 0x18, 0x51, 0x79, 0xd4, 0x22, 0xad, 0x8f, 0x80, 0xcc, 0xa6, 0xcc, 0x6, 0x2b, 0x62, 0x2, 0x47, 0xba, 0xf9}}
return a, nil
}
@ -108,7 +110,7 @@ func _000001_initUpDbSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "000001_init.up.db.sql", size: 2719, mode: os.FileMode(0644), modTime: time.Unix(1581443166, 0)}
info := bindataFileInfo{name: "000001_init.up.db.sql", size: 2719, mode: os.FileMode(0644), modTime: time.Unix(1578682784, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x60, 0xdc, 0xeb, 0xe, 0xc2, 0x4f, 0x75, 0xa, 0xf6, 0x3e, 0xc7, 0xc4, 0x4, 0xe2, 0xe1, 0xa4, 0x73, 0x2f, 0x4a, 0xad, 0x1a, 0x0, 0xc3, 0x93, 0x9d, 0x77, 0x3e, 0x31, 0x91, 0x77, 0x2e, 0xc8}}
return a, nil
}
@ -128,7 +130,7 @@ func _000002_add_last_ens_clock_valueDownSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "000002_add_last_ens_clock_value.down.sql", size: 0, mode: os.FileMode(0644), modTime: time.Unix(1581443166, 0)}
info := bindataFileInfo{name: "000002_add_last_ens_clock_value.down.sql", size: 0, mode: os.FileMode(0644), modTime: time.Unix(1584434371, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}
return a, nil
}
@ -148,11 +150,51 @@ func _000002_add_last_ens_clock_valueUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "000002_add_last_ens_clock_value.up.sql", size: 77, mode: os.FileMode(0644), modTime: time.Unix(1581443166, 0)}
info := bindataFileInfo{name: "000002_add_last_ens_clock_value.up.sql", size: 77, mode: os.FileMode(0644), modTime: time.Unix(1584434371, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4d, 0x3, 0x8f, 0xd5, 0x85, 0x83, 0x47, 0xbe, 0xf9, 0x82, 0x7e, 0x81, 0xa4, 0xbd, 0xaa, 0xd5, 0x98, 0x18, 0x5, 0x2d, 0x82, 0x42, 0x3b, 0x3, 0x50, 0xc3, 0x1e, 0x84, 0x35, 0xf, 0xb6, 0x2b}}
return a, nil
}
var __1586358095_add_replaceDownSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x01\x00\x00\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00")
func _1586358095_add_replaceDownSqlBytes() ([]byte, error) {
return bindataRead(
__1586358095_add_replaceDownSql,
"1586358095_add_replace.down.sql",
)
}
func _1586358095_add_replaceDownSql() (*asset, error) {
bytes, err := _1586358095_add_replaceDownSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "1586358095_add_replace.down.sql", size: 0, mode: os.FileMode(0644), modTime: time.Unix(1586355165, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}
return a, nil
}
var __1586358095_add_replaceUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\xce\x31\x0a\xc2\x40\x10\x46\xe1\x3e\xa7\xf8\xc9\x09\xec\x53\x4d\xdc\x09\x08\xe3\x2c\xe8\x2c\xd8\x85\x10\x06\x11\xd6\x28\xd9\xe4\xfe\x56\x56\x5a\x6c\xfd\xe0\xe3\x91\x18\x5f\x60\xd4\x0b\x63\x2f\xbe\x8e\x4f\x2f\x65\xba\x7b\x01\x85\x80\x63\x94\x74\x56\xac\xfe\xce\xd3\xec\xdf\x06\xe3\x9b\x41\xa3\x41\x93\x08\x02\x0f\x94\xc4\xd0\xb6\x5d\x53\xc7\x6d\x19\x7d\x8c\xc2\xa4\xbf\xca\x40\x72\xe5\x4a\x28\x3f\x16\x1f\xe7\xd7\xbe\x6c\x38\xe9\x9f\xa3\x43\xd7\x7c\x02\x00\x00\xff\xff\xca\x94\x3f\xe0\xe0\x00\x00\x00")
func _1586358095_add_replaceUpSqlBytes() ([]byte, error) {
return bindataRead(
__1586358095_add_replaceUpSql,
"1586358095_add_replace.up.sql",
)
}
func _1586358095_add_replaceUpSql() (*asset, error) {
bytes, err := _1586358095_add_replaceUpSqlBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "1586358095_add_replace.up.sql", size: 224, mode: os.FileMode(0644), modTime: time.Unix(1586352560, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd2, 0xb3, 0xa9, 0xc7, 0x7f, 0x9d, 0x8f, 0x43, 0x8c, 0x9e, 0x58, 0x8d, 0x44, 0xbc, 0xfa, 0x6b, 0x5f, 0x3f, 0x5a, 0xbe, 0xe8, 0xb1, 0x16, 0xf, 0x91, 0x2a, 0xa0, 0x71, 0xbb, 0x8d, 0x6b, 0xcb}}
return a, nil
}
var _docGo = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x8f\xbb\x6e\xc3\x30\x0c\x45\x77\x7f\xc5\x45\x96\x2c\xb5\xb4\x74\xea\xd6\xb1\x7b\x7f\x80\x91\x68\x89\x88\x1e\xae\x48\xe7\xf1\xf7\x85\xd3\x02\xcd\xd6\xf5\x00\xe7\xf0\xd2\x7b\x7c\x66\x51\x2c\x52\x18\xa2\x68\x1c\x58\x95\xc6\x1d\x27\x0e\xb4\x29\xe3\x90\xc4\xf2\x76\x72\xa1\x57\xaf\x46\xb6\xe9\x2c\xd5\x57\x49\x83\x8c\xfd\xe5\xf5\x30\x79\x8f\x40\xed\x68\xc8\xd4\x62\xe1\x47\x4b\xa1\x46\xc3\xa4\x25\x5c\xc5\x32\x08\xeb\xe0\x45\x6e\x0e\xef\x86\xc2\xa4\x06\xcb\x64\x47\x85\x65\x46\x20\xe5\x3d\xb3\xf4\x81\xd4\xe7\x93\xb4\x48\x46\x6e\x47\x1f\xcb\x13\xd9\x17\x06\x2a\x85\x23\x96\xd1\xeb\xc3\x55\xaa\x8c\x28\x83\x83\xf5\x71\x7f\x01\xa9\xb2\xa1\x51\x65\xdd\xfd\x4c\x17\x46\xeb\xbf\xe7\x41\x2d\xfe\xff\x11\xae\x7d\x9c\x15\xa4\xe0\xdb\xca\xc1\x38\xba\x69\x5a\x29\x9c\x29\x31\xf4\xab\x88\xf1\x34\x79\x9f\xfa\x5b\xe2\xc6\xbb\xf5\xbc\x71\x5e\xcf\x09\x3f\x35\xe9\x4d\x31\x77\x38\xe7\xff\x80\x4b\x1d\x6e\xfa\x0e\x00\x00\xff\xff\x9d\x60\x3d\x88\x79\x01\x00\x00")
func docGoBytes() ([]byte, error) {
@ -168,7 +210,7 @@ func docGo() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "doc.go", size: 377, mode: os.FileMode(0644), modTime: time.Unix(1581443166, 0)}
info := bindataFileInfo{name: "doc.go", size: 377, mode: os.FileMode(0644), modTime: time.Unix(1574354941, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xef, 0xaf, 0xdf, 0xcf, 0x65, 0xae, 0x19, 0xfc, 0x9d, 0x29, 0xc1, 0x91, 0xaf, 0xb5, 0xd5, 0xb1, 0x56, 0xf3, 0xee, 0xa8, 0xba, 0x13, 0x65, 0xdb, 0xab, 0xcf, 0x4e, 0xac, 0x92, 0xe9, 0x60, 0xf1}}
return a, nil
}
@ -272,6 +314,10 @@ var _bindata = map[string]func() (*asset, error){
"000002_add_last_ens_clock_value.up.sql": _000002_add_last_ens_clock_valueUpSql,
"1586358095_add_replace.down.sql": _1586358095_add_replaceDownSql,
"1586358095_add_replace.up.sql": _1586358095_add_replaceUpSql,
"doc.go": docGo,
}
@ -320,6 +366,8 @@ var _bintree = &bintree{nil, map[string]*bintree{
"000001_init.up.db.sql": &bintree{_000001_initUpDbSql, map[string]*bintree{}},
"000002_add_last_ens_clock_value.down.sql": &bintree{_000002_add_last_ens_clock_valueDownSql, map[string]*bintree{}},
"000002_add_last_ens_clock_value.up.sql": &bintree{_000002_add_last_ens_clock_valueUpSql, map[string]*bintree{}},
"1586358095_add_replace.down.sql": &bintree{_1586358095_add_replaceDownSql, map[string]*bintree{}},
"1586358095_add_replace.up.sql": &bintree{_1586358095_add_replaceUpSql, map[string]*bintree{}},
"doc.go": &bintree{docGo, map[string]*bintree{}},
}}

View File

@ -0,0 +1,3 @@
ALTER TABLE user_messages ADD COLUMN replace_message TEXT NOT NULL DEFAULT "";
ALTER TABLE user_messages ADD COLUMN rtl BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE user_messages ADD COLUMN line_count INT NOT NULL DEFAULT 0;

View File

@ -4,6 +4,14 @@
// If go-bindata is called from the same directory, asset names
// have no prefix and "github.com/status-im/migrate/v4" works as expected.
// NOTE: The naming of the file needs to follow the unix timestamp convention
// unlike the two initial files.
// That's because currently the encryption migrations and protocol migrations
// share the same table, which means that the latest migration in terms of filename
// is from the encryption namespace files, which uses a unix timestamp.
// If we add 00003_... for example, this will be ignored in existing clients as
// older than the latest encryption migration.
package sqlite
//go:generate go-bindata -pkg migrations -o ../migrations.go .

View File

@ -41,6 +41,9 @@ func (db sqlitePersistence) tableUserMessagesLegacyAllFields() string {
command_transaction_hash,
command_state,
command_signature,
replace_message,
rtl,
line_count,
response_to`
}
@ -69,6 +72,9 @@ func (db sqlitePersistence) tableUserMessagesLegacyAllFieldsJoin() string {
m1.command_transaction_hash,
m1.command_state,
m1.command_signature,
m1.replace_message,
m1.rtl,
m1.line_count,
m1.response_to,
m2.source,
m2.text,
@ -118,6 +124,9 @@ func (db sqlitePersistence) tableUserMessagesLegacyScanAllFields(row scanner, me
&command.TransactionHash,
&command.CommandState,
&command.Signature,
&message.Replace,
&message.RTL,
&message.LineCount,
&message.ResponseTo,
&quotedFrom,
&quotedText,
@ -182,6 +191,9 @@ func (db sqlitePersistence) tableUserMessagesLegacyAllValues(message *Message) (
command.TransactionHash,
command.CommandState,
command.Signature,
message.Replace,
message.RTL,
message.LineCount,
message.ResponseTo,
}, nil
}
@ -313,6 +325,52 @@ func (db sqlitePersistence) MessagesExist(ids []string) (map[string]bool, error)
return result, nil
}
func (db sqlitePersistence) MessagesByIDs(ids []string) ([]*Message, error) {
if len(ids) == 0 {
return nil, nil
}
idsArgs := make([]interface{}, 0, len(ids))
for _, id := range ids {
idsArgs = append(idsArgs, id)
}
allFields := db.tableUserMessagesLegacyAllFieldsJoin()
inVector := strings.Repeat("?, ", len(ids)-1) + "?"
// nolint: gosec
rows, err := db.db.Query(fmt.Sprintf(`
SELECT
%s
FROM
user_messages m1
LEFT JOIN
user_messages m2
ON
m1.response_to = m2.id
LEFT JOIN
contacts c
ON
m1.source = c.id
WHERE m1.hide != 1 AND m1.id IN (%s)`, allFields, inVector), idsArgs...)
if err != nil {
return nil, err
}
defer rows.Close()
var result []*Message
for rows.Next() {
var message Message
if err := db.tableUserMessagesLegacyScanAllFields(rows, &message); err != nil {
return nil, err
}
result = append(result, &message)
}
return result, nil
}
// MessageByChatID returns all messages for a given chatID in descending order.
// Ordering is accomplished using two concatenated values: ClockValue and ID.
// These two values are also used to compose a cursor which is returned to the result.

View File

@ -37,6 +37,24 @@ func TestSaveMessages(t *testing.T) {
}
}
func TestMessagesByIDs(t *testing.T) {
db, err := openTestDB()
require.NoError(t, err)
p := sqlitePersistence{db: db}
var ids []string
for i := 0; i < 10; i++ {
id := strconv.Itoa(i)
err := insertMinimalMessage(p, id)
require.NoError(t, err)
ids = append(ids, id)
}
m, err := p.MessagesByIDs(ids)
require.NoError(t, err)
require.Len(t, m, 10)
}
func TestMessageByID(t *testing.T) {
db, err := openTestDB()
require.NoError(t, err)

View File

@ -117,7 +117,7 @@ func ConfigCliFleetEthProdJson() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "../config/cli/fleet-eth.prod.json", size: 3625, mode: os.FileMode(0644), modTime: time.Unix(1585119952, 0)}
info := bindataFileInfo{name: "../config/cli/fleet-eth.prod.json", size: 3625, mode: os.FileMode(0644), modTime: time.Unix(1585559578, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x1c, 0x86, 0x93, 0x3b, 0x8c, 0xb4, 0x1b, 0x8b, 0x70, 0xe6, 0x90, 0x3, 0xc9, 0x93, 0x99, 0x97, 0x18, 0xb7, 0x49, 0xbf, 0x2d, 0x35, 0x98, 0xb8, 0x49, 0xa8, 0x73, 0x30, 0xf5, 0x2d, 0x7f, 0x5c}}
return a, nil
}
@ -137,7 +137,7 @@ func ConfigCliFleetEthStagingJson() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "../config/cli/fleet-eth.staging.json", size: 1846, mode: os.FileMode(0644), modTime: time.Unix(1585119952, 0)}
info := bindataFileInfo{name: "../config/cli/fleet-eth.staging.json", size: 1846, mode: os.FileMode(0644), modTime: time.Unix(1585559578, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd1, 0xef, 0x3d, 0x5c, 0x41, 0xb1, 0xc8, 0x54, 0xd3, 0x82, 0x60, 0xcf, 0x5f, 0x8b, 0x92, 0xd0, 0x41, 0xa7, 0xff, 0xfd, 0x2c, 0x95, 0x44, 0x68, 0xd3, 0xad, 0x16, 0x8c, 0x49, 0x10, 0x4e, 0xb7}}
return a, nil
}
@ -157,7 +157,7 @@ func ConfigCliFleetEthTestJson() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "../config/cli/fleet-eth.test.json", size: 1855, mode: os.FileMode(0644), modTime: time.Unix(1585119952, 0)}
info := bindataFileInfo{name: "../config/cli/fleet-eth.test.json", size: 1855, mode: os.FileMode(0644), modTime: time.Unix(1585559578, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe4, 0x90, 0x4, 0x6, 0x2b, 0xe4, 0x67, 0x7a, 0xfc, 0x70, 0xa9, 0xb, 0x85, 0x8c, 0xb3, 0x8, 0x8e, 0xee, 0xec, 0x64, 0xb7, 0xac, 0x4c, 0x6, 0x4d, 0xcc, 0x4f, 0xda, 0xbd, 0xed, 0x17, 0x53}}
return a, nil
}