diff --git a/appdatabase/migrations/bindata.go b/appdatabase/migrations/bindata.go index d644414e0..14a81c226 100644 --- a/appdatabase/migrations/bindata.go +++ b/appdatabase/migrations/bindata.go @@ -50,7 +50,7 @@ // 1630485153_networks.up.sql (394B) // 1632262444_profile_pictures_show_to.up.sql (81B) // 1635942153_add_telemetry_server_url_to_settings.up.sql (128B) -// 1635942154_add_backup_setting.up.sql (186B) +// 1635942154_add_backup_setting.up.sql (287B) // doc.go (74B) package migrations @@ -1120,7 +1120,7 @@ func _1635942153_add_telemetry_server_url_to_settingsUpSql() (*asset, error) { return a, nil } -var __1635942154_add_backup_settingUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xce\x31\x0a\xc2\x40\x10\x46\xe1\x3e\xa7\xf8\x8f\xa0\xf5\x62\x31\x71\x47\x10\xc6\x59\x89\x33\x75\xd8\xe8\x22\x62\x08\xc2\xae\xf7\xb7\xb0\x11\x2d\x3c\xc0\xfb\x78\x24\xc6\x03\x8c\x7a\x61\xd4\xd2\xda\x6d\xb9\x56\x50\x8c\xd8\x26\xf1\x83\x62\xca\xe7\xfb\xf3\x31\x96\x25\x4f\x73\xb9\xa0\x4f\x49\x98\x14\x9a\x0c\xea\x22\x88\xbc\x23\x17\x83\x0d\xce\xa1\xfb\xa7\xcd\xb9\xb6\xf1\x4d\x62\xaf\xf6\xcb\xac\x42\xe7\xc7\x48\xf6\x91\x9f\xd8\xbe\x2f\x36\x58\x87\xee\x15\x00\x00\xff\xff\x6b\x1d\x44\xa1\xba\x00\x00\x00") +var __1635942154_add_backup_settingUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xcf\xcb\x0a\xc2\x30\x10\x85\xe1\x7d\x9f\xe2\x3c\x82\xae\x4b\x17\x53\x33\x05\x61\x4c\xa4\x9d\xac\x4b\x5b\xe3\x05\x4b\x11\x12\xdf\x5f\xc4\x0b\x22\x62\xf7\xc3\x37\xff\x21\x51\xae\xa1\x54\x0a\x23\x86\x94\x4e\xd3\x21\x82\x8c\xc1\xca\x89\xdf\x58\xf4\xdd\x70\xbe\x5e\xda\x30\x75\xfd\x18\x76\x28\x9d\x13\x26\x0b\xc3\x15\x79\x51\x68\xed\x39\xcf\xe6\x90\xb1\x8b\xa9\x7d\x48\x58\x5b\x85\x75\x0a\xeb\x45\xde\xcc\x62\xde\x78\x86\xec\x43\x1a\x8e\x3f\x42\x2a\x92\x86\xf3\xcc\x6f\x0d\xe9\x07\xd0\xb0\x7e\x4f\x28\xb0\xfc\x7b\xf7\xfa\x50\xdc\xab\x6e\x01\x00\x00\xff\xff\x3f\xf3\xd1\x35\x1f\x01\x00\x00") func _1635942154_add_backup_settingUpSqlBytes() ([]byte, error) { return bindataRead( @@ -1135,8 +1135,8 @@ func _1635942154_add_backup_settingUpSql() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "1635942154_add_backup_setting.up.sql", size: 186, mode: os.FileMode(0644), modTime: time.Unix(1636971544, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x42, 0xc3, 0x25, 0xc6, 0xd2, 0x86, 0x53, 0xa3, 0x52, 0x71, 0x1b, 0x54, 0xc9, 0x0, 0xf0, 0xe6, 0x46, 0x3c, 0x64, 0xa0, 0x2b, 0xfb, 0xb2, 0x2e, 0xc6, 0x67, 0x67, 0x6c, 0x6c, 0x40, 0xee, 0x2f}} + info := bindataFileInfo{name: "1635942154_add_backup_setting.up.sql", size: 287, mode: os.FileMode(0644), modTime: time.Unix(1636971794, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xb7, 0xe7, 0xfb, 0x70, 0x80, 0x5, 0xb4, 0x7b, 0x67, 0x8, 0x6e, 0x5f, 0x45, 0x17, 0xd9, 0x5f, 0x18, 0x66, 0x2f, 0x8a, 0x4f, 0xd4, 0x15, 0xe5, 0x2b, 0xbb, 0x25, 0x7a, 0x30, 0xad, 0x4c, 0x1a}} return a, nil } diff --git a/appdatabase/migrations/sql/1635942154_add_backup_setting.up.sql b/appdatabase/migrations/sql/1635942154_add_backup_setting.up.sql index 3d766e52c..c45d0d60e 100644 --- a/appdatabase/migrations/sql/1635942154_add_backup_setting.up.sql +++ b/appdatabase/migrations/sql/1635942154_add_backup_setting.up.sql @@ -1,3 +1,5 @@ -ALTER TABLE settings ADD COLUMN backup_enabled BOOLEAN NOT NULL DEFAULT TRUE; +ALTER TABLE settings ADD COLUMN backup_enabled BOOLEAN DEFAULT TRUE; ALTER TABLE settings ADD COLUMN last_backup INT NOT NULL DEFAULT 0; +ALTER TABLE settings ADD COLUMN backup_fetched BOOLEAN DEFAULT FALSE; UPDATE settings SET backup_enabled = 1; +UPDATE settings SET backup_fetched = 0; diff --git a/cmd/populate-db/main.go b/cmd/populate-db/main.go index fb28164a8..add905c5d 100644 --- a/cmd/populate-db/main.go +++ b/cmd/populate-db/main.go @@ -34,6 +34,7 @@ import ( "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/identity/alias" "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/protocol/requests" wakuextn "github.com/status-im/status-go/services/wakuext" ) @@ -165,7 +166,7 @@ func main() { } keyString := common.PubkeyToHex(&key.PublicKey) - _, err = wakuext.AddContact(context.Background(), keyString) + _, err = wakuext.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(keyString)}) if err != nil { logger.Error("failed Add contact", "err", err) return @@ -185,7 +186,7 @@ func main() { return } - _, err = wakuext.AddContact(context.Background(), contact.ID) + _, err = wakuext.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(contact.ID)}) if err != nil { return } diff --git a/multiaccounts/accounts/database.go b/multiaccounts/accounts/database.go index 75d9cc43b..be7a360ed 100644 --- a/multiaccounts/accounts/database.go +++ b/multiaccounts/accounts/database.go @@ -765,6 +765,20 @@ func (db *Database) SetLastBackup(time uint64) error { return err } +func (db *Database) SetBackupFetched(fetched bool) error { + _, err := db.db.Exec("UPDATE settings SET backup_fetched = ?", fetched) + return err +} + +func (db *Database) BackupFetched() (bool, error) { + var result bool + err := db.db.QueryRow("SELECT backup_fetched FROM settings WHERE synthetic_id = 'id'").Scan(&result) + if err == sql.ErrNoRows { + return true, nil + } + return result, err +} + func (db *Database) ENSName() (string, error) { var result sql.NullString err := db.db.QueryRow("SELECT preferred_name FROM settings WHERE synthetic_id = 'id'").Scan(&result) diff --git a/protocol/anon_metrics_test.go b/protocol/anon_metrics_test.go new file mode 100644 index 000000000..e3b841887 --- /dev/null +++ b/protocol/anon_metrics_test.go @@ -0,0 +1,209 @@ +// In order to run these tests, you must run a PostgreSQL database. +// +// Using Docker: +// docker run -e POSTGRES_HOST_AUTH_METHOD=trust -d -p 5432:5432 postgres:9.6-alpine +// + +package protocol + +import ( + "context" + "crypto/ecdsa" + "testing" + + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + + bindata "github.com/status-im/migrate/v4/source/go_bindata" + + appmetricsDB "github.com/status-im/status-go/appmetrics" + gethbridge "github.com/status-im/status-go/eth-node/bridge/geth" + "github.com/status-im/status-go/eth-node/crypto" + "github.com/status-im/status-go/eth-node/types" + "github.com/status-im/status-go/postgres" + "github.com/status-im/status-go/protocol/anonmetrics" + "github.com/status-im/status-go/protocol/anonmetrics/migrations" + "github.com/status-im/status-go/protocol/tt" + "github.com/status-im/status-go/services/appmetrics" + "github.com/status-im/status-go/waku" +) + +func TestMessengerAnonMetricsSuite(t *testing.T) { + suite.Run(t, new(MessengerAnonMetricsSuite)) +} + +type MessengerAnonMetricsSuite struct { + suite.Suite + alice *Messenger // client instance of Messenger + bob *Messenger // server instance of Messenger + + aliceKey *ecdsa.PrivateKey // private key for the alice instance of Messenger + bobKey *ecdsa.PrivateKey // private key for the bob instance of Messenger + + // If one wants to send messages between different instances of Messenger, + // a single Waku service should be shared. + shh types.Waku + logger *zap.Logger +} + +func (s *MessengerAnonMetricsSuite) SetupSuite() { + // ResetDefaultTestPostgresDB Required to completely reset the Postgres DB + err := postgres.ResetDefaultTestPostgresDB() + s.NoError(err) +} + +func (s *MessengerAnonMetricsSuite) SetupTest() { + var err error + + s.logger = tt.MustCreateTestLogger() + + // Setup Waku things + config := waku.DefaultConfig + config.MinimumAcceptedPoW = 0 + shh := waku.New(&config, s.logger) + s.shh = gethbridge.NewGethWakuWrapper(shh) + s.Require().NoError(shh.Start()) + + // Generate private keys for Alice and Bob + s.aliceKey, err = crypto.GenerateKey() + s.Require().NoError(err) + + s.bobKey, err = crypto.GenerateKey() + s.Require().NoError(err) + + // Generate Alice Messenger as the client + amcc := &anonmetrics.ClientConfig{ + ShouldSend: true, + SendAddress: &s.bobKey.PublicKey, + Active: anonmetrics.ActiveClientPhrase, + } + s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)}) + s.Require().NoError(err) + _, err = s.alice.Start() + s.Require().NoError(err) + + // Generate Bob Messenger as the Server + amsc := &anonmetrics.ServerConfig{ + Enabled: true, + PostgresURI: postgres.DefaultTestURI, + Active: anonmetrics.ActiveServerPhrase, + } + s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)}) + s.Require().NoError(err) + + _, err = s.bob.Start() + s.Require().NoError(err) +} + +func (s *MessengerAnonMetricsSuite) TearDownTest() { + // Down migrate the DB + if s.bob.anonMetricsServer != nil { + postgresMigration := bindata.Resource(migrations.AssetNames(), migrations.Asset) + m, err := anonmetrics.MakeMigration(s.bob.anonMetricsServer.PostgresDB, postgresMigration) + s.NoError(err) + + err = m.Down() + s.NoError(err) + } + + // Shutdown messengers + s.NoError(s.alice.Shutdown()) + s.alice = nil + s.NoError(s.bob.Shutdown()) + s.bob = nil + _ = s.logger.Sync() +} + +func (s *MessengerAnonMetricsSuite) TestReceiveAnonMetric() { + // Create the appmetrics API to simulate incoming metrics from `status-react` + ama := appmetrics.NewAPI(appmetricsDB.NewDB(s.alice.database)) + + // Generate and store some metrics to Alice + ams := appmetricsDB.GenerateMetrics(10) + err := ama.SaveAppMetrics(context.Background(), ams) + s.Require().NoError(err) + + // Check that we have what we stored + amsdb, err := ama.GetAppMetrics(context.Background(), 100, 0) + s.Require().NoError(err) + s.Require().Len(amsdb.AppMetrics, 10) + + // Wait for messages to arrive at bob + _, err = WaitOnMessengerResponse( + s.bob, + func(r *MessengerResponse) bool { return len(r.AnonymousMetrics) > 0 }, + "no anonymous metrics received", + ) + s.Require().NoError(err) + + // Get app metrics from postgres DB + bobMetrics, err := s.bob.anonMetricsServer.GetAppMetrics(100, 0) + s.Require().NoError(err) + s.Require().Len(bobMetrics, 5) + + // Check the values of received and stored metrics against the broadcast metrics + for i, bobMetric := range bobMetrics { + s.Require().True(bobMetric.CreatedAt.Equal(amsdb.AppMetrics[i].CreatedAt), "created_at values are equal") + s.Require().Exactly(bobMetric.SessionID, amsdb.AppMetrics[i].SessionID, "session_id matched exactly") + s.Require().Exactly(bobMetric.Value, amsdb.AppMetrics[i].Value, "value matches exactly") + s.Require().Exactly(bobMetric.Event, amsdb.AppMetrics[i].Event, "event matches exactly") + s.Require().Exactly(bobMetric.OS, amsdb.AppMetrics[i].OS, "operating system matches exactly") + s.Require().Exactly(bobMetric.AppVersion, amsdb.AppMetrics[i].AppVersion, "app version matches exactly") + } +} + +// TestActivationIsOff tests if using the incorrect activation phrase for the anon metric client / server deactivates +// the client / server. This test can be removed when / if the anon metrics functionality is reintroduced / re-approved. +func (s *MessengerAnonMetricsSuite) TestActivationIsOff() { + var err error + + // Check the set up messengers are in the expected state with the correct activation phrases + s.NotNil(s.alice.anonMetricsClient) + s.NotNil(s.bob.anonMetricsServer) + + // Generate Alice Messenger as the client with an incorrect phrase + amcc := &anonmetrics.ClientConfig{ + ShouldSend: true, + SendAddress: &s.bobKey.PublicKey, + Active: "the wrong client phrase", + } + s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)}) + s.NoError(err) + _, err = s.alice.Start() + s.Require().NoError(err) + + s.Nil(s.alice.anonMetricsClient) + + // Generate Alice Messenger as the client with an no activation phrase + amcc = &anonmetrics.ClientConfig{ + ShouldSend: true, + SendAddress: &s.bobKey.PublicKey, + } + s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)}) + s.NoError(err) + _, err = s.alice.Start() + s.Require().NoError(err) + + s.Nil(s.alice.anonMetricsClient) + + // Generate Bob Messenger as the Server with an incorrect phrase + amsc := &anonmetrics.ServerConfig{ + Enabled: true, + PostgresURI: postgres.DefaultTestURI, + Active: "the wrong server phrase", + } + s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)}) + s.Require().NoError(err) + + s.Nil(s.bob.anonMetricsServer) + + // Generate Bob Messenger as the Server with no activation phrase + amsc = &anonmetrics.ServerConfig{ + Enabled: true, + PostgresURI: postgres.DefaultTestURI, + } + s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)}) + s.Require().NoError(err) + + s.Nil(s.bob.anonMetricsServer) +} diff --git a/protocol/messenger.go b/protocol/messenger.go index df2a964e8..9efcb65de 100644 --- a/protocol/messenger.go +++ b/protocol/messenger.go @@ -3418,6 +3418,10 @@ func (m *Messenger) MessageByChatID(chatID, cursor string, limit int) ([]*common return nil, "", err } + if chat == nil { + return nil, "", ErrChatNotFound + } + if chat.Timeline() { var chatIDs = []string{"@" + contactIDFromPublicKey(&m.identity.PublicKey)} m.allContacts.Range(func(contactID string, contact *Contact) (shouldContinue bool) { diff --git a/protocol/messenger_backup.go b/protocol/messenger_backup.go index 96c723786..1553c5b52 100644 --- a/protocol/messenger_backup.go +++ b/protocol/messenger_backup.go @@ -54,6 +54,7 @@ func (m *Messenger) startBackupLoop() { lastBackup, err := m.lastBackup() if err != nil { m.logger.Error("failed to fetch last backup time") + continue } now := time.Now().Unix() @@ -63,7 +64,9 @@ func (m *Messenger) startBackupLoop() { } m.logger.Debug("backing up data") - _, err = m.BackupData(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + _, err = m.BackupData(ctx) if err != nil { m.logger.Error("failed to backup data", zap.Error(err)) } diff --git a/protocol/messenger_contacts.go b/protocol/messenger_contacts.go index 95f8bf69b..50fbdfa2c 100644 --- a/protocol/messenger_contacts.go +++ b/protocol/messenger_contacts.go @@ -292,7 +292,7 @@ func (m *Messenger) BlockContact(contact *Contact) ([]*Chat, error) { func (m *Messenger) UnblockContact(contactID string) error { contact, ok := m.allContacts.Load(contactID) - if !ok || !contact.Added { + if !ok || !contact.Blocked { return nil } diff --git a/protocol/messenger_installations_test.go b/protocol/messenger_installations_test.go index 969652d25..bfaa3e51d 100644 --- a/protocol/messenger_installations_test.go +++ b/protocol/messenger_installations_test.go @@ -148,7 +148,7 @@ func (s *MessengerInstallationSuite) TestSyncInstallation() { contact.LocalNickname = "Test Nickname" _, err = s.m.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(contact.ID)}) s.Require().NoError(err) - _, err = s.m.SetContactLocalNickname(&requests.SetContactLocalNickname{ID: []byte(contact.ID), Nickname: contact.LocalNickname}) + _, err = s.m.SetContactLocalNickname(&requests.SetContactLocalNickname{ID: types.Hex2Bytes(contact.ID), Nickname: contact.LocalNickname}) s.Require().NoError(err) // add chat diff --git a/protocol/messenger_mailserver.go b/protocol/messenger_mailserver.go index d61be8a42..e65e24271 100644 --- a/protocol/messenger_mailserver.go +++ b/protocol/messenger_mailserver.go @@ -21,6 +21,7 @@ import ( // tolerance is how many seconds of potentially out-of-order messages we want to fetch var tolerance uint32 = 60 var mailserverRequestTimeout = 45 * time.Second +var oneMonthInSeconds uint32 = 31 * 24 * 60 * 6 func (m *Messenger) shouldSync() (bool, error) { if m.mailserver == nil || !m.online() { @@ -83,6 +84,7 @@ func (m *Messenger) scheduleSyncFilter(filter *transport.Filter) { } } + func (m *Messenger) scheduleSyncFilters(filters []*transport.Filter) (bool, error) { shouldSync, err := m.shouldSync() if err != nil { @@ -170,6 +172,23 @@ func (m *Messenger) syncChat(chatID string) (*MessengerResponse, error) { return m.syncFilters(filters) } +func (m *Messenger) syncBackup() error { + + filter := m.transport.PersonalTopicFilter() + if filter == nil { + return errors.New("personal topic filter not loaded") + } + + to := m.calculateMailserverTo() + from := uint32(m.getTimesource().GetCurrentTime()/1000) - oneMonthInSeconds + batch := MailserverBatch{From: from, To: to, Topics: []types.TopicType{filter.Topic}} + err := m.processMailserverBatch(batch) + if err != nil { + return err + } + return m.settings.SetBackupFetched(true) +} + func (m *Messenger) defaultSyncPeriodFromNow() (uint32, error) { defaultSyncPeriod, err := m.settings.GetDefaultSyncPeriod() if err != nil { @@ -201,6 +220,20 @@ func (m *Messenger) RequestAllHistoricMessages() (*MessengerResponse, error) { return nil, nil } + backupFetched, err := m.settings.BackupFetched() + if err != nil { + return nil, err + } + + if !backupFetched { + m.logger.Info("fetching backup") + err := m.syncBackup() + if err != nil { + return nil, err + } + m.logger.Info("backup fetched") + } + return m.syncFilters(m.transport.Filters()) } diff --git a/protocol/messenger_test.go b/protocol/messenger_test.go index 071a3b764..cffb991a6 100644 --- a/protocol/messenger_test.go +++ b/protocol/messenger_test.go @@ -2365,7 +2365,7 @@ func (s *MessengerSuite) TestResendExpiredEmojis() { //make sure it was resent and SendCount incremented rawMessage, err = s.m.persistence.RawMessageByID(emojiID) s.NoError(err) - s.Equal(2, rawMessage.SendCount) + s.True(rawMessage.SendCount >= 2) } type testTimeSource struct{} diff --git a/protocol/migrations/migrations.go b/protocol/migrations/migrations.go index 45d4432c3..3eaec6d3b 100644 --- a/protocol/migrations/migrations.go +++ b/protocol/migrations/migrations.go @@ -1053,7 +1053,7 @@ func _1634896007_add_last_updated_locally_and_removedUpSql() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "1634896007_add_last_updated_locally_and_removed.up.sql", size: 131, mode: os.FileMode(0644), modTime: time.Unix(1635867803, 0)} + info := bindataFileInfo{name: "1634896007_add_last_updated_locally_and_removed.up.sql", size: 131, mode: os.FileMode(0644), modTime: time.Unix(1636971774, 0)} a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2e, 0xa8, 0x34, 0xe2, 0xc0, 0x62, 0xc8, 0xd6, 0x5a, 0x87, 0xe3, 0x70, 0xe1, 0xc4, 0x16, 0x9c, 0x60, 0x2e, 0x98, 0xf0, 0x91, 0x84, 0xbe, 0xe0, 0xdf, 0x3e, 0x4d, 0x24, 0xc4, 0x6c, 0x40, 0x17}} return a, nil } diff --git a/protocol/transport/filters_manager.go b/protocol/transport/filters_manager.go index d7351e4df..923cbedcb 100644 --- a/protocol/transport/filters_manager.go +++ b/protocol/transport/filters_manager.go @@ -473,6 +473,12 @@ func (f *FiltersManager) LoadDiscovery() ([]*Filter, error) { return []*Filter{personalDiscoveryChat}, nil } +func (f *FiltersManager) PersonalTopicFilter() *Filter { + personalDiscoveryTopic := PersonalDiscoveryTopic(&f.privateKey.PublicKey) + + return f.filters[personalDiscoveryTopic] +} + // LoadPublic adds a filter for a public chat. func (f *FiltersManager) LoadPublic(chatID string) (*Filter, error) { f.mutex.Lock() diff --git a/protocol/transport/transport.go b/protocol/transport/transport.go index 74bb09495..d8bc13a1d 100644 --- a/protocol/transport/transport.go +++ b/protocol/transport/transport.go @@ -331,6 +331,10 @@ func (t *Transport) SendPrivateOnPersonalTopic(ctx context.Context, newMessage * return t.api.Post(ctx, *newMessage) } +func (t *Transport) PersonalTopicFilter() *Filter { + return t.filters.PersonalTopicFilter() +} + func (t *Transport) LoadKeyFilters(key *ecdsa.PrivateKey) (*Filter, error) { return t.filters.LoadEphemeral(&key.PublicKey, key, true) }