Set last backup

appdatabase/migrations/bindata.go

@@ -50,7 +50,7 @@
 // 1630485153_networks.up.sql (394B)
 // 1632262444_profile_pictures_show_to.up.sql (81B)
 // 1635942153_add_telemetry_server_url_to_settings.up.sql (128B)
-// 1635942154_add_backup_setting.up.sql (186B)
+// 1635942154_add_backup_setting.up.sql (287B)
 // doc.go (74B)
 
 package migrations
@@ -1120,7 +1120,7 @@ func _1635942153_add_telemetry_server_url_to_settingsUpSql() (*asset, error) {
 	return a, nil
 }
 
-var __1635942154_add_backup_settingUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xce\x31\x0a\xc2\x40\x10\x46\xe1\x3e\xa7\xf8\x8f\xa0\xf5\x62\x31\x71\x47\x10\xc6\x59\x89\x33\x75\xd8\xe8\x22\x62\x08\xc2\xae\xf7\xb7\xb0\x11\x2d\x3c\xc0\xfb\x78\x24\xc6\x03\x8c\x7a\x61\xd4\xd2\xda\x6d\xb9\x56\x50\x8c\xd8\x26\xf1\x83\x62\xca\xe7\xfb\xf3\x31\x96\x25\x4f\x73\xb9\xa0\x4f\x49\x98\x14\x9a\x0c\xea\x22\x88\xbc\x23\x17\x83\x0d\xce\xa1\xfb\xa7\xcd\xb9\xb6\xf1\x4d\x62\xaf\xf6\xcb\xac\x42\xe7\xc7\x48\xf6\x91\x9f\xd8\xbe\x2f\x36\x58\x87\xee\x15\x00\x00\xff\xff\x6b\x1d\x44\xa1\xba\x00\x00\x00")
+var __1635942154_add_backup_settingUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xcf\xcb\x0a\xc2\x30\x10\x85\xe1\x7d\x9f\xe2\x3c\x82\xae\x4b\x17\x53\x33\x05\x61\x4c\xa4\x9d\xac\x4b\x5b\xe3\x05\x4b\x11\x12\xdf\x5f\xc4\x0b\x22\x62\xf7\xc3\x37\xff\x21\x51\xae\xa1\x54\x0a\x23\x86\x94\x4e\xd3\x21\x82\x8c\xc1\xca\x89\xdf\x58\xf4\xdd\x70\xbe\x5e\xda\x30\x75\xfd\x18\x76\x28\x9d\x13\x26\x0b\xc3\x15\x79\x51\x68\xed\x39\xcf\xe6\x90\xb1\x8b\xa9\x7d\x48\x58\x5b\x85\x75\x0a\xeb\x45\xde\xcc\x62\xde\x78\x86\xec\x43\x1a\x8e\x3f\x42\x2a\x92\x86\xf3\xcc\x6f\x0d\xe9\x07\xd0\xb0\x7e\x4f\x28\xb0\xfc\x7b\xf7\xfa\x50\xdc\xab\x6e\x01\x00\x00\xff\xff\x3f\xf3\xd1\x35\x1f\x01\x00\x00")
 
 func _1635942154_add_backup_settingUpSqlBytes() ([]byte, error) {
 	return bindataRead(
@@ -1135,8 +1135,8 @@ func _1635942154_add_backup_settingUpSql() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "1635942154_add_backup_setting.up.sql", size: 186, mode: os.FileMode(0644), modTime: time.Unix(1636971544, 0)}
+	info := bindataFileInfo{name: "1635942154_add_backup_setting.up.sql", size: 287, mode: os.FileMode(0644), modTime: time.Unix(1636971794, 0)}
-	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x42, 0xc3, 0x25, 0xc6, 0xd2, 0x86, 0x53, 0xa3, 0x52, 0x71, 0x1b, 0x54, 0xc9, 0x0, 0xf0, 0xe6, 0x46, 0x3c, 0x64, 0xa0, 0x2b, 0xfb, 0xb2, 0x2e, 0xc6, 0x67, 0x67, 0x6c, 0x6c, 0x40, 0xee, 0x2f}}
+	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xb7, 0xe7, 0xfb, 0x70, 0x80, 0x5, 0xb4, 0x7b, 0x67, 0x8, 0x6e, 0x5f, 0x45, 0x17, 0xd9, 0x5f, 0x18, 0x66, 0x2f, 0x8a, 0x4f, 0xd4, 0x15, 0xe5, 0x2b, 0xbb, 0x25, 0x7a, 0x30, 0xad, 0x4c, 0x1a}}
 	return a, nil
 }
 

appdatabase/migrations/sql/1635942154_add_backup_setting.up.sql

@@ -1,3 +1,5 @@
-ALTER TABLE settings ADD COLUMN backup_enabled BOOLEAN NOT NULL DEFAULT TRUE;
+ALTER TABLE settings ADD COLUMN backup_enabled BOOLEAN DEFAULT TRUE;
 ALTER TABLE settings ADD COLUMN last_backup INT NOT NULL DEFAULT 0;
+ALTER TABLE settings ADD COLUMN backup_fetched BOOLEAN DEFAULT FALSE;
 UPDATE settings SET backup_enabled = 1;
+UPDATE settings SET backup_fetched = 0;

cmd/populate-db/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/status-im/status-go/protocol/common"
 	"github.com/status-im/status-go/protocol/identity/alias"
 	"github.com/status-im/status-go/protocol/protobuf"
+	"github.com/status-im/status-go/protocol/requests"
 	wakuextn "github.com/status-im/status-go/services/wakuext"
 )
 
@@ -165,7 +166,7 @@ func main() {
 		}
 
 		keyString := common.PubkeyToHex(&key.PublicKey)
-		_, err = wakuext.AddContact(context.Background(), keyString)
+		_, err = wakuext.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(keyString)})
 		if err != nil {
 			logger.Error("failed Add contact", "err", err)
 			return
@@ -185,7 +186,7 @@ func main() {
 			return
 		}
 
-		_, err = wakuext.AddContact(context.Background(), contact.ID)
+		_, err = wakuext.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(contact.ID)})
 		if err != nil {
 			return
 		}

multiaccounts/accounts/database.go

@@ -765,6 +765,20 @@ func (db *Database) SetLastBackup(time uint64) error {
 	return err
 }
 
+func (db *Database) SetBackupFetched(fetched bool) error {
+	_, err := db.db.Exec("UPDATE settings SET backup_fetched = ?", fetched)
+	return err
+}
+
+func (db *Database) BackupFetched() (bool, error) {
+	var result bool
+	err := db.db.QueryRow("SELECT backup_fetched FROM settings WHERE synthetic_id = 'id'").Scan(&result)
+	if err == sql.ErrNoRows {
+		return true, nil
+	}
+	return result, err
+}
+
 func (db *Database) ENSName() (string, error) {
 	var result sql.NullString
 	err := db.db.QueryRow("SELECT preferred_name FROM settings WHERE synthetic_id = 'id'").Scan(&result)

protocol/anon_metrics_test.go

@@ -0,0 +1,209 @@
+// In order to run these tests, you must run a PostgreSQL database.
+//
+// Using Docker:
+//   docker run -e POSTGRES_HOST_AUTH_METHOD=trust -d -p 5432:5432 postgres:9.6-alpine
+//
+
+package protocol
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"go.uber.org/zap"
+
+	bindata "github.com/status-im/migrate/v4/source/go_bindata"
+
+	appmetricsDB "github.com/status-im/status-go/appmetrics"
+	gethbridge "github.com/status-im/status-go/eth-node/bridge/geth"
+	"github.com/status-im/status-go/eth-node/crypto"
+	"github.com/status-im/status-go/eth-node/types"
+	"github.com/status-im/status-go/postgres"
+	"github.com/status-im/status-go/protocol/anonmetrics"
+	"github.com/status-im/status-go/protocol/anonmetrics/migrations"
+	"github.com/status-im/status-go/protocol/tt"
+	"github.com/status-im/status-go/services/appmetrics"
+	"github.com/status-im/status-go/waku"
+)
+
+func TestMessengerAnonMetricsSuite(t *testing.T) {
+	suite.Run(t, new(MessengerAnonMetricsSuite))
+}
+
+type MessengerAnonMetricsSuite struct {
+	suite.Suite
+	alice *Messenger // client instance of Messenger
+	bob   *Messenger // server instance of Messenger
+
+	aliceKey *ecdsa.PrivateKey // private key for the alice instance of Messenger
+	bobKey   *ecdsa.PrivateKey // private key for the bob instance of Messenger
+
+	// If one wants to send messages between different instances of Messenger,
+	// a single Waku service should be shared.
+	shh    types.Waku
+	logger *zap.Logger
+}
+
+func (s *MessengerAnonMetricsSuite) SetupSuite() {
+	// ResetDefaultTestPostgresDB Required to completely reset the Postgres DB
+	err := postgres.ResetDefaultTestPostgresDB()
+	s.NoError(err)
+}
+
+func (s *MessengerAnonMetricsSuite) SetupTest() {
+	var err error
+
+	s.logger = tt.MustCreateTestLogger()
+
+	// Setup Waku things
+	config := waku.DefaultConfig
+	config.MinimumAcceptedPoW = 0
+	shh := waku.New(&config, s.logger)
+	s.shh = gethbridge.NewGethWakuWrapper(shh)
+	s.Require().NoError(shh.Start())
+
+	// Generate private keys for Alice and Bob
+	s.aliceKey, err = crypto.GenerateKey()
+	s.Require().NoError(err)
+
+	s.bobKey, err = crypto.GenerateKey()
+	s.Require().NoError(err)
+
+	// Generate Alice Messenger as the client
+	amcc := &anonmetrics.ClientConfig{
+		ShouldSend:  true,
+		SendAddress: &s.bobKey.PublicKey,
+		Active:      anonmetrics.ActiveClientPhrase,
+	}
+	s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)})
+	s.Require().NoError(err)
+	_, err = s.alice.Start()
+	s.Require().NoError(err)
+
+	// Generate Bob Messenger as the Server
+	amsc := &anonmetrics.ServerConfig{
+		Enabled:     true,
+		PostgresURI: postgres.DefaultTestURI,
+		Active:      anonmetrics.ActiveServerPhrase,
+	}
+	s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)})
+	s.Require().NoError(err)
+
+	_, err = s.bob.Start()
+	s.Require().NoError(err)
+}
+
+func (s *MessengerAnonMetricsSuite) TearDownTest() {
+	// Down migrate the DB
+	if s.bob.anonMetricsServer != nil {
+		postgresMigration := bindata.Resource(migrations.AssetNames(), migrations.Asset)
+		m, err := anonmetrics.MakeMigration(s.bob.anonMetricsServer.PostgresDB, postgresMigration)
+		s.NoError(err)
+
+		err = m.Down()
+		s.NoError(err)
+	}
+
+	// Shutdown messengers
+	s.NoError(s.alice.Shutdown())
+	s.alice = nil
+	s.NoError(s.bob.Shutdown())
+	s.bob = nil
+	_ = s.logger.Sync()
+}
+
+func (s *MessengerAnonMetricsSuite) TestReceiveAnonMetric() {
+	// Create the appmetrics API to simulate incoming metrics from `status-react`
+	ama := appmetrics.NewAPI(appmetricsDB.NewDB(s.alice.database))
+
+	// Generate and store some metrics to Alice
+	ams := appmetricsDB.GenerateMetrics(10)
+	err := ama.SaveAppMetrics(context.Background(), ams)
+	s.Require().NoError(err)
+
+	// Check that we have what we stored
+	amsdb, err := ama.GetAppMetrics(context.Background(), 100, 0)
+	s.Require().NoError(err)
+	s.Require().Len(amsdb.AppMetrics, 10)
+
+	// Wait for messages to arrive at bob
+	_, err = WaitOnMessengerResponse(
+		s.bob,
+		func(r *MessengerResponse) bool { return len(r.AnonymousMetrics) > 0 },
+		"no anonymous metrics received",
+	)
+	s.Require().NoError(err)
+
+	// Get app metrics from postgres DB
+	bobMetrics, err := s.bob.anonMetricsServer.GetAppMetrics(100, 0)
+	s.Require().NoError(err)
+	s.Require().Len(bobMetrics, 5)
+
+	// Check the values of received and stored metrics against the broadcast metrics
+	for i, bobMetric := range bobMetrics {
+		s.Require().True(bobMetric.CreatedAt.Equal(amsdb.AppMetrics[i].CreatedAt), "created_at values are equal")
+		s.Require().Exactly(bobMetric.SessionID, amsdb.AppMetrics[i].SessionID, "session_id matched exactly")
+		s.Require().Exactly(bobMetric.Value, amsdb.AppMetrics[i].Value, "value matches exactly")
+		s.Require().Exactly(bobMetric.Event, amsdb.AppMetrics[i].Event, "event matches exactly")
+		s.Require().Exactly(bobMetric.OS, amsdb.AppMetrics[i].OS, "operating system matches exactly")
+		s.Require().Exactly(bobMetric.AppVersion, amsdb.AppMetrics[i].AppVersion, "app version matches exactly")
+	}
+}
+
+// TestActivationIsOff tests if using the incorrect activation phrase for the anon metric client / server deactivates
+// the client / server. This test can be removed when / if the anon metrics functionality is reintroduced / re-approved.
+func (s *MessengerAnonMetricsSuite) TestActivationIsOff() {
+	var err error
+
+	// Check the set up messengers are in the expected state with the correct activation phrases
+	s.NotNil(s.alice.anonMetricsClient)
+	s.NotNil(s.bob.anonMetricsServer)
+
+	// Generate Alice Messenger as the client with an incorrect phrase
+	amcc := &anonmetrics.ClientConfig{
+		ShouldSend:  true,
+		SendAddress: &s.bobKey.PublicKey,
+		Active:      "the wrong client phrase",
+	}
+	s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)})
+	s.NoError(err)
+	_, err = s.alice.Start()
+	s.Require().NoError(err)
+
+	s.Nil(s.alice.anonMetricsClient)
+
+	// Generate Alice Messenger as the client with an no activation phrase
+	amcc = &anonmetrics.ClientConfig{
+		ShouldSend:  true,
+		SendAddress: &s.bobKey.PublicKey,
+	}
+	s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)})
+	s.NoError(err)
+	_, err = s.alice.Start()
+	s.Require().NoError(err)
+
+	s.Nil(s.alice.anonMetricsClient)
+
+	// Generate Bob Messenger as the Server with an incorrect phrase
+	amsc := &anonmetrics.ServerConfig{
+		Enabled:     true,
+		PostgresURI: postgres.DefaultTestURI,
+		Active:      "the wrong server phrase",
+	}
+	s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)})
+	s.Require().NoError(err)
+
+	s.Nil(s.bob.anonMetricsServer)
+
+	// Generate Bob Messenger as the Server with no activation phrase
+	amsc = &anonmetrics.ServerConfig{
+		Enabled:     true,
+		PostgresURI: postgres.DefaultTestURI,
+	}
+	s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)})
+	s.Require().NoError(err)
+
+	s.Nil(s.bob.anonMetricsServer)
+}

protocol/messenger.go

@@ -3418,6 +3418,10 @@ func (m *Messenger) MessageByChatID(chatID, cursor string, limit int) ([]*common
 		return nil, "", err
 	}
 
+	if chat == nil {
+		return nil, "", ErrChatNotFound
+	}
+
 	if chat.Timeline() {
 		var chatIDs = []string{"@" + contactIDFromPublicKey(&m.identity.PublicKey)}
 		m.allContacts.Range(func(contactID string, contact *Contact) (shouldContinue bool) {

protocol/messenger_backup.go

@@ -54,6 +54,7 @@ func (m *Messenger) startBackupLoop() {
 				lastBackup, err := m.lastBackup()
 				if err != nil {
 					m.logger.Error("failed to fetch last backup time")
+					continue
 				}
 
 				now := time.Now().Unix()
@@ -63,7 +64,9 @@ func (m *Messenger) startBackupLoop() {
 				}
 				m.logger.Debug("backing up data")
 
-				_, err = m.BackupData(context.Background())
+				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+				defer cancel()
+				_, err = m.BackupData(ctx)
 				if err != nil {
 					m.logger.Error("failed to backup data", zap.Error(err))
 				}

protocol/messenger_contacts.go

@@ -292,7 +292,7 @@ func (m *Messenger) BlockContact(contact *Contact) ([]*Chat, error) {
 
 func (m *Messenger) UnblockContact(contactID string) error {
 	contact, ok := m.allContacts.Load(contactID)
-	if !ok || !contact.Added {
+	if !ok || !contact.Blocked {
 		return nil
 	}
 

protocol/messenger_installations_test.go

@@ -148,7 +148,7 @@ func (s *MessengerInstallationSuite) TestSyncInstallation() {
 	contact.LocalNickname = "Test Nickname"
 	_, err = s.m.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(contact.ID)})
 	s.Require().NoError(err)
-	_, err = s.m.SetContactLocalNickname(&requests.SetContactLocalNickname{ID: []byte(contact.ID), Nickname: contact.LocalNickname})
+	_, err = s.m.SetContactLocalNickname(&requests.SetContactLocalNickname{ID: types.Hex2Bytes(contact.ID), Nickname: contact.LocalNickname})
 	s.Require().NoError(err)
 
 	// add chat

protocol/messenger_mailserver.go

@@ -21,6 +21,7 @@ import (
 // tolerance is how many seconds of potentially out-of-order messages we want to fetch
 var tolerance uint32 = 60
 var mailserverRequestTimeout = 45 * time.Second
+var oneMonthInSeconds uint32 = 31 * 24 * 60 * 6
 
 func (m *Messenger) shouldSync() (bool, error) {
 	if m.mailserver == nil || !m.online() {
@@ -83,6 +84,7 @@ func (m *Messenger) scheduleSyncFilter(filter *transport.Filter) {
 	}
 
 }
+
 func (m *Messenger) scheduleSyncFilters(filters []*transport.Filter) (bool, error) {
 	shouldSync, err := m.shouldSync()
 	if err != nil {
@@ -170,6 +172,23 @@ func (m *Messenger) syncChat(chatID string) (*MessengerResponse, error) {
 	return m.syncFilters(filters)
 }
 
+func (m *Messenger) syncBackup() error {
+
+	filter := m.transport.PersonalTopicFilter()
+	if filter == nil {
+		return errors.New("personal topic filter not loaded")
+	}
+
+	to := m.calculateMailserverTo()
+	from := uint32(m.getTimesource().GetCurrentTime()/1000) - oneMonthInSeconds
+	batch := MailserverBatch{From: from, To: to, Topics: []types.TopicType{filter.Topic}}
+	err := m.processMailserverBatch(batch)
+	if err != nil {
+		return err
+	}
+	return m.settings.SetBackupFetched(true)
+}
+
 func (m *Messenger) defaultSyncPeriodFromNow() (uint32, error) {
 	defaultSyncPeriod, err := m.settings.GetDefaultSyncPeriod()
 	if err != nil {
@@ -201,6 +220,20 @@ func (m *Messenger) RequestAllHistoricMessages() (*MessengerResponse, error) {
 		return nil, nil
 	}
 
+	backupFetched, err := m.settings.BackupFetched()
+	if err != nil {
+		return nil, err
+	}
+
+	if !backupFetched {
+		m.logger.Info("fetching backup")
+		err := m.syncBackup()
+		if err != nil {
+			return nil, err
+		}
+		m.logger.Info("backup fetched")
+	}
+
 	return m.syncFilters(m.transport.Filters())
 }
 

protocol/messenger_test.go

@@ -2365,7 +2365,7 @@ func (s *MessengerSuite) TestResendExpiredEmojis() {
 	//make sure it was resent and SendCount incremented
 	rawMessage, err = s.m.persistence.RawMessageByID(emojiID)
 	s.NoError(err)
-	s.Equal(2, rawMessage.SendCount)
+	s.True(rawMessage.SendCount >= 2)
 }
 
 type testTimeSource struct{}

protocol/migrations/migrations.go

@@ -1053,7 +1053,7 @@ func _1634896007_add_last_updated_locally_and_removedUpSql() (*asset, error) {
 		return nil, err
 	}
 
-	info := bindataFileInfo{name: "1634896007_add_last_updated_locally_and_removed.up.sql", size: 131, mode: os.FileMode(0644), modTime: time.Unix(1635867803, 0)}
+	info := bindataFileInfo{name: "1634896007_add_last_updated_locally_and_removed.up.sql", size: 131, mode: os.FileMode(0644), modTime: time.Unix(1636971774, 0)}
 	a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2e, 0xa8, 0x34, 0xe2, 0xc0, 0x62, 0xc8, 0xd6, 0x5a, 0x87, 0xe3, 0x70, 0xe1, 0xc4, 0x16, 0x9c, 0x60, 0x2e, 0x98, 0xf0, 0x91, 0x84, 0xbe, 0xe0, 0xdf, 0x3e, 0x4d, 0x24, 0xc4, 0x6c, 0x40, 0x17}}
 	return a, nil
 }

protocol/transport/filters_manager.go

@@ -473,6 +473,12 @@ func (f *FiltersManager) LoadDiscovery() ([]*Filter, error) {
 	return []*Filter{personalDiscoveryChat}, nil
 }
 
+func (f *FiltersManager) PersonalTopicFilter() *Filter {
+	personalDiscoveryTopic := PersonalDiscoveryTopic(&f.privateKey.PublicKey)
+
+	return f.filters[personalDiscoveryTopic]
+}
+
 // LoadPublic adds a filter for a public chat.
 func (f *FiltersManager) LoadPublic(chatID string) (*Filter, error) {
 	f.mutex.Lock()

protocol/transport/transport.go

@@ -331,6 +331,10 @@ func (t *Transport) SendPrivateOnPersonalTopic(ctx context.Context, newMessage *
 	return t.api.Post(ctx, *newMessage)
 }
 
+func (t *Transport) PersonalTopicFilter() *Filter {
+	return t.filters.PersonalTopicFilter()
+}
+
 func (t *Transport) LoadKeyFilters(key *ecdsa.PrivateKey) (*Filter, error) {
 	return t.filters.LoadEphemeral(&key.PublicKey, key, true)
 }