status-im
/
status-go
mirror of https://github.com/status-im/status-go.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Set last backup

This commit is contained in:
Andrea Maria Piana 2021-10-27 11:59:43 +01:00
parent be1a261d9b
commit ca6246d5f2
14 changed files with 288 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
appdatabase/migrations/bindata.go
View File

@ -50,7 +50,7 @@
// 1630485153_networks.up.sql (394B)
// 1632262444_profile_pictures_show_to.up.sql (81B)
// 1635942153_add_telemetry_server_url_to_settings.up.sql (128B)
// 1635942154_add_backup_setting.up.sql (186B)
// 1635942154_add_backup_setting.up.sql (287B)
// doc.go (74B)
package migrations
@ -1120,7 +1120,7 @@ func _1635942153_add_telemetry_server_url_to_settingsUpSql() (*asset, error) {
return a, nil
}
var __1635942154_add_backup_settingUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xce\x31\x0a\xc2\x40\x10\x46\xe1\x3e\xa7\xf8\x8f\xa0\xf5\x62\x31\x71\x47\x10\xc6\x59\x89\x33\x75\xd8\xe8\x22\x62\x08\xc2\xae\xf7\xb7\xb0\x11\x2d\x3c\xc0\xfb\x78\x24\xc6\x03\x8c\x7a\x61\xd4\xd2\xda\x6d\xb9\x56\x50\x8c\xd8\x26\xf1\x83\x62\xca\xe7\xfb\xf3\x31\x96\x25\x4f\x73\xb9\xa0\x4f\x49\x98\x14\x9a\x0c\xea\x22\x88\xbc\x23\x17\x83\x0d\xce\xa1\xfb\xa7\xcd\xb9\xb6\xf1\x4d\x62\xaf\xf6\xcb\xac\x42\xe7\xc7\x48\xf6\x91\x9f\xd8\xbe\x2f\x36\x58\x87\xee\x15\x00\x00\xff\xff\x6b\x1d\x44\xa1\xba\x00\x00\x00")
var __1635942154_add_backup_settingUpSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xcf\xcb\x0a\xc2\x30\x10\x85\xe1\x7d\x9f\xe2\x3c\x82\xae\x4b\x17\x53\x33\x05\x61\x4c\xa4\x9d\xac\x4b\x5b\xe3\x05\x4b\x11\x12\xdf\x5f\xc4\x0b\x22\x62\xf7\xc3\x37\xff\x21\x51\xae\xa1\x54\x0a\x23\x86\x94\x4e\xd3\x21\x82\x8c\xc1\xca\x89\xdf\x58\xf4\xdd\x70\xbe\x5e\xda\x30\x75\xfd\x18\x76\x28\x9d\x13\x26\x0b\xc3\x15\x79\x51\x68\xed\x39\xcf\xe6\x90\xb1\x8b\xa9\x7d\x48\x58\x5b\x85\x75\x0a\xeb\x45\xde\xcc\x62\xde\x78\x86\xec\x43\x1a\x8e\x3f\x42\x2a\x92\x86\xf3\xcc\x6f\x0d\xe9\x07\xd0\xb0\x7e\x4f\x28\xb0\xfc\x7b\xf7\xfa\x50\xdc\xab\x6e\x01\x00\x00\xff\xff\x3f\xf3\xd1\x35\x1f\x01\x00\x00")
func _1635942154_add_backup_settingUpSqlBytes() ([]byte, error) {
return bindataRead(
@ -1135,8 +1135,8 @@ func _1635942154_add_backup_settingUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "1635942154_add_backup_setting.up.sql", size: 186, mode: os.FileMode(0644), modTime: time.Unix(1636971544, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x42, 0xc3, 0x25, 0xc6, 0xd2, 0x86, 0x53, 0xa3, 0x52, 0x71, 0x1b, 0x54, 0xc9, 0x0, 0xf0, 0xe6, 0x46, 0x3c, 0x64, 0xa0, 0x2b, 0xfb, 0xb2, 0x2e, 0xc6, 0x67, 0x67, 0x6c, 0x6c, 0x40, 0xee, 0x2f}}
info := bindataFileInfo{name: "1635942154_add_backup_setting.up.sql", size: 287, mode: os.FileMode(0644), modTime: time.Unix(1636971794, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xb7, 0xe7, 0xfb, 0x70, 0x80, 0x5, 0xb4, 0x7b, 0x67, 0x8, 0x6e, 0x5f, 0x45, 0x17, 0xd9, 0x5f, 0x18, 0x66, 0x2f, 0x8a, 0x4f, 0xd4, 0x15, 0xe5, 0x2b, 0xbb, 0x25, 0x7a, 0x30, 0xad, 0x4c, 0x1a}}
return a, nil
}

4
appdatabase/migrations/sql/1635942154_add_backup_setting.up.sql
View File

@ -1,3 +1,5 @@
ALTER TABLE settings ADD COLUMN backup_enabled BOOLEAN NOT NULL DEFAULT TRUE;
ALTER TABLE settings ADD COLUMN backup_enabled BOOLEAN DEFAULT TRUE;
ALTER TABLE settings ADD COLUMN last_backup INT NOT NULL DEFAULT 0;
ALTER TABLE settings ADD COLUMN backup_fetched BOOLEAN DEFAULT FALSE;
UPDATE settings SET backup_enabled = 1;
UPDATE settings SET backup_fetched = 0;

5
cmd/populate-db/main.go
View File

@ -34,6 +34,7 @@ import (
"github.com/status-im/status-go/protocol/common"
"github.com/status-im/status-go/protocol/identity/alias"
"github.com/status-im/status-go/protocol/protobuf"
"github.com/status-im/status-go/protocol/requests"
wakuextn "github.com/status-im/status-go/services/wakuext"
)
@ -165,7 +166,7 @@ func main() {
}
keyString := common.PubkeyToHex(&key.PublicKey)
_, err = wakuext.AddContact(context.Background(), keyString)
_, err = wakuext.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(keyString)})
if err != nil {
logger.Error("failed Add contact", "err", err)
return
@ -185,7 +186,7 @@ func main() {
return
}
_, err = wakuext.AddContact(context.Background(), contact.ID)
_, err = wakuext.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(contact.ID)})
if err != nil {
return
}

14
multiaccounts/accounts/database.go
View File

@ -765,6 +765,20 @@ func (db *Database) SetLastBackup(time uint64) error {
return err
}
func (db *Database) SetBackupFetched(fetched bool) error {
_, err := db.db.Exec("UPDATE settings SET backup_fetched = ?", fetched)
return err
}
func (db *Database) BackupFetched() (bool, error) {
var result bool
err := db.db.QueryRow("SELECT backup_fetched FROM settings WHERE synthetic_id = 'id'").Scan(&result)
if err == sql.ErrNoRows {
return true, nil
}
return result, err
}
func (db *Database) ENSName() (string, error) {
var result sql.NullString
err := db.db.QueryRow("SELECT preferred_name FROM settings WHERE synthetic_id = 'id'").Scan(&result)

209
protocol/anon_metrics_test.go Normal file
View File

@ -0,0 +1,209 @@
// In order to run these tests, you must run a PostgreSQL database.
//
// Using Docker:
// docker run -e POSTGRES_HOST_AUTH_METHOD=trust -d -p 5432:5432 postgres:9.6-alpine
//
package protocol
import (
"context"
"crypto/ecdsa"
"testing"
"github.com/stretchr/testify/suite"
"go.uber.org/zap"
bindata "github.com/status-im/migrate/v4/source/go_bindata"
appmetricsDB "github.com/status-im/status-go/appmetrics"
gethbridge "github.com/status-im/status-go/eth-node/bridge/geth"
"github.com/status-im/status-go/eth-node/crypto"
"github.com/status-im/status-go/eth-node/types"
"github.com/status-im/status-go/postgres"
"github.com/status-im/status-go/protocol/anonmetrics"
"github.com/status-im/status-go/protocol/anonmetrics/migrations"
"github.com/status-im/status-go/protocol/tt"
"github.com/status-im/status-go/services/appmetrics"
"github.com/status-im/status-go/waku"
)
func TestMessengerAnonMetricsSuite(t *testing.T) {
suite.Run(t, new(MessengerAnonMetricsSuite))
}
type MessengerAnonMetricsSuite struct {
suite.Suite
alice *Messenger // client instance of Messenger
bob *Messenger // server instance of Messenger
aliceKey *ecdsa.PrivateKey // private key for the alice instance of Messenger
bobKey *ecdsa.PrivateKey // private key for the bob instance of Messenger
// If one wants to send messages between different instances of Messenger,
// a single Waku service should be shared.
shh types.Waku
logger *zap.Logger
}
func (s *MessengerAnonMetricsSuite) SetupSuite() {
// ResetDefaultTestPostgresDB Required to completely reset the Postgres DB
err := postgres.ResetDefaultTestPostgresDB()
s.NoError(err)
}
func (s *MessengerAnonMetricsSuite) SetupTest() {
var err error
s.logger = tt.MustCreateTestLogger()
// Setup Waku things
config := waku.DefaultConfig
config.MinimumAcceptedPoW = 0
shh := waku.New(&config, s.logger)
s.shh = gethbridge.NewGethWakuWrapper(shh)
s.Require().NoError(shh.Start())
// Generate private keys for Alice and Bob
s.aliceKey, err = crypto.GenerateKey()
s.Require().NoError(err)
s.bobKey, err = crypto.GenerateKey()
s.Require().NoError(err)
// Generate Alice Messenger as the client
amcc := &anonmetrics.ClientConfig{
ShouldSend: true,
SendAddress: &s.bobKey.PublicKey,
Active: anonmetrics.ActiveClientPhrase,
}
s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)})
s.Require().NoError(err)
_, err = s.alice.Start()
s.Require().NoError(err)
// Generate Bob Messenger as the Server
amsc := &anonmetrics.ServerConfig{
Enabled: true,
PostgresURI: postgres.DefaultTestURI,
Active: anonmetrics.ActiveServerPhrase,
}
s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)})
s.Require().NoError(err)
_, err = s.bob.Start()
s.Require().NoError(err)
}
func (s *MessengerAnonMetricsSuite) TearDownTest() {
// Down migrate the DB
if s.bob.anonMetricsServer != nil {
postgresMigration := bindata.Resource(migrations.AssetNames(), migrations.Asset)
m, err := anonmetrics.MakeMigration(s.bob.anonMetricsServer.PostgresDB, postgresMigration)
s.NoError(err)
err = m.Down()
s.NoError(err)
}
// Shutdown messengers
s.NoError(s.alice.Shutdown())
s.alice = nil
s.NoError(s.bob.Shutdown())
s.bob = nil
_ = s.logger.Sync()
}
func (s *MessengerAnonMetricsSuite) TestReceiveAnonMetric() {
// Create the appmetrics API to simulate incoming metrics from `status-react`
ama := appmetrics.NewAPI(appmetricsDB.NewDB(s.alice.database))
// Generate and store some metrics to Alice
ams := appmetricsDB.GenerateMetrics(10)
err := ama.SaveAppMetrics(context.Background(), ams)
s.Require().NoError(err)
// Check that we have what we stored
amsdb, err := ama.GetAppMetrics(context.Background(), 100, 0)
s.Require().NoError(err)
s.Require().Len(amsdb.AppMetrics, 10)
// Wait for messages to arrive at bob
_, err = WaitOnMessengerResponse(
s.bob,
func(r *MessengerResponse) bool { return len(r.AnonymousMetrics) > 0 },
"no anonymous metrics received",
)
s.Require().NoError(err)
// Get app metrics from postgres DB
bobMetrics, err := s.bob.anonMetricsServer.GetAppMetrics(100, 0)
s.Require().NoError(err)
s.Require().Len(bobMetrics, 5)
// Check the values of received and stored metrics against the broadcast metrics
for i, bobMetric := range bobMetrics {
s.Require().True(bobMetric.CreatedAt.Equal(amsdb.AppMetrics[i].CreatedAt), "created_at values are equal")
s.Require().Exactly(bobMetric.SessionID, amsdb.AppMetrics[i].SessionID, "session_id matched exactly")
s.Require().Exactly(bobMetric.Value, amsdb.AppMetrics[i].Value, "value matches exactly")
s.Require().Exactly(bobMetric.Event, amsdb.AppMetrics[i].Event, "event matches exactly")
s.Require().Exactly(bobMetric.OS, amsdb.AppMetrics[i].OS, "operating system matches exactly")
s.Require().Exactly(bobMetric.AppVersion, amsdb.AppMetrics[i].AppVersion, "app version matches exactly")
}
}
// TestActivationIsOff tests if using the incorrect activation phrase for the anon metric client / server deactivates
// the client / server. This test can be removed when / if the anon metrics functionality is reintroduced / re-approved.
func (s *MessengerAnonMetricsSuite) TestActivationIsOff() {
var err error
// Check the set up messengers are in the expected state with the correct activation phrases
s.NotNil(s.alice.anonMetricsClient)
s.NotNil(s.bob.anonMetricsServer)
// Generate Alice Messenger as the client with an incorrect phrase
amcc := &anonmetrics.ClientConfig{
ShouldSend: true,
SendAddress: &s.bobKey.PublicKey,
Active: "the wrong client phrase",
}
s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)})
s.NoError(err)
_, err = s.alice.Start()
s.Require().NoError(err)
s.Nil(s.alice.anonMetricsClient)
// Generate Alice Messenger as the client with an no activation phrase
amcc = &anonmetrics.ClientConfig{
ShouldSend: true,
SendAddress: &s.bobKey.PublicKey,
}
s.alice, err = newMessengerWithKey(s.shh, s.aliceKey, s.logger, []Option{WithAnonMetricsClientConfig(amcc)})
s.NoError(err)
_, err = s.alice.Start()
s.Require().NoError(err)
s.Nil(s.alice.anonMetricsClient)
// Generate Bob Messenger as the Server with an incorrect phrase
amsc := &anonmetrics.ServerConfig{
Enabled: true,
PostgresURI: postgres.DefaultTestURI,
Active: "the wrong server phrase",
}
s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)})
s.Require().NoError(err)
s.Nil(s.bob.anonMetricsServer)
// Generate Bob Messenger as the Server with no activation phrase
amsc = &anonmetrics.ServerConfig{
Enabled: true,
PostgresURI: postgres.DefaultTestURI,
}
s.bob, err = newMessengerWithKey(s.shh, s.bobKey, s.logger, []Option{WithAnonMetricsServerConfig(amsc)})
s.Require().NoError(err)
s.Nil(s.bob.anonMetricsServer)
}

4
protocol/messenger.go
View File

@ -3418,6 +3418,10 @@ func (m *Messenger) MessageByChatID(chatID, cursor string, limit int) ([]*common
return nil, "", err
}
if chat == nil {
return nil, "", ErrChatNotFound
}
if chat.Timeline() {
var chatIDs = []string{"@" + contactIDFromPublicKey(&m.identity.PublicKey)}
m.allContacts.Range(func(contactID string, contact *Contact) (shouldContinue bool) {

5
protocol/messenger_backup.go
View File

@ -54,6 +54,7 @@ func (m *Messenger) startBackupLoop() {
lastBackup, err := m.lastBackup()
if err != nil {
m.logger.Error("failed to fetch last backup time")
continue
}
now := time.Now().Unix()
@ -63,7 +64,9 @@ func (m *Messenger) startBackupLoop() {
}
m.logger.Debug("backing up data")
_, err = m.BackupData(context.Background())
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
_, err = m.BackupData(ctx)
if err != nil {
m.logger.Error("failed to backup data", zap.Error(err))
}

2
protocol/messenger_contacts.go
View File

@ -292,7 +292,7 @@ func (m *Messenger) BlockContact(contact *Contact) ([]*Chat, error) {
func (m *Messenger) UnblockContact(contactID string) error {
contact, ok := m.allContacts.Load(contactID)
if !ok || !contact.Added {
if !ok || !contact.Blocked {
return nil
}

2
protocol/messenger_installations_test.go
View File

@ -148,7 +148,7 @@ func (s *MessengerInstallationSuite) TestSyncInstallation() {
contact.LocalNickname = "Test Nickname"
_, err = s.m.AddContact(context.Background(), &requests.AddContact{ID: types.Hex2Bytes(contact.ID)})
s.Require().NoError(err)
_, err = s.m.SetContactLocalNickname(&requests.SetContactLocalNickname{ID: []byte(contact.ID), Nickname: contact.LocalNickname})
_, err = s.m.SetContactLocalNickname(&requests.SetContactLocalNickname{ID: types.Hex2Bytes(contact.ID), Nickname: contact.LocalNickname})
s.Require().NoError(err)
// add chat

33
protocol/messenger_mailserver.go
View File

@ -21,6 +21,7 @@ import (
// tolerance is how many seconds of potentially out-of-order messages we want to fetch
var tolerance uint32 = 60
var mailserverRequestTimeout = 45 * time.Second
var oneMonthInSeconds uint32 = 31 * 24 * 60 * 6
func (m *Messenger) shouldSync() (bool, error) {
if m.mailserver == nil || !m.online() {
@ -83,6 +84,7 @@ func (m *Messenger) scheduleSyncFilter(filter *transport.Filter) {
}
}
func (m *Messenger) scheduleSyncFilters(filters []*transport.Filter) (bool, error) {
shouldSync, err := m.shouldSync()
if err != nil {
@ -170,6 +172,23 @@ func (m *Messenger) syncChat(chatID string) (*MessengerResponse, error) {
return m.syncFilters(filters)
}
func (m *Messenger) syncBackup() error {
filter := m.transport.PersonalTopicFilter()
if filter == nil {
return errors.New("personal topic filter not loaded")
}
to := m.calculateMailserverTo()
from := uint32(m.getTimesource().GetCurrentTime()/1000) - oneMonthInSeconds
batch := MailserverBatch{From: from, To: to, Topics: []types.TopicType{filter.Topic}}
err := m.processMailserverBatch(batch)
if err != nil {
return err
}
return m.settings.SetBackupFetched(true)
}
func (m *Messenger) defaultSyncPeriodFromNow() (uint32, error) {
defaultSyncPeriod, err := m.settings.GetDefaultSyncPeriod()
if err != nil {
@ -201,6 +220,20 @@ func (m *Messenger) RequestAllHistoricMessages() (*MessengerResponse, error) {
return nil, nil
}
backupFetched, err := m.settings.BackupFetched()
if err != nil {
return nil, err
}
if !backupFetched {
m.logger.Info("fetching backup")
err := m.syncBackup()
if err != nil {
return nil, err
}
m.logger.Info("backup fetched")
}
return m.syncFilters(m.transport.Filters())
}

2
protocol/messenger_test.go
View File

@ -2365,7 +2365,7 @@ func (s *MessengerSuite) TestResendExpiredEmojis() {
//make sure it was resent and SendCount incremented
rawMessage, err = s.m.persistence.RawMessageByID(emojiID)
s.NoError(err)
s.Equal(2, rawMessage.SendCount)
s.True(rawMessage.SendCount >= 2)
}
type testTimeSource struct{}

2
protocol/migrations/migrations.go
View File

@ -1053,7 +1053,7 @@ func _1634896007_add_last_updated_locally_and_removedUpSql() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "1634896007_add_last_updated_locally_and_removed.up.sql", size: 131, mode: os.FileMode(0644), modTime: time.Unix(1635867803, 0)}
info := bindataFileInfo{name: "1634896007_add_last_updated_locally_and_removed.up.sql", size: 131, mode: os.FileMode(0644), modTime: time.Unix(1636971774, 0)}
a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2e, 0xa8, 0x34, 0xe2, 0xc0, 0x62, 0xc8, 0xd6, 0x5a, 0x87, 0xe3, 0x70, 0xe1, 0xc4, 0x16, 0x9c, 0x60, 0x2e, 0x98, 0xf0, 0x91, 0x84, 0xbe, 0xe0, 0xdf, 0x3e, 0x4d, 0x24, 0xc4, 0x6c, 0x40, 0x17}}
return a, nil
}

6
protocol/transport/filters_manager.go
View File

@ -473,6 +473,12 @@ func (f *FiltersManager) LoadDiscovery() ([]*Filter, error) {
return []*Filter{personalDiscoveryChat}, nil
}
func (f *FiltersManager) PersonalTopicFilter() *Filter {
personalDiscoveryTopic := PersonalDiscoveryTopic(&f.privateKey.PublicKey)
return f.filters[personalDiscoveryTopic]
}
// LoadPublic adds a filter for a public chat.
func (f *FiltersManager) LoadPublic(chatID string) (*Filter, error) {
f.mutex.Lock()

4
protocol/transport/transport.go
View File

@ -331,6 +331,10 @@ func (t *Transport) SendPrivateOnPersonalTopic(ctx context.Context, newMessage *
return t.api.Post(ctx, *newMessage)
}
func (t *Transport) PersonalTopicFilter() *Filter {
return t.filters.PersonalTopicFilter()
}
func (t *Transport) LoadKeyFilters(key *ecdsa.PrivateKey) (*Filter, error) {
return t.filters.LoadEphemeral(&key.PublicKey, key, true)
}