2020-07-03 10:08:47 +00:00
|
|
|
package protocol
|
|
|
|
|
|
|
|
import (
|
|
|
|
"database/sql"
|
2021-02-17 23:14:48 +00:00
|
|
|
"encoding/json"
|
2020-07-22 07:41:40 +00:00
|
|
|
|
2022-06-02 12:17:52 +00:00
|
|
|
"github.com/status-im/status-go/rpc"
|
2022-05-09 13:07:57 +00:00
|
|
|
"github.com/status-im/status-go/server"
|
2022-01-17 03:42:11 +00:00
|
|
|
"github.com/status-im/status-go/services/browsers"
|
|
|
|
|
2020-07-22 07:41:40 +00:00
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2021-02-15 23:18:08 +00:00
|
|
|
"github.com/status-im/status-go/appdatabase/migrations"
|
2020-11-24 23:16:19 +00:00
|
|
|
"github.com/status-im/status-go/multiaccounts"
|
2021-02-17 23:14:48 +00:00
|
|
|
"github.com/status-im/status-go/multiaccounts/accounts"
|
2022-03-23 18:47:00 +00:00
|
|
|
"github.com/status-im/status-go/multiaccounts/settings"
|
2021-02-17 23:14:48 +00:00
|
|
|
"github.com/status-im/status-go/params"
|
2021-09-01 12:02:18 +00:00
|
|
|
"github.com/status-im/status-go/protocol/anonmetrics"
|
2020-07-06 08:54:22 +00:00
|
|
|
"github.com/status-im/status-go/protocol/common"
|
2021-04-19 12:09:46 +00:00
|
|
|
"github.com/status-im/status-go/protocol/communities"
|
feat: introduce messenger APIs to extract discord channels
As part of the new Discord <-> Status Community Import functionality,
we're adding an API that extracts all discord categories and channels
from a previously exported discord export file.
These APIs can be used in clients to show the user what categories and
channels will be imported later on.
There are two APIs:
1. `Messenger.ExtractDiscordCategoriesAndChannels(filesToimport
[]string) (*MessengerResponse, map[string]*discord.ImportError)`
This takes a list of exported discord export (JSON) files (typically one per
channel), reads them, and extracts the categories and channels into
dedicated data structures (`[]DiscordChannel` and `[]DiscordCategory`)
It also returns the oldest message timestamp found in all extracted
channels.
The API is synchronous and returns the extracted data as
a `*MessengerResponse`. This allows to make the API available
status-go's RPC interface.
The error case is a `map[string]*discord.ImportError` where each key
is a file path of a JSON file that we tried to extract data from, and
the value a `discord.ImportError` which holds an error message and an
error code, allowing for distinguishing between "critical" errors and
"non-critical" errors.
2. `Messenger.RequestExtractDiscordCategoriesAndChannels(filesToImport
[]string)`
This is the asynchronous counterpart to
`ExtractDiscordCategoriesAndChannels`. The reason this API has been
added is because discord servers can have a lot of message and
channel data, which causes `ExtractDiscordCategoriesAndChannels` to
block the thread for too long, making apps potentially feel like they
are stuck.
This API runs inside a go routine, eventually calls
`ExtractDiscordCategoriesAndChannels`, and then emits a newly
introduced `DiscordCategoriesAndChannelsExtractedSignal` that clients
can react to.
Failure of extraction has to be determined by the
`discord.ImportErrors` emitted by the signal.
**A note about exported discord history files**
We expect users to export their discord histories via the
[DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter/wiki/GUI%2C-CLI-and-Formats-explained#exportguild)
tool. The tool allows to export the data in different formats, such as
JSON, HTML and CSV.
We expect users to have their data exported as JSON.
Closes: https://github.com/status-im/status-desktop/issues/6690
2022-07-13 09:33:53 +00:00
|
|
|
"github.com/status-im/status-go/protocol/discord"
|
2020-07-03 10:08:47 +00:00
|
|
|
"github.com/status-im/status-go/protocol/protobuf"
|
2020-07-22 07:41:40 +00:00
|
|
|
"github.com/status-im/status-go/protocol/pushnotificationclient"
|
|
|
|
"github.com/status-im/status-go/protocol/pushnotificationserver"
|
2020-07-03 10:08:47 +00:00
|
|
|
"github.com/status-im/status-go/protocol/transport"
|
2021-01-14 22:15:13 +00:00
|
|
|
"github.com/status-im/status-go/services/mailservers"
|
2020-07-03 10:08:47 +00:00
|
|
|
)
|
|
|
|
|
2021-02-23 15:47:45 +00:00
|
|
|
type MessageDeliveredHandler func(string, string)
|
|
|
|
|
2021-04-19 12:09:46 +00:00
|
|
|
type MessengerSignalsHandler interface {
|
|
|
|
MessageDelivered(chatID string, messageID string)
|
|
|
|
CommunityInfoFound(community *communities.Community)
|
2021-03-25 15:15:22 +00:00
|
|
|
MessengerResponse(response *MessengerResponse)
|
2022-11-24 21:09:17 +00:00
|
|
|
HistoryRequestStarted(numBatches int)
|
|
|
|
HistoryRequestCompleted()
|
|
|
|
|
2021-10-11 15:39:52 +00:00
|
|
|
BackupPerformed(uint64)
|
2022-03-21 14:18:36 +00:00
|
|
|
HistoryArchivesProtocolEnabled()
|
|
|
|
HistoryArchivesProtocolDisabled()
|
|
|
|
CreatingHistoryArchives(communityID string)
|
|
|
|
NoHistoryArchivesCreated(communityID string, from int, to int)
|
|
|
|
HistoryArchivesCreated(communityID string, from int, to int)
|
|
|
|
HistoryArchivesSeeding(communityID string)
|
|
|
|
HistoryArchivesUnseeded(communityID string)
|
|
|
|
HistoryArchiveDownloaded(communityID string, from int, to int)
|
2022-12-01 14:02:17 +00:00
|
|
|
DownloadingHistoryArchivesStarted(communityID string)
|
2022-09-15 07:59:02 +00:00
|
|
|
DownloadingHistoryArchivesFinished(communityID string)
|
2022-12-02 12:45:41 +00:00
|
|
|
ImportingHistoryArchiveMessages(communityID string)
|
2022-08-02 23:08:01 +00:00
|
|
|
StatusUpdatesTimedOut(statusUpdates *[]UserStatus)
|
feat: introduce messenger APIs to extract discord channels
As part of the new Discord <-> Status Community Import functionality,
we're adding an API that extracts all discord categories and channels
from a previously exported discord export file.
These APIs can be used in clients to show the user what categories and
channels will be imported later on.
There are two APIs:
1. `Messenger.ExtractDiscordCategoriesAndChannels(filesToimport
[]string) (*MessengerResponse, map[string]*discord.ImportError)`
This takes a list of exported discord export (JSON) files (typically one per
channel), reads them, and extracts the categories and channels into
dedicated data structures (`[]DiscordChannel` and `[]DiscordCategory`)
It also returns the oldest message timestamp found in all extracted
channels.
The API is synchronous and returns the extracted data as
a `*MessengerResponse`. This allows to make the API available
status-go's RPC interface.
The error case is a `map[string]*discord.ImportError` where each key
is a file path of a JSON file that we tried to extract data from, and
the value a `discord.ImportError` which holds an error message and an
error code, allowing for distinguishing between "critical" errors and
"non-critical" errors.
2. `Messenger.RequestExtractDiscordCategoriesAndChannels(filesToImport
[]string)`
This is the asynchronous counterpart to
`ExtractDiscordCategoriesAndChannels`. The reason this API has been
added is because discord servers can have a lot of message and
channel data, which causes `ExtractDiscordCategoriesAndChannels` to
block the thread for too long, making apps potentially feel like they
are stuck.
This API runs inside a go routine, eventually calls
`ExtractDiscordCategoriesAndChannels`, and then emits a newly
introduced `DiscordCategoriesAndChannelsExtractedSignal` that clients
can react to.
Failure of extraction has to be determined by the
`discord.ImportErrors` emitted by the signal.
**A note about exported discord history files**
We expect users to export their discord histories via the
[DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter/wiki/GUI%2C-CLI-and-Formats-explained#exportguild)
tool. The tool allows to export the data in different formats, such as
JSON, HTML and CSV.
We expect users to have their data exported as JSON.
Closes: https://github.com/status-im/status-desktop/issues/6690
2022-07-13 09:33:53 +00:00
|
|
|
DiscordCategoriesAndChannelsExtracted(categories []*discord.Category, channels []*discord.Channel, oldestMessageTimestamp int64, errors map[string]*discord.ImportError)
|
2022-09-29 11:50:23 +00:00
|
|
|
DiscordCommunityImportProgress(importProgress *discord.ImportProgress)
|
|
|
|
DiscordCommunityImportFinished(communityID string)
|
|
|
|
DiscordCommunityImportCancelled(communityID string)
|
2021-04-19 12:09:46 +00:00
|
|
|
}
|
|
|
|
|
2020-07-03 10:08:47 +00:00
|
|
|
type config struct {
|
|
|
|
// This needs to be exposed until we move here mailserver logic
|
|
|
|
// as otherwise the client is not notified of a new filter and
|
|
|
|
// won't be pulling messages from mailservers until it reloads the chats/filters
|
2021-01-11 10:32:51 +00:00
|
|
|
onContactENSVerified func(*MessengerResponse)
|
2020-07-03 10:08:47 +00:00
|
|
|
|
|
|
|
// systemMessagesTranslations holds translations for system-messages
|
2021-03-29 15:41:30 +00:00
|
|
|
systemMessagesTranslations *systemMessageTranslationsMap
|
2020-07-03 10:08:47 +00:00
|
|
|
// Config for the envelopes monitor
|
|
|
|
envelopesMonitorConfig *transport.EnvelopesMonitorConfig
|
|
|
|
|
2020-12-21 08:41:50 +00:00
|
|
|
featureFlags common.FeatureFlags
|
2020-07-03 10:08:47 +00:00
|
|
|
|
|
|
|
// A path to a database or a database instance is required.
|
|
|
|
// The database instance has a higher priority.
|
2021-01-14 22:15:13 +00:00
|
|
|
dbConfig dbConfig
|
|
|
|
db *sql.DB
|
2021-02-15 23:18:08 +00:00
|
|
|
afterDbCreatedHooks []Option
|
2021-01-14 22:15:13 +00:00
|
|
|
multiAccount *multiaccounts.Database
|
|
|
|
mailserversDatabase *mailservers.Database
|
|
|
|
account *multiaccounts.Account
|
2022-01-12 16:02:01 +00:00
|
|
|
clusterConfig params.ClusterConfig
|
2022-01-17 03:42:11 +00:00
|
|
|
browserDatabase *browsers.Database
|
2022-03-08 13:17:26 +00:00
|
|
|
torrentConfig *params.TorrentConfig
|
2022-06-15 14:49:31 +00:00
|
|
|
httpServer *server.MediaServer
|
2022-06-02 12:17:52 +00:00
|
|
|
rpcClient *rpc.Client
|
2020-07-03 10:08:47 +00:00
|
|
|
|
2021-01-11 10:32:51 +00:00
|
|
|
verifyTransactionClient EthClient
|
|
|
|
verifyENSURL string
|
|
|
|
verifyENSContractAddress string
|
2020-07-03 10:08:47 +00:00
|
|
|
|
2021-09-01 12:02:18 +00:00
|
|
|
anonMetricsClientConfig *anonmetrics.ClientConfig
|
|
|
|
anonMetricsServerConfig *anonmetrics.ServerConfig
|
|
|
|
|
2020-07-22 07:41:40 +00:00
|
|
|
pushNotificationServerConfig *pushnotificationserver.Config
|
|
|
|
pushNotificationClientConfig *pushnotificationclient.Config
|
2020-07-03 10:08:47 +00:00
|
|
|
|
|
|
|
logger *zap.Logger
|
2021-02-23 15:47:45 +00:00
|
|
|
|
2022-08-24 12:06:48 +00:00
|
|
|
outputMessagesCSV bool
|
|
|
|
|
2021-04-19 12:09:46 +00:00
|
|
|
messengerSignalsHandler MessengerSignalsHandler
|
2021-11-03 12:38:37 +00:00
|
|
|
|
|
|
|
telemetryServerURL string
|
2020-07-03 10:08:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type Option func(*config) error
|
|
|
|
|
|
|
|
// WithSystemMessagesTranslations is required for Group Chats which are currently disabled.
|
|
|
|
// nolint: unused
|
|
|
|
func WithSystemMessagesTranslations(t map[protobuf.MembershipUpdateEvent_EventType]string) Option {
|
|
|
|
return func(c *config) error {
|
2021-03-29 15:41:30 +00:00
|
|
|
c.systemMessagesTranslations.Init(t)
|
2020-07-03 10:08:47 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithCustomLogger(logger *zap.Logger) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.logger = logger
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-27 20:27:20 +00:00
|
|
|
func WithDatabaseConfig(dbPath string, dbKey string, dbKDFIterations int) Option {
|
2020-07-03 10:08:47 +00:00
|
|
|
return func(c *config) error {
|
2022-09-27 20:27:20 +00:00
|
|
|
c.dbConfig = dbConfig{dbPath: dbPath, dbKey: dbKey, dbKDFIterations: dbKDFIterations}
|
2020-07-03 10:08:47 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithVerifyTransactionClient(client EthClient) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.verifyTransactionClient = client
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithDatabase(db *sql.DB) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.db = db
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-15 23:18:08 +00:00
|
|
|
func WithToplevelDatabaseMigrations() Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error {
|
|
|
|
return migrations.Migrate(c.db)
|
|
|
|
})
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-23 18:47:00 +00:00
|
|
|
func WithAppSettings(s settings.Settings, nc params.NodeConfig) Option {
|
2021-02-17 23:14:48 +00:00
|
|
|
return func(c *config) error {
|
|
|
|
c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error {
|
|
|
|
if s.Networks == nil {
|
|
|
|
networks := new(json.RawMessage)
|
|
|
|
if err := networks.UnmarshalJSON([]byte("net")); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
s.Networks = networks
|
|
|
|
}
|
|
|
|
|
2022-03-23 18:47:00 +00:00
|
|
|
sDB, err := accounts.NewDB(c.db)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2021-02-17 23:14:48 +00:00
|
|
|
return sDB.CreateSettings(s, nc)
|
|
|
|
})
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-24 13:13:46 +00:00
|
|
|
func WithMultiAccounts(ma *multiaccounts.Database) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.multiAccount = ma
|
|
|
|
return nil
|
2021-01-14 22:15:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithMailserversDatabase(ma *mailservers.Database) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.mailserversDatabase = ma
|
|
|
|
return nil
|
2020-11-24 13:13:46 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-09 14:03:43 +00:00
|
|
|
func WithAccount(acc *multiaccounts.Account) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.account = acc
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-01-17 03:42:11 +00:00
|
|
|
func WithBrowserDatabase(bd *browsers.Database) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.browserDatabase = bd
|
|
|
|
if c.browserDatabase == nil {
|
|
|
|
c.afterDbCreatedHooks = append(c.afterDbCreatedHooks, func(c *config) error {
|
|
|
|
c.browserDatabase = browsers.NewDB(c.db)
|
|
|
|
return nil
|
|
|
|
})
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-01 12:02:18 +00:00
|
|
|
func WithAnonMetricsClientConfig(anonMetricsClientConfig *anonmetrics.ClientConfig) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.anonMetricsClientConfig = anonMetricsClientConfig
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func WithAnonMetricsServerConfig(anonMetricsServerConfig *anonmetrics.ServerConfig) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.anonMetricsServerConfig = anonMetricsServerConfig
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-03 12:38:37 +00:00
|
|
|
func WithTelemetry(serverURL string) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.telemetryServerURL = serverURL
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-22 07:41:40 +00:00
|
|
|
func WithPushNotificationServerConfig(pushNotificationServerConfig *pushnotificationserver.Config) Option {
|
2020-07-03 10:08:47 +00:00
|
|
|
return func(c *config) error {
|
|
|
|
c.pushNotificationServerConfig = pushNotificationServerConfig
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-22 07:41:40 +00:00
|
|
|
func WithPushNotificationClientConfig(pushNotificationClientConfig *pushnotificationclient.Config) Option {
|
2020-07-15 12:25:01 +00:00
|
|
|
return func(c *config) error {
|
|
|
|
c.pushNotificationClientConfig = pushNotificationClientConfig
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-03 10:08:47 +00:00
|
|
|
func WithDatasync() func(c *config) error {
|
|
|
|
return func(c *config) error {
|
2020-07-06 08:54:22 +00:00
|
|
|
c.featureFlags.Datasync = true
|
2020-07-03 10:08:47 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-27 15:20:16 +00:00
|
|
|
func WithPushNotifications() func(c *config) error {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.featureFlags.PushNotifications = true
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-03 10:08:47 +00:00
|
|
|
func WithEnvelopesMonitorConfig(emc *transport.EnvelopesMonitorConfig) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.envelopesMonitorConfig = emc
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2021-02-23 15:47:45 +00:00
|
|
|
|
2021-04-19 12:09:46 +00:00
|
|
|
func WithSignalsHandler(h MessengerSignalsHandler) Option {
|
2021-02-23 15:47:45 +00:00
|
|
|
return func(c *config) error {
|
2021-04-19 12:09:46 +00:00
|
|
|
c.messengerSignalsHandler = h
|
2021-02-23 15:47:45 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2021-01-11 10:32:51 +00:00
|
|
|
|
|
|
|
func WithENSVerificationConfig(onENSVerified func(*MessengerResponse), url, address string) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.onContactENSVerified = onENSVerified
|
|
|
|
c.verifyENSURL = url
|
|
|
|
c.verifyENSContractAddress = address
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2022-01-12 16:02:01 +00:00
|
|
|
|
|
|
|
func WithClusterConfig(cc params.ClusterConfig) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.clusterConfig = cc
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2022-03-08 13:17:26 +00:00
|
|
|
|
|
|
|
func WithTorrentConfig(tc *params.TorrentConfig) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.torrentConfig = tc
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2022-05-09 13:07:57 +00:00
|
|
|
|
2022-06-15 14:49:31 +00:00
|
|
|
func WithHTTPServer(s *server.MediaServer) Option {
|
2022-05-09 13:07:57 +00:00
|
|
|
return func(c *config) error {
|
|
|
|
c.httpServer = s
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2022-06-02 12:17:52 +00:00
|
|
|
|
|
|
|
func WithRPCClient(r *rpc.Client) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.rpcClient = r
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2022-08-24 12:06:48 +00:00
|
|
|
|
|
|
|
func WithMessageCSV(enabled bool) Option {
|
|
|
|
return func(c *config) error {
|
|
|
|
c.outputMessagesCSV = enabled
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|