status-im
/
status-go
mirror of https://github.com/status-im/status-go.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(import-tool): process import data in chunks 

This commit refactors the discord import tool such that,
instead of loading all data to be imported into memory at
once, it will now perform the import on a per file basis.

This improves the memory pressure for the node performing
the import and seems to increase its performance as well.
This commit is contained in:
Pascal Precht 2023-01-26 13:52:43 +01:00 committed by r4bbit
parent ee9f8edfcf
commit f31e40264e
5 changed files with 417 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
protocol/communities/manager.go
View File

@ -504,7 +504,7 @@ func (m *Manager) ImportCommunity(key *ecdsa.PrivateKey) (*Community, error) {
return community, nil
}
func (m *Manager) CreateChat(communityID types.HexBytes, chat *protobuf.CommunityChat, publish bool) (*Community, *CommunityChanges, error) {
func (m *Manager) CreateChat(communityID types.HexBytes, chat *protobuf.CommunityChat, publish bool, thirdPartyID string) (*Community, *CommunityChanges, error) {
community, err := m.GetByID(communityID)
if err != nil {
return nil, nil, err
@ -513,6 +513,10 @@ func (m *Manager) CreateChat(communityID types.HexBytes, chat *protobuf.Communit
return nil, nil, ErrOrgNotFound
}
chatID := uuid.New().String()
if thirdPartyID != "" {
chatID = chatID + thirdPartyID
}
changes, err := community.CreateChat(chatID, chat)
if err != nil {
return nil, nil, err
@ -598,7 +602,11 @@ func (m *Manager) CreateCategory(request *requests.CreateCommunityCategory, publ
if community == nil {
return nil, nil, ErrOrgNotFound
}
categoryID := uuid.New().String()
if request.ThirdPartyID != "" {
categoryID = categoryID + request.ThirdPartyID
}
// Remove communityID prefix from chatID if exists
for i, cid := range request.ChatIDs {

2
protocol/communities/manager_test.go
View File

@ -707,7 +707,7 @@ func (s *ManagerSuite) buildCommunityWithChat() (*Community, string, error) {
},
Members: make(map[string]*protobuf.CommunityMember),
}
_, changes, err := s.manager.CreateChat(community.ID(), chat, true)
_, changes, err := s.manager.CreateChat(community.ID(), chat, true, "")
if err != nil {
return nil, "", err
}

6
protocol/discord/types.go
View File

@ -160,10 +160,12 @@ type ImportProgress struct {
ErrorsCount uint `json:"errorsCount"`
WarningsCount uint `json:"warningsCount"`
Stopped bool `json:"stopped"`
TotalChunkCount int `json:"totalChunksCount,omitempty"`
CurrentChunk int `json:"currentChunk,omitempty"`
m sync.Mutex
}
func (progress *ImportProgress) Init(tasks []ImportTask) {
func (progress *ImportProgress) Init(totalChunkCount int, tasks []ImportTask) {
progress.Progress = 0
progress.Tasks = make([]*ImportTaskProgress, 0)
for _, task := range tasks {
@ -180,6 +182,8 @@ func (progress *ImportProgress) Init(tasks []ImportTask) {
progress.ErrorsCount = 0
progress.WarningsCount = 0
progress.Stopped = false
progress.TotalChunkCount = totalChunkCount
progress.CurrentChunk = 0
}
func (progress *ImportProgress) Stop() {

322
protocol/messenger_communities.go
View File

@ -904,7 +904,7 @@ func (m *Messenger) CreateCommunityChat(communityID types.HexBytes, c *protobuf.
var response MessengerResponse
c.Identity.FirstMessageTimestamp = FirstMessageTimestampNoMessage
community, changes, err := m.communitiesManager.CreateChat(communityID, c, true)
community, changes, err := m.communitiesManager.CreateChat(communityID, c, true, "")
if err != nil {
return nil, err
}
@ -2339,13 +2339,15 @@ func (m *Messenger) RequestExtractDiscordChannelsAndCategories(filesToImport []s
func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscordCommunity) {
go func() {
totalImportChunkCount := len(request.FilesToImport)
progressUpdates := make(chan *discord.ImportProgress)
done := make(chan struct{})
cancel := make(chan string)
m.startPublishImportProgressInterval(progressUpdates, cancel, done)
importProgress := &discord.ImportProgress{}
importProgress.Init([]discord.ImportTask{
importProgress.Init(totalImportChunkCount, []discord.ImportTask{
discord.CommunityCreationTask,
discord.ChannelsCreationTask,
discord.ImportMessagesTask,
@ -2357,31 +2359,6 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
// initial progress immediately
m.publishImportProgress(importProgress)
exportData, errs := m.ExtractDiscordDataFromImportFiles(request.FilesToImport)
if len(errs) > 0 {
for _, err := range errs {
importProgress.AddTaskError(discord.CommunityCreationTask, err)
}
progressUpdates <- importProgress
return
}
totalChannelsCount := len(exportData.ExportedData)
totalMessageCount := exportData.MessageCount
if totalChannelsCount == 0 || totalMessageCount == 0 {
importError := discord.Error(discord.ErrNoChannelData.Error())
if totalMessageCount == 0 {
importError.Message = discord.ErrNoMessageData.Error()
}
importProgress.AddTaskError(discord.CommunityCreationTask, importError)
importProgress.StopTask(discord.CommunityCreationTask)
progressUpdates <- importProgress
return
}
importProgress.UpdateTaskProgress(discord.CommunityCreationTask, 0.5)
progressUpdates <- importProgress
createCommunityRequest := request.ToCreateCommunityRequest()
// We're calling `CreateCommunity` on `communitiesManager` directly, instead of
@ -2434,7 +2411,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
importProgress.CommunityImages[t] = images.IdentityImage{Name: t, Payload: i.Payload}
}
importProgress.UpdateTaskProgress(discord.CommunityCreationTask, 0.75)
importProgress.UpdateTaskProgress(discord.CommunityCreationTask, 1)
progressUpdates <- importProgress
if m.DiscordImportMarkedAsCancelled(communityID) {
@ -2444,15 +2421,54 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
return
}
//This is a map of discord category IDs <-> Status category IDs
var chatsToSave []*Chat
processedChannelIds := make(map[string]string, 0)
processedCategoriesIds := make(map[string]string, 0)
totalCategoriesCount := len(exportData.Categories)
for i, importFile := range request.FilesToImport {
exportData, errs := m.ExtractDiscordDataFromImportFiles([]string{importFile})
if len(errs) > 0 {
for _, err := range errs {
importProgress.AddTaskError(discord.CommunityCreationTask, err)
}
progressUpdates <- importProgress
return
}
totalChannelsCount := len(exportData.ExportedData)
totalMessageCount := exportData.MessageCount
if totalChannelsCount == 0 || totalMessageCount == 0 {
importError := discord.Error(fmt.Errorf("No channel to import messages from in file: %s", importFile).Error())
if totalMessageCount == 0 {
importError.Message = fmt.Errorf("No messages to import in file: %s", importFile).Error()
}
importProgress.AddTaskError(discord.ChannelsCreationTask, importError)
progressUpdates <- importProgress
continue
}
importProgress.CurrentChunk = i + 1
// We actually only ever receive a single category
// from `exportData` but since it's a map, we still have to
// iterate over it to access its values
for _, category := range exportData.Categories {
categories := discordCommunity.Categories()
exists := false
for catID := range categories {
if strings.HasSuffix(catID, category.ID) {
exists = true
break
}
}
if !exists {
createCommunityCategoryRequest := &requests.CreateCommunityCategory{
CommunityID: discordCommunity.ID(),
CategoryName: category.Name,
ThirdPartyID: category.ID,
ChatIDs: make([]string, 0),
}
// We call `CreateCategory` on `communitiesManager` directly so we can control
@ -2472,13 +2488,13 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
for _, addedCategory := range changes.CategoriesAdded {
processedCategoriesIds[category.ID] = addedCategory.CategoryId
}
}
}
// We're multiplying `progressValue` by 0.25 as it's added to the previous 0.75 progress
progressValue := (float32(len(processedCategoriesIds)) / float32(totalCategoriesCount)) * 0.25
importProgress.UpdateTaskProgress(discord.CommunityCreationTask, 0.75+progressValue)
progressValue := calculateProgress(i+1, totalImportChunkCount, (float32(1) / 2))
importProgress.UpdateTaskProgress(discord.ChannelsCreationTask, progressValue)
progressUpdates <- importProgress
}
if m.DiscordImportMarkedAsCancelled(communityID) {
importProgress.StopTask(discord.CommunityCreationTask)
@ -2487,15 +2503,25 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
return
}
var chatsToSave []*Chat
processedChannelIds := make(map[string]string, 0)
messagesToSave := make(map[string]*common.Message, 0)
pinMessagesToSave := make([]*common.PinMessage, 0)
authorProfilesToSave := make(map[string]*protobuf.DiscordMessageAuthor, 0)
messageAttachmentsToDownload := make([]*protobuf.DiscordMessageAttachment, 0)
for _, channel := range exportData.ExportedData {
// Save to access the first item here as we process
// exported data by files which only holds a single channel
channel := exportData.ExportedData[0]
chatIDs := discordCommunity.ChatIDs()
exists := false
for _, chatID := range chatIDs {
if strings.HasSuffix(chatID, channel.Channel.ID) {
exists = true
break
}
}
if !exists {
communityChat := &protobuf.CommunityChat{
Permissions: &protobuf.CommunityPermissions{
Access: protobuf.CommunityPermissions_NO_MEMBERSHIP,
@ -2511,7 +2537,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
// We call `CreateChat` on `communitiesManager` directly to get more control
// over whether we want to publish the updated community description.
communityWithChats, changes, err := m.communitiesManager.CreateChat(discordCommunity.ID(), communityChat, false)
communityWithChats, changes, err := m.communitiesManager.CreateChat(discordCommunity.ID(), communityChat, false, channel.Channel.ID)
if err != nil {
m.cleanUpImport(communityID)
errmsg := err.Error()
@ -2533,24 +2559,24 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
chatsToSave = append(chatsToSave, c)
processedChannelIds[channel.Channel.ID] = c.ID
}
}
progressValue := float32(len(processedChannelIds)) / float32(totalChannelsCount)
progressValue = calculateProgress(i+1, totalImportChunkCount, 1)
importProgress.UpdateTaskProgress(discord.ChannelsCreationTask, progressValue)
progressUpdates <- importProgress
for _, discordMessage := range channel.Messages {
progressValue := float32(len(messagesToSave)) / float32(totalMessageCount)
for ii, discordMessage := range channel.Messages {
timestamp, err := time.Parse(discordTimestampLayout, discordMessage.Timestamp)
if err != nil {
m.logger.Error("failed to parse discord message timestamp", zap.Error(err))
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Warning(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
if timestamp.Unix() < request.From {
progressUpdates <- importProgress
continue
}
@ -2558,7 +2584,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
if err != nil {
m.logger.Error("failed to check if message author exists in database", zap.Error(err))
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Error(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
@ -2566,7 +2592,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
err := m.persistence.SaveDiscordMessageAuthor(discordMessage.Author)
if err != nil {
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Error(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
}
@ -2575,7 +2601,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
if err != nil {
m.logger.Error("failed to check if message avatar payload exists in database", zap.Error(err))
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Error(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
@ -2591,7 +2617,6 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
if err != nil {
m.logger.Error("failed to parse discord message timestamp", zap.Error(err))
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Warning(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
@ -2641,7 +2666,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
if err != nil {
m.logger.Error("failed to prepare message content", zap.Error(err))
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Error(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
@ -2662,7 +2687,6 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
if err != nil {
m.logger.Error("failed to parse marshal pin message", zap.Error(err))
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Warning(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
@ -2671,7 +2695,6 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
if err != nil {
m.logger.Error("failed to wrap pin message", zap.Error(err))
importProgress.AddTaskError(discord.ImportMessagesTask, discord.Warning(err.Error()))
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
continue
}
@ -2692,13 +2715,11 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
} else {
messagesToSave[communityID+discordMessage.Id] = messageToSave
}
}
// We're multiplying `progressValue` by `0.5` so we leave 50% for actual save operations
// The 0.5 could be calculated but we'd need to know the total message chunks count,
// which we don't at this point
progressValue = float32(len(messagesToSave)) / float32(totalMessageCount) * 0.5
progressValue := calculateProgress(i+1, totalImportChunkCount, float32(ii+1)/float32(len(channel.Messages))*0.5)
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
}
if m.DiscordImportMarkedAsCancelled(communityID) {
importProgress.StopTask(discord.ImportMessagesTask)
@ -2706,7 +2727,6 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
cancel <- communityID
return
}
}
var discordMessages []*protobuf.DiscordMessage
for _, msg := range messagesToSave {
@ -2718,11 +2738,8 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
discordMessageChunks := chunkSlice(discordMessages, maxChunkSizeMessages)
chunksCount := len(discordMessageChunks)
// Signal to clients that save operations are starting
importProgress.UpdateTaskState(discord.ImportMessagesTask, discord.TaskStateSaving)
progressUpdates <- importProgress
for i, msgs := range discordMessageChunks {
for ii, msgs := range discordMessageChunks {
m.communitiesManager.LogStdout(fmt.Sprintf("saving %d/%d chunk with %d discord messages", ii+1, chunksCount, len(msgs)))
err = m.persistence.SaveDiscordMessages(msgs)
if err != nil {
m.cleanUpImport(communityID)
@ -2741,9 +2758,10 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
// We're multiplying `chunksCount` by `0.25` so we leave 25% for additional save operations
// 0.5 are the previous 50% of progress
currentCount := i + 1
progressValue := 0.5 + (float32(currentCount) / float32(chunksCount) * 0.25)
currentCount := ii + 1
progressValue := calculateProgress(i+1, totalImportChunkCount, 0.5+(float32(currentCount)/float32(chunksCount))*0.25)
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
// We slow down the saving of message chunks to keep the database responsive
if currentCount < chunksCount {
@ -2751,8 +2769,6 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
}
}
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, 0.75)
var messages []*common.Message
for _, msg := range messagesToSave {
messages = append(messages, msg)
@ -2763,7 +2779,8 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
messageChunks := chunkSlice(messages, maxChunkSizeMessages)
chunksCount = len(messageChunks)
for i, msgs := range messageChunks {
for ii, msgs := range messageChunks {
m.communitiesManager.LogStdout(fmt.Sprintf("saving %d/%d chunk with %d app messages", ii+1, chunksCount, len(msgs)))
err = m.persistence.SaveMessages(msgs)
if err != nil {
m.cleanUpImport(communityID)
@ -2782,9 +2799,11 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
// 0.75 are the previous 75% of progress, hence we multiply our chunk progress
// by 0.25
currentCount := i + 1
progressValue := 0.75 + ((float32(currentCount) / float32(chunksCount)) * 0.25)
currentCount := ii + 1
progressValue := calculateProgress(i+1, totalImportChunkCount, 0.75+(float32(currentCount)/float32(chunksCount))*0.25)
// progressValue := 0.75 + ((float32(currentCount) / float32(chunksCount)) * 0.25)
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, progressValue)
progressUpdates <- importProgress
// We slow down the saving of message chunks to keep the database responsive
if currentCount < chunksCount {
@ -2811,9 +2830,6 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
}
}
importProgress.UpdateTaskProgress(discord.ImportMessagesTask, 1)
progressUpdates <- importProgress
totalAssetsCount := len(messageAttachmentsToDownload) + len(authorProfilesToSave)
var assetCounter discord.AssetCounter
@ -2823,6 +2839,8 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
wg.Add(1)
go func(id string, author *protobuf.DiscordMessageAuthor) {
defer wg.Done()
m.communitiesManager.LogStdout(fmt.Sprintf("downloading asset %d/%d", assetCounter.Value()+1, totalAssetsCount))
imagePayload, err := discord.DownloadAvatarAsset(author.AvatarUrl)
if err != nil {
errmsg := fmt.Sprintf("Couldn't download profile avatar '%s': %s", author.AvatarUrl, err.Error())
@ -2844,17 +2862,18 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
author.AvatarImagePayload = imagePayload
authorProfilesToSave[id] = author
assetCounter.Increase()
progressValue := float32(assetCounter.Value()) / float32(totalAssetsCount)
importProgress.UpdateTaskProgress(discord.DownloadAssetsTask, progressValue)
progressUpdates <- importProgress
if m.DiscordImportMarkedAsCancelled(discordCommunity.IDString()) {
importProgress.StopTask(discord.DownloadAssetsTask)
progressUpdates <- importProgress
cancel <- discordCommunity.IDString()
return
}
assetCounter.Increase()
progressValue := calculateProgress(i+1, totalImportChunkCount, (float32(assetCounter.Value())/float32(totalAssetsCount))*0.25)
importProgress.UpdateTaskProgress(discord.DownloadAssetsTask, progressValue)
progressUpdates <- importProgress
}(id, author)
}
wg.Wait()
@ -2871,7 +2890,10 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
wg.Add(1)
go func(attachments []*protobuf.DiscordMessageAttachment) {
defer wg.Done()
for i, attachment := range attachments {
for ii, attachment := range attachments {
m.communitiesManager.LogStdout(fmt.Sprintf("downloading asset %d/%d", assetCounter.Value()+1, totalAssetsCount))
assetPayload, contentType, err := discord.DownloadAsset(attachment.Url)
if err != nil {
errmsg := fmt.Sprintf("Couldn't download message attachment '%s': %s", attachment.Url, err.Error())
@ -2885,15 +2907,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
attachment.Payload = assetPayload
attachment.ContentType = contentType
messageAttachmentsToDownload[i] = attachment
assetCounter.Increase()
// Multiplying progress by `0.5` to leave 50% for saving assets to DB
// similar to how it's done for messages
progressValue := (float32(assetCounter.Value()) / float32(totalAssetsCount)) * 0.5
importProgress.UpdateTaskProgress(discord.DownloadAssetsTask, progressValue)
progressUpdates <- importProgress
messageAttachmentsToDownload[ii] = attachment
if m.DiscordImportMarkedAsCancelled(communityID) {
importProgress.StopTask(discord.DownloadAssetsTask)
@ -2901,6 +2915,11 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
cancel <- communityID
return
}
assetCounter.Increase()
progressValue := calculateProgress(i+1, totalImportChunkCount, 0.25+(float32(assetCounter.Value())/float32(totalAssetsCount))*0.25)
importProgress.UpdateTaskProgress(discord.DownloadAssetsTask, progressValue)
progressUpdates <- importProgress
}
}(attachments)
}
@ -2913,16 +2932,11 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
return
}
// Signal to the client that save operations are starting
importProgress.UpdateTaskState(discord.DownloadAssetsTask, discord.TaskStateSaving)
progressUpdates <- importProgress
// We chunk message attachments by `maxChunkSizeBytes` to ensure individual
// save operations don't take too long and block the database
attachmentChunks := chunkAttachmentsByByteSize(messageAttachmentsToDownload, maxChunkSizeBytes)
chunksCount = len(attachmentChunks)
for i, attachments := range attachmentChunks {
for ii, attachments := range attachmentChunks {
m.communitiesManager.LogStdout(fmt.Sprintf("saving %d/%d chunk with %d discord message attachments", ii+1, chunksCount, len(attachments)))
err = m.persistence.SaveDiscordMessageAttachments(attachments)
if err != nil {
m.cleanUpImport(communityID)
@ -2941,9 +2955,10 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
// 0.5 are the previous 50% of progress, hence we multiply our chunk progress
// by 0.5
currentCount := i + 1
progressValue := 0.5 + ((float32(currentCount) / float32(chunksCount)) * 0.5)
currentCount := ii + 1
progressValue := calculateProgress(i+1, totalImportChunkCount, 0.5+(float32(currentCount)/float32(chunksCount))*0.5)
importProgress.UpdateTaskProgress(discord.DownloadAssetsTask, progressValue)
progressUpdates <- importProgress
// We slow down the saving of attachment chunks to keep the database responsive
if currentCount < chunksCount {
@ -2951,8 +2966,58 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
}
}
importProgress.UpdateTaskProgress(discord.DownloadAssetsTask, 1)
progressUpdates <- importProgress
_, err := m.transport.JoinPublic(processedChannelIds[channel.Channel.ID])
if err != nil {
m.logger.Error("failed to load filter for chat", zap.Error(err))
continue
}
wakuChatMessages, err := m.chatMessagesToWakuMessages(messages, discordCommunity)
if err != nil {
m.logger.Error("failed to convert chat messages into waku messages", zap.Error(err))
continue
}
wakuPinMessages, err := m.pinMessagesToWakuMessages(pinMessagesToSave, discordCommunity)
if err != nil {
m.logger.Error("failed to convert pin messages into waku messages", zap.Error(err))
continue
}
wakuMessages := append(wakuChatMessages, wakuPinMessages...)
topics, err := m.communitiesManager.GetCommunityChatsTopics(discordCommunity.ID())
if err != nil {
m.logger.Error("failed to get community chat topics", zap.Error(err))
continue
}
startDate := time.Unix(int64(exportData.OldestMessageTimestamp), 0)
endDate := time.Now()
_, err = m.communitiesManager.CreateHistoryArchiveTorrentFromMessages(
discordCommunity.ID(),
wakuMessages,
topics,
startDate,
endDate,
messageArchiveInterval,
discordCommunity.Encrypted(),
)
if err != nil {
m.logger.Error("failed to create history archive torrent", zap.Error(err))
continue
}
if m.torrentClientReady() && communitySettings.HistoryArchiveSupportEnabled {
err = m.communitiesManager.SeedHistoryArchiveTorrent(discordCommunity.ID())
if err != nil {
m.logger.Error("failed to seed history archive", zap.Error(err))
}
go m.communitiesManager.StartHistoryArchiveTasksInterval(discordCommunity, messageArchiveInterval)
}
}
err = m.publishOrg(discordCommunity)
if err != nil {
@ -2980,6 +3045,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
progressUpdates <- importProgress
return
}
importProgress.UpdateTaskProgress(discord.InitCommunityTask, 0.15)
progressUpdates <- importProgress
@ -3009,12 +3075,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
return
}
filterChatIds := discordCommunity.DefaultFilters()
for _, chatID := range processedChannelIds {
filterChatIds = append(filterChatIds, chatID)
}
filters, err := m.transport.InitPublicFilters(filterChatIds)
_, err = m.transport.InitPublicFilters(discordCommunity.DefaultFilters())
if err != nil {
m.cleanUpImport(communityID)
importProgress.AddTaskError(discord.InitCommunityTask, discord.Error(err.Error()))
@ -3033,6 +3094,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
return
}
filters := m.transport.Filters()
_, err = m.scheduleSyncFilters(filters)
if err != nil {
m.cleanUpImport(communityID)
@ -3071,55 +3133,17 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor
m.config.messengerSignalsHandler.DiscordCommunityImportFinished(communityID)
close(done)
wakuChatMessages, err := m.chatMessagesToWakuMessages(messages, discordCommunity)
if err != nil {
m.logger.Error("failed to convert chat messages into waku messages", zap.Error(err))
return
}
wakuPinMessages, err := m.pinMessagesToWakuMessages(pinMessagesToSave, discordCommunity)
if err != nil {
m.logger.Error("failed to convert pin messages into waku messages", zap.Error(err))
return
}
wakuMessages := append(wakuChatMessages, wakuPinMessages...)
topics, err := m.communitiesManager.GetCommunityChatsTopics(discordCommunity.ID())
if err != nil {
m.logger.Error("failed to get community chat topics", zap.Error(err))
return
}
startDate := time.Unix(int64(exportData.OldestMessageTimestamp), 0)
endDate := time.Now()
_, err = m.communitiesManager.CreateHistoryArchiveTorrentFromMessages(
discordCommunity.ID(),
wakuMessages,
topics,
startDate,
endDate,
messageArchiveInterval,
discordCommunity.Encrypted(),
)
if err != nil {
m.logger.Error("failed to create history archive torrent", zap.Error(err))
return
}
if m.torrentClientReady() && communitySettings.HistoryArchiveSupportEnabled {
err = m.communitiesManager.SeedHistoryArchiveTorrent(discordCommunity.ID())
if err != nil {
m.logger.Error("failed to seed history archive", zap.Error(err))
}
go m.communitiesManager.StartHistoryArchiveTasksInterval(discordCommunity, messageArchiveInterval)
}
}()
}
func calculateProgress(i int, t int, currentProgress float32) float32 {
current := float32(1) / float32(t) * currentProgress
if i > 1 {
return float32(i-1)/float32(t) + current
}
return current
}
func (m *Messenger) MarkDiscordCommunityImportAsCancelled(communityID string) {
m.importingCommunities[communityID] = true
}

1
protocol/requests/create_community_category.go
View File

@ -13,6 +13,7 @@ type CreateCommunityCategory struct {
CommunityID types.HexBytes `json:"communityId"`
CategoryName string `json:"categoryName"`
ChatIDs []string `json:"chatIds"`
ThirdPartyID string `json:"thirdPartyID,omitempty"`
}
func (j *CreateCommunityCategory) Validate() error {