From 9be2a8d79944bebd0c613903f9c26738c9129717 Mon Sep 17 00:00:00 2001 From: Alex Jbanca Date: Thu, 3 Aug 2023 23:30:34 +0300 Subject: [PATCH] feat(Community Overview): Trigger chart data updates on specific actions + optimise the backend calls This commit includes the following changes: 1. Request from backend the messages count in a specific interval as opposed to all messages timestamps in that interval. 2. Update the chart end date before refreshing the data 3. Fix metrics type parsing in community service 4. Fix a bug where the new incoming data was not processed by ChartJs without a hover event on the chart. The fix here is to manually request paint() on model changes.d Issues found and not handled here: 1. On large communities the backend request can take 3 minutes to complete 2. CPU usage can easily go to 100% when switching chart tabs on large communities. All the requests are processed by the backend. --- .../modules/main/chat_section/controller.nim | 11 ++-- .../main/chat_section/io_interface.nim | 5 +- src/app/modules/main/chat_section/module.nim | 9 ++- src/app/modules/main/chat_section/view.nim | 5 +- .../service/community/dto/community.nim | 2 +- src/app_service/service/community/service.nim | 2 +- storybook/pages/OverviewSettingsChartPage.qml | 54 +++++++++++++++--- .../Components/private/chart/Chart.qml | 2 + ui/app/AppLayouts/Chat/stores/RootStore.qml | 4 ++ .../panels/OverviewSettingsChart.qml | 57 ++++++++++++++----- .../panels/OverviewSettingsPanel.qml | 8 +-- .../views/CommunitySettingsView.qml | 4 +- 12 files changed, 121 insertions(+), 42 deletions(-) diff --git a/src/app/modules/main/chat_section/controller.nim b/src/app/modules/main/chat_section/controller.nim index 74ab849280..9b2b7ed37b 100644 --- a/src/app/modules/main/chat_section/controller.nim +++ b/src/app/modules/main/chat_section/controller.nim @@ -173,11 +173,7 @@ proc init*(self: Controller) = let args = CommunityMetricsArgs(e) if args.communityId == self.sectionId: let metrics = self.communityService.getCommunityMetrics(args.communityId, args.metricsType) - var strings: seq[string] - for interval in metrics.intervals: - for timestamp in interval.timestamps: - strings.add($timestamp) - self.delegate.setOverviewChartData("[" & join(strings, ", ") & "]") + self.delegate.setCommunityMetrics(metrics) self.events.on(SIGNAL_COMMUNITY_CHANNEL_DELETED) do(e:Args): let args = CommunityChatIdArgs(e) @@ -664,4 +660,7 @@ proc getCommunityTokenList*(self: Controller): seq[CommunityTokenDto] = return self.communityTokensService.getCommunityTokens(self.getMySectionId()) proc collectCommunityMetricsMessagesTimestamps*(self: Controller, intervals: string) = - self.communityService.collectCommunityMetricsMessagesTimestamps(self.getMySectionId(), intervals) \ No newline at end of file + self.communityService.collectCommunityMetricsMessagesTimestamps(self.getMySectionId(), intervals) + +proc collectCommunityMetricsMessagesCount*(self: Controller, intervals: string) = + self.communityService.collectCommunityMetricsMessagesCount(self.getMySectionId(), intervals) \ No newline at end of file diff --git a/src/app/modules/main/chat_section/io_interface.nim b/src/app/modules/main/chat_section/io_interface.nim index 4839b94a29..52b935be21 100644 --- a/src/app/modules/main/chat_section/io_interface.nim +++ b/src/app/modules/main/chat_section/io_interface.nim @@ -349,7 +349,10 @@ method deleteCommunityTokenPermission*(self: AccessInterface, communityId: strin method collectCommunityMetricsMessagesTimestamps*(self: AccessInterface, intervals: string) {.base.} = raise newException(ValueError, "No implementation available") -method setOverviewChartData*(self: AccessInterface, metrics: string) {.base.} = +method collectCommunityMetricsMessagesCount*(self: AccessInterface, intervals: string) {.base.} = + raise newException(ValueError, "No implementation available") + +method setCommunityMetrics*(self: AccessInterface, metrics: CommunityMetricsDto) {.base.} = raise newException(ValueError, "No implementation available") method onCommunityTokenPermissionCreated*(self: AccessInterface, communityId: string, tokenPermission: CommunityTokenPermissionDto) {.base.} = diff --git a/src/app/modules/main/chat_section/module.nim b/src/app/modules/main/chat_section/module.nim index 6f1c5ec1ca..e7b84626f0 100644 --- a/src/app/modules/main/chat_section/module.nim +++ b/src/app/modules/main/chat_section/module.nim @@ -1,4 +1,4 @@ -import NimQml, Tables, chronicles, json, sequtils, strutils, strformat, sugar +import NimQml, Tables, chronicles, json, sequtils, strutils, strformat, sugar, marshal import io_interface import ../io_interface as delegate_interface @@ -1312,5 +1312,8 @@ method onDeactivateChatLoader*(self: Module, chatId: string) = method collectCommunityMetricsMessagesTimestamps*(self: Module, intervals: string) = self.controller.collectCommunityMetricsMessagesTimestamps(intervals) -method setOverviewChartData*(self: Module, metrics: string) = - self.view.setOverviewChartData(metrics) +method setCommunityMetrics*(self: Module, metrics: CommunityMetricsDto) = + self.view.setCommunityMetrics($$metrics) + +method collectCommunityMetricsMessagesCount*(self: Module, intervals: string) = + self.controller.collectCommunityMetricsMessagesCount(intervals) diff --git a/src/app/modules/main/chat_section/view.nim b/src/app/modules/main/chat_section/view.nim index 328ee218b0..ce960cda07 100644 --- a/src/app/modules/main/chat_section/view.nim +++ b/src/app/modules/main/chat_section/view.nim @@ -422,9 +422,12 @@ QtObject: read = getOverviewChartData notify = overviewChartDataChanged - proc setOverviewChartData*(self: View, communityMetrics: string) = + proc setCommunityMetrics*(self: View, communityMetrics: string) = self.communityMetrics = communityMetrics self.overviewChartDataChanged() proc collectCommunityMetricsMessagesTimestamps*(self: View, intervals: string) {.slot.} = self.delegate.collectCommunityMetricsMessagesTimestamps(intervals) + + proc collectCommunityMetricsMessagesCount*(self: View, intervals: string) {.slot.} = + self.delegate.collectCommunityMetricsMessagesCount(intervals) \ No newline at end of file diff --git a/src/app_service/service/community/dto/community.nim b/src/app_service/service/community/dto/community.nim index 2cc7af2ca9..bdc7db3289 100644 --- a/src/app_service/service/community/dto/community.nim +++ b/src/app_service/service/community/dto/community.nim @@ -338,7 +338,7 @@ proc toCommunityMetricsDto*(jsonObj: JsonNode): CommunityMetricsDto = result.metricsType = CommunityMetricsType.MessagesTimestamps var metricsTypeInt: int - if (jsonObj.getProp("metricsType", metricsTypeInt) and (metricsTypeInt >= ord(low(CommunityMetricsType)) and + if (jsonObj.getProp("type", metricsTypeInt) and (metricsTypeInt >= ord(low(CommunityMetricsType)) and metricsTypeInt <= ord(high(CommunityMetricsType)))): result.metricsType = CommunityMetricsType(metricsTypeInt) diff --git a/src/app_service/service/community/service.nim b/src/app_service/service/community/service.nim index e2464383c4..9285849dd2 100644 --- a/src/app_service/service/community/service.nim +++ b/src/app_service/service/community/service.nim @@ -1379,7 +1379,7 @@ QtObject: return let communityId = rpcResponseObj{"communityId"}.getStr() - let metricsType = rpcResponseObj{"metricsType"}.getInt() + let metricsType = rpcResponseObj{"response"}{"result"}{"type"}.getInt() var metrics = rpcResponseObj{"response"}{"result"}.toCommunityMetricsDto() self.communityMetrics[communityId] = metrics diff --git a/storybook/pages/OverviewSettingsChartPage.qml b/storybook/pages/OverviewSettingsChartPage.qml index fe0cc14404..9d4a483d4f 100644 --- a/storybook/pages/OverviewSettingsChartPage.qml +++ b/storybook/pages/OverviewSettingsChartPage.qml @@ -4,28 +4,66 @@ import QtQuick.Controls 2.15 import AppLayouts.Communities.panels 1.0 import Models 1.0 +import Storybook 1.0 + SplitView { + id: root + + orientation: Qt.Vertical OverviewSettingsChart { id: chart SplitView.fillWidth: true SplitView.fillHeight: true - - model: generateRandomModel() + onCollectCommunityMetricsMessagesCount: generateRandomModel(intervals) } - function generateRandomModel() { + function generateRandomModel(intervalsStr) { + if(!intervalsStr) return + + var response = { + communityId: "", + metricsType: timestampMetrics.checked ? "MessagesTimestamps" : "MessagesCount", + intervals: [] + } + + var intervals = JSON.parse(intervalsStr) + + response.intervals = intervals.map( x => { + var timestamps = generateRandomDate(x.startTimestamp, x.endTimestamp, Math.random() * 10) + + return { + startTimestamp: x.startTimestamp, + endTimestamp: x.endTimestamp, + timestamps: timestamps, + count: timestamps.length + } + }) + + chart.model = response + } + + function generateRandomDate(from, to, count) { var newModel = [] - const now = Date.now() - for(var i = 0; i < 500000; i++) { - var date = generateRandomDate(1463154962000, now) + for(var i = 0; i < count; i++) { + var date = from + Math.random() * (to - from) newModel.push(date) } return newModel } - function generateRandomDate(from, to) { - return from + Math.random() * (to - from) + LogsAndControlsPanel { + id: logsAndControlsPanel + + SplitView.minimumHeight: 100 + SplitView.preferredHeight: 150 + + CheckBox { + id: timestampMetrics + text: "Metrics using timestamps" + checked: false + onCheckedChanged: chart.reset() + } } } diff --git a/ui/StatusQ/src/StatusQ/Components/private/chart/Chart.qml b/ui/StatusQ/src/StatusQ/Components/private/chart/Chart.qml index 29bc1440d1..1a1b6e2a34 100644 --- a/ui/StatusQ/src/StatusQ/Components/private/chart/Chart.qml +++ b/ui/StatusQ/src/StatusQ/Components/private/chart/Chart.qml @@ -27,6 +27,8 @@ Canvas { function updateToNewData() { + if(!jsChart) return + jsChart.update('none'); root.requestPaint(); } diff --git a/ui/app/AppLayouts/Chat/stores/RootStore.qml b/ui/app/AppLayouts/Chat/stores/RootStore.qml index d0f33b66ec..6a2cadf16f 100644 --- a/ui/app/AppLayouts/Chat/stores/RootStore.qml +++ b/ui/app/AppLayouts/Chat/stores/RootStore.qml @@ -423,6 +423,10 @@ QtObject { chatCommunitySectionModule.collectCommunityMetricsMessagesTimestamps(intervals) } + function collectCommunityMetricsMessagesCount(intervals) { + chatCommunitySectionModule.collectCommunityMetricsMessagesCount(intervals) + } + function requestCommunityInfo(id, importing = false) { communitiesModuleInst.requestCommunityInfo(id, importing) } diff --git a/ui/app/AppLayouts/Communities/panels/OverviewSettingsChart.qml b/ui/app/AppLayouts/Communities/panels/OverviewSettingsChart.qml index bfe6878e61..56a6fc85ff 100644 --- a/ui/app/AppLayouts/Communities/panels/OverviewSettingsChart.qml +++ b/ui/app/AppLayouts/Communities/panels/OverviewSettingsChart.qml @@ -17,22 +17,22 @@ StatusChartPanel { /** * Flat model to use for the chart containing timestamps * type: {Array} + * default: [] + * example: {"communityId": "", "metricsType": "MessagesCount", "intervals": [{"startTimestamp": 1691047800000, "endTimestamp": 1691062200000, "timestamps": [], "count": 0}]} */ property var model: [] - signal collectCommunityMetricsMessagesTimestamps(var intervals) + signal collectCommunityMetricsMessagesCount(var intervals) - function requestCommunityMetrics() { - let intervals = d.selectedTabInfo.modelItems.map(item => { - return { - startTimestamp: item.start, - endTimestamp: item.end - } - }) - collectCommunityMetricsMessagesTimestamps(JSON.stringify(intervals)) + function reset() { + d.now = Date.now() + d.requestCommunityMetrics() } - onVisibleChanged: if (visible) requestCommunityMetrics() + onVisibleChanged: if(visible) d.resetWithSpamProtection() + onTimeRangeTabBarIndexChanged: reset() + onModelChanged: chart.updateToNewData() + onCollectCommunityMetricsMessagesCount: d.lastRequestModelMetadata = d.selectedTabInfo.modelItems QtObject { id: d @@ -49,7 +49,8 @@ StatusChartPanel { readonly property var hoveredModelMetadata: modelMetadata[root.timeRangeTabBarIndex].modelItems[hoveredBarIndex] readonly property var tooltipConfig: modelMetadata[root.timeRangeTabBarIndex].tooltipConfig readonly property var graphTabsModel: [{text: messagesLabel, enabled: true}] - readonly property var now: Date.now() + property var now: Date.now() + property var lastRequestModelMetadata: null readonly property var chartData: selectedTabInfo.modelItems.map(x => d.itemsCountInRange(root.model, x.start, x.end)) readonly property var labels: selectedTabInfo.modelItems.map(x => x.label) @@ -165,8 +166,35 @@ StatusChartPanel { } ] - function itemsCountInRange(array, start, end) { - return array ? array.filter(x => x <= end && x > start).length : 0 + function resetWithSpamProtection() { + if(Date.now() - d.now > LocaleUtils.minutesToMs(5) || d.lastRequestModelMetadata != selectedTabInfo.modelItems) { + root.reset() + } + } + + function requestCommunityMetrics() { + let intervals = d.selectedTabInfo.modelItems.map(item => { + return { + startTimestamp: item.start, + endTimestamp: item.end + } + }) + root.collectCommunityMetricsMessagesCount(JSON.stringify(intervals)) + } + + function itemsCountInRange(model, start, end) { + if (model == undefined || model.intervals == undefined) + return 0 + + const interval = model.intervals.find(x => x.startTimestamp == start && x.endTimestamp == end) + + if (!interval) + return 0 + + if(model.metricsType === "MessagesTimestamps") + return interval.timestamps.length + + return interval.count } function minutesStr(before = 0, timeReference = now, roundCurrentTime = true) { @@ -233,9 +261,8 @@ StatusChartPanel { return leftPositon ? Qt.point(relativeMousePoint.x - toolTip.width - 15, relativeMousePoint.y - 5) : Qt.point(relativeMousePoint.x + 15, relativeMousePoint.y - 5) } - - onSelectedTabInfoChanged: root.requestCommunityMetrics() } + headerLeftPadding: 0 headerBottomPadding: Style.current.bigPadding graphsModel: d.graphTabsModel diff --git a/ui/app/AppLayouts/Communities/panels/OverviewSettingsPanel.qml b/ui/app/AppLayouts/Communities/panels/OverviewSettingsPanel.qml index d74d2b43fd..2ab1885709 100644 --- a/ui/app/AppLayouts/Communities/panels/OverviewSettingsPanel.qml +++ b/ui/app/AppLayouts/Communities/panels/OverviewSettingsPanel.qml @@ -49,7 +49,7 @@ StackLayout { root.currentIndex = 0 } - signal collectCommunityMetricsMessagesTimestamps(var intervals) + signal collectCommunityMetricsMessagesCount(var intervals) signal edited(Item item) // item containing edited fields (name, description, logoImagePath, color, options, etc..) @@ -118,8 +118,8 @@ StackLayout { OverviewSettingsChart { model: JSON.parse(root.overviewChartData) - onCollectCommunityMetricsMessagesTimestamps: { - root.collectCommunityMetricsMessagesTimestamps(intervals) + onCollectCommunityMetricsMessagesCount: { + root.collectCommunityMetricsMessagesCount(intervals) } Layout.topMargin: 16 Layout.fillWidth: true @@ -128,7 +128,7 @@ StackLayout { Connections { target: root - onCommunityIdChanged: requestCommunityMetrics() + onCommunityIdChanged: reset() } } diff --git a/ui/app/AppLayouts/Communities/views/CommunitySettingsView.qml b/ui/app/AppLayouts/Communities/views/CommunitySettingsView.qml index a0030f2b5c..9be71c5eaf 100644 --- a/ui/app/AppLayouts/Communities/views/CommunitySettingsView.qml +++ b/ui/app/AppLayouts/Communities/views/CommunitySettingsView.qml @@ -178,8 +178,8 @@ StatusSectionLayout { communitySettingsDisabled: root.communitySettingsDisabled overviewChartData: rootStore.overviewChartData - onCollectCommunityMetricsMessagesTimestamps: { - rootStore.collectCommunityMetricsMessagesTimestamps(intervals) + onCollectCommunityMetricsMessagesCount: { + rootStore.collectCommunityMetricsMessagesCount(intervals) } onEdited: {