From 401402368d9075f93692d180cb30156785eed5a8 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Mon, 27 Jan 2025 10:44:59 +0100 Subject: [PATCH] chore: waku_archive add protection against queries longer than 24h (#3256) * store test adaptations because the tests were using future times * waku_store_sync.nim, test_waku_archive, test_rln_group_manager_onchain.nim nph (unrelated change) --- tests/node/test_wakunode_legacy_store.nim | 65 +++++------ tests/node/test_wakunode_store.nim | 107 ++++++++---------- tests/waku_archive/test_waku_archive.nim | 43 +++++-- .../test_rln_group_manager_onchain.nim | 1 - waku/waku_archive/archive.nim | 30 +++++ waku/waku_store_sync.nim | 4 +- 6 files changed, 143 insertions(+), 107 deletions(-) diff --git a/tests/node/test_wakunode_legacy_store.nim b/tests/node/test_wakunode_legacy_store.nim index 5b0409d86..b52dc6e6e 100644 --- a/tests/node/test_wakunode_legacy_store.nim +++ b/tests/node/test_wakunode_legacy_store.nim @@ -46,16 +46,16 @@ suite "Waku Store - End to End - Sorted Archive": let timeOrigin = now() archiveMessages = @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + fakeWakuMessage(@[byte 00], ts = ts(-90, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(-80, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(-70, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(-60, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(-50, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(-40, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(-30, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(-20, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(-10, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(00, timeOrigin)), ] historyQuery = HistoryQuery( @@ -657,23 +657,23 @@ suite "Waku Store - End to End - Archive with Multiple Topics": pageSize: 5, ) - let timeOrigin = now() + let timeOrigin = now() - 90 originTs = proc(offset = 0): Timestamp {.gcsafe, raises: [].} = ts(offset, timeOrigin) archiveMessages = @[ - fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic), - fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic), - fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic), - fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 00], ts = originTs(-90), contentTopic = contentTopic), + fakeWakuMessage(@[byte 01], ts = originTs(-80), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 02], ts = originTs(-70), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 03], ts = originTs(-60), contentTopic = contentTopic), + fakeWakuMessage(@[byte 04], ts = originTs(-50), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 05], ts = originTs(-40), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 06], ts = originTs(-30), contentTopic = contentTopic), + fakeWakuMessage(@[byte 07], ts = originTs(-20), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 08], ts = originTs(-10), contentTopic = contentTopicC), fakeWakuMessage( - @[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials + @[byte 09], ts = originTs(00), contentTopic = contentTopicSpecials ), ] @@ -827,8 +827,9 @@ suite "Waku Store - End to End - Archive with Multiple Topics": suite "Validation of Time-based Filtering": asyncTest "Basic Time Filtering": # Given a history query with start and end time - historyQuery.startTime = some(originTs(20)) - historyQuery.endTime = some(originTs(40)) + + historyQuery.startTime = some(originTs(-90)) + historyQuery.endTime = some(originTs(-70)) # When making a history query let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) @@ -836,12 +837,13 @@ suite "Waku Store - End to End - Archive with Multiple Topics": # Then the response contains the messages check: queryResponse.get().messages == - @[archiveMessages[2], archiveMessages[3], archiveMessages[4]] + @[archiveMessages[0], archiveMessages[1], archiveMessages[2]] asyncTest "Only Start Time Specified": # Given a history query with only start time - historyQuery.startTime = some(originTs(20)) + historyQuery.startTime = some(originTs(-20)) historyQuery.endTime = none(Timestamp) + historyQuery.pubsubTopic = none(string) # When making a history query let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) @@ -849,12 +851,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": # Then the response contains the messages check: queryResponse.get().messages == - @[ - archiveMessages[2], - archiveMessages[3], - archiveMessages[4], - archiveMessages[5], - ] + @[archiveMessages[7], archiveMessages[8], archiveMessages[9]] asyncTest "Only End Time Specified": # Given a history query with only end time @@ -889,8 +886,8 @@ suite "Waku Store - End to End - Archive with Multiple Topics": asyncTest "Time Filtering with Content Filtering": # Given a history query with time and content filtering - historyQuery.startTime = some(originTs(20)) - historyQuery.endTime = some(originTs(60)) + historyQuery.startTime = some(originTs(-90)) + historyQuery.endTime = some(originTs(-60)) historyQuery.contentTopics = @[contentTopicC] # When making a history query @@ -898,7 +895,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": # Then the response contains the messages check: - queryResponse.get().messages == @[archiveMessages[2], archiveMessages[5]] + queryResponse.get().messages == @[archiveMessages[2]] asyncTest "Messages Outside of Time Range": # Given a history query with a valid time range which does not contain any messages diff --git a/tests/node/test_wakunode_store.nim b/tests/node/test_wakunode_store.nim index 49c24c6d8..4442fb8fe 100644 --- a/tests/node/test_wakunode_store.nim +++ b/tests/node/test_wakunode_store.nim @@ -47,16 +47,16 @@ suite "Waku Store - End to End - Sorted Archive": let timeOrigin = now() let messages = @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + fakeWakuMessage(@[byte 00], ts = ts(-90, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(-80, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(-70, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(-60, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(-50, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(-40, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(-30, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(-20, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(-10, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(00, timeOrigin)), ] archiveMessages = messages.mapIt( WakuMessageKeyValue( @@ -909,17 +909,17 @@ suite "Waku Store - End to End - Archive with Multiple Topics": let messages = @[ - fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic), - fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic), - fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic), - fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 00], ts = originTs(-90), contentTopic = contentTopic), + fakeWakuMessage(@[byte 01], ts = originTs(-80), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 02], ts = originTs(-70), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 03], ts = originTs(-60), contentTopic = contentTopic), + fakeWakuMessage(@[byte 04], ts = originTs(-50), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 05], ts = originTs(-40), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 06], ts = originTs(-30), contentTopic = contentTopic), + fakeWakuMessage(@[byte 07], ts = originTs(-20), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 08], ts = originTs(-10), contentTopic = contentTopicC), fakeWakuMessage( - @[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials + @[byte 09], ts = originTs(00), contentTopic = contentTopicSpecials ), ] @@ -1089,44 +1089,14 @@ suite "Waku Store - End to End - Archive with Multiple Topics": suite "Validation of Time-based Filtering": asyncTest "Basic Time Filtering": # Given a history query with start and end time - storeQuery.startTime = some(originTs(20)) - storeQuery.endTime = some(originTs(40)) + storeQuery.startTime = some(originTs(-90)) + storeQuery.endTime = some(originTs(-60)) + storeQuery.contentTopics = @[contentTopic, contentTopicB, contentTopicC] # When making a history query let queryResponse = await client.query(storeQuery, serverRemotePeerInfo) # Then the response contains the messages - check: - queryResponse.get().messages == - @[archiveMessages[2], archiveMessages[3], archiveMessages[4]] - - asyncTest "Only Start Time Specified": - # Given a history query with only start time - storeQuery.startTime = some(originTs(20)) - storeQuery.endTime = none(Timestamp) - - # When making a history query - let queryResponse = await client.query(storeQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - archiveMessages[2], - archiveMessages[3], - archiveMessages[4], - archiveMessages[5], - ] - - asyncTest "Only End Time Specified": - # Given a history query with only end time - storeQuery.startTime = none(Timestamp) - storeQuery.endTime = some(originTs(40)) - - # When making a history query - let queryResponse = await client.query(storeQuery, serverRemotePeerInfo) - - # Then the response contains no messages check: queryResponse.get().messages == @[ @@ -1134,9 +1104,32 @@ suite "Waku Store - End to End - Archive with Multiple Topics": archiveMessages[1], archiveMessages[2], archiveMessages[3], - archiveMessages[4], ] + asyncTest "Only Start Time Specified": + # Given a history query with only start time + storeQuery.startTime = some(originTs(-40)) + storeQuery.endTime = none(Timestamp) + + # When making a history query + let queryResponse = await client.query(storeQuery, serverRemotePeerInfo) + + # Then the response contains the messages + check: + queryResponse.get().messages == @[archiveMessages[5]] + + asyncTest "Only End Time Specified": + # Given a history query with only end time + storeQuery.startTime = none(Timestamp) + storeQuery.endTime = some(originTs(-80)) + + # When making a history query + let queryResponse = await client.query(storeQuery, serverRemotePeerInfo) + + # Then the response contains no messages + check: + queryResponse.get().messages == @[archiveMessages[0], archiveMessages[1]] + asyncTest "Invalid Time Range": # Given a history query with invalid time range storeQuery.startTime = some(originTs(60)) @@ -1151,8 +1144,8 @@ suite "Waku Store - End to End - Archive with Multiple Topics": asyncTest "Time Filtering with Content Filtering": # Given a history query with time and content filtering - storeQuery.startTime = some(originTs(20)) - storeQuery.endTime = some(originTs(60)) + storeQuery.startTime = some(originTs(-60)) + storeQuery.endTime = some(originTs(-20)) storeQuery.contentTopics = @[contentTopicC] # When making a history query @@ -1160,7 +1153,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": # Then the response contains the messages check: - queryResponse.get().messages == @[archiveMessages[2], archiveMessages[5]] + queryResponse.get().messages == @[archiveMessages[5]] asyncTest "Messages Outside of Time Range": # Given a history query with a valid time range which does not contain any messages diff --git a/tests/waku_archive/test_waku_archive.nim b/tests/waku_archive/test_waku_archive.nim index 9e1b927e0..5162d310c 100644 --- a/tests/waku_archive/test_waku_archive.nim +++ b/tests/waku_archive/test_waku_archive.nim @@ -491,7 +491,8 @@ procSuite "Waku Archive - find messages": response.messages.mapIt(it.timestamp) == @[ts(30, timeOrigin), ts(50, timeOrigin)] test "handle temporal history query with a zero-size time window": - ## A zero-size window results in an empty list of history messages + ## A zero-size window results in an error to the store client. That kind of queries + ## are pointless and we need to rapidly inform about that to the client. ## Given let req = ArchiveQuery( contentTopics: @[ContentTopic("1")], @@ -503,27 +504,45 @@ procSuite "Waku Archive - find messages": let res = waitFor archiveA.findMessages(req) ## Then - check res.isOk() - - let response = res.tryGet() - check: - response.messages.len == 0 + check not res.isOk() test "handle temporal history query with an invalid time window": - ## A history query with an invalid time range results in an empty list of history messages + ## A query with an invalid time range should immediately return a query error to the client ## Given let req = ArchiveQuery( contentTopics: @[ContentTopic("1")], startTime: some(Timestamp(5)), - endTime: some(Timestamp(2)), + endTime: some(Timestamp(4)), ) ## When let res = waitFor archiveA.findMessages(req) ## Then - check res.isOk() + check not res.isOk() - let response = res.tryGet() - check: - response.messages.len == 0 + test "time range should be smaller than 24h": + let oneDayRangeNanos = 86_400_000_000_000 + let now = getNowInNanosecondTime() + + var res = waitFor archiveA.findMessages( + ArchiveQuery( + contentTopics: @[ContentTopic("1")], + startTime: some(Timestamp(now - oneDayRangeNanos - 1)), + endTime: some(Timestamp(now)), + ) + ) + + ## It fails if range is a bit bigger than 24h + check not res.isOk() + + res = waitFor archiveA.findMessages( + ArchiveQuery( + contentTopics: @[ContentTopic("1")], + startTime: some(Timestamp(now - oneDayRangeNanos)), + endTime: some(Timestamp(now)), + ) + ) + + ## Ok if range is 24h + check res.isOk() diff --git a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim index 7b4d6dbfe..5a338c499 100644 --- a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim +++ b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim @@ -128,7 +128,6 @@ suite "Onchain group manager": (await manager.startGroupSync()).isOkOr: raiseAssert $error - asyncTest "startGroupSync: should guard against uninitialized state": (await manager.startGroupSync()).isErrOr: raiseAssert "Expected error when not initialized" diff --git a/waku/waku_archive/archive.nim b/waku/waku_archive/archive.nim index c357d71d0..68234947f 100644 --- a/waku/waku_archive/archive.nim +++ b/waku/waku_archive/archive.nim @@ -146,11 +146,41 @@ proc syncMessageIngress*( return ok() +proc validateTimeRange( + startTime: Option[Timestamp], endTime: Option[Timestamp] +): Result[void, ArchiveError] = + ## Returns ok if the given time range is shorter than one day, and error otherwise. + ## We restrict the maximum allowed time of 24h to prevent excessive big queries. + + let oneDayRangeNanos = 86_400_000_000_000 + let now = getNowInNanosecondTime() + + var startTimeToValidate = now - oneDayRangeNanos + if startTime.isSome(): + startTimeToValidate = startTime.get() + + var endTimeToValidate = now + if endTime.isSome(): + endTimeToValidate = endTime.get() + + if startTimeToValidate >= endTimeToValidate: + return err(ArchiveError.invalidQuery("startTime should be before endTime")) + + if (endTimeToValidate - startTimeToValidate) > oneDayRangeNanos: + return err( + ArchiveError.invalidQuery("time range should be smaller than one day in nanos") + ) + + return ok() + proc findMessages*( self: WakuArchive, query: ArchiveQuery ): Future[ArchiveResult] {.async, gcsafe.} = ## Search the archive to return a single page of messages matching the query criteria + validateTimeRange(query.startTime, query.endTime).isOkOr: + return err(error) + if query.cursor.isSome(): let cursor = query.cursor.get() diff --git a/waku/waku_store_sync.nim b/waku/waku_store_sync.nim index 06699d9fd..03c1b33af 100644 --- a/waku/waku_store_sync.nim +++ b/waku/waku_store_sync.nim @@ -1,8 +1,6 @@ {.push raises: [].} import - ./waku_store_sync/reconciliation, - ./waku_store_sync/transfer, - ./waku_store_sync/common + ./waku_store_sync/reconciliation, ./waku_store_sync/transfer, ./waku_store_sync/common export reconciliation, transfer, common