From f54ba10bc70cab7669ca78c6f452e9aa4741ba69 Mon Sep 17 00:00:00 2001 From: Simon-Pierre Vivier Date: Fri, 12 Jul 2024 12:19:12 -0400 Subject: [PATCH] chore(archive): archive and drivers refactor (#2761) * queue driver refactor (#2753) * chore(archive): archive refactor (#2752) * chore(archive): sqlite driver refactor (#2754) * chore(archive): postgres driver refactor (#2755) * chore(archive): renaming & copies (#2751) * posgres legacy: stop using the storedAt field * migration script 6: we still need the id column The id column is needed because it contains the message digest which is used in store v2, and we need to keep support to store v2 for a while * legacy archive: set target migration version to 6 * waku_node: try to use wakuLegacyArchive if wakuArchive is nil * node_factory, waku_node: mount legacy and future store simultaneously We want the nwaku node to simultaneously support store-v2 requests and store-v3 requests. Only the legacy archive is in charge of archiving messages, and the archived information is suitable to fulfill both store-v2 and store-v3 needs. * postgres_driver: adding temporary code until store-v2 is removed --------- Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Co-authored-by: gabrielmer <101006718+gabrielmer@users.noreply.github.com> Co-authored-by: Ivan Folgueira Bande --- apps/chat2/chat2.nim | 21 +- .../content_script_version_6.nim | 12 + .../pg_migration_manager.nim | 3 +- tests/all_tests_waku.nim | 12 +- tests/node/test_wakunode_legacy_store.nim | 22 +- tests/testlib/postgres_legacy.nim | 27 + tests/waku_archive/archive_utils.nim | 17 +- tests/waku_archive/test_all.nim | 1 + tests/waku_archive/test_driver_postgres.nim | 68 +- .../test_driver_postgres_query.nim | 353 +-- tests/waku_archive/test_driver_queue.nim | 16 +- .../waku_archive/test_driver_queue_index.nim | 204 +- .../test_driver_queue_pagination.nim | 59 +- .../waku_archive/test_driver_queue_query.nim | 274 +-- tests/waku_archive/test_driver_sqlite.nim | 7 +- .../waku_archive/test_driver_sqlite_query.nim | 295 +-- tests/waku_archive/test_retention_policy.nim | 28 +- tests/waku_archive/test_waku_archive.nim | 40 +- tests/waku_archive_legacy/archive_utils.nim | 53 + tests/waku_archive_legacy/test_all.nim | 13 + .../test_driver_postgres.nim | 201 ++ .../test_driver_postgres_query.nim | 1931 +++++++++++++++++ .../waku_archive_legacy/test_driver_queue.nim | 182 ++ .../test_driver_queue_index.nim | 219 ++ .../test_driver_queue_pagination.nim | 405 ++++ .../test_driver_queue_query.nim | 1795 +++++++++++++++ .../test_driver_sqlite.nim | 60 + .../test_driver_sqlite_query.nim | 1875 ++++++++++++++++ .../test_retention_policy.nim | 169 ++ .../waku_archive_legacy/test_waku_archive.nim | 543 +++++ tests/waku_store/test_wakunode_store.nim | 7 +- tests/waku_store_legacy/test_resume.nim | 6 +- .../waku_store_legacy/test_wakunode_store.nim | 27 +- tests/wakunode_rest/test_rest_store.nim | 11 +- waku/factory/external_config.nim | 6 + waku/factory/node_factory.nim | 55 +- waku/node/waku_node.nim | 70 +- waku/waku_archive/archive.nim | 158 +- waku/waku_archive/common.nim | 35 +- waku/waku_archive/driver.nim | 23 +- .../driver/postgres_driver/migrations.nim | 2 +- .../postgres_driver/partitions_manager.nim | 2 +- .../postgres_driver/postgres_driver.nim | 772 ++++--- .../driver/queue_driver/index.nim | 81 +- .../driver/queue_driver/queue_driver.nim | 45 +- .../driver/sqlite_driver/queries.nim | 494 ++--- .../driver/sqlite_driver/sqlite_driver.nim | 81 +- waku/waku_archive_legacy.nim | 7 + waku/waku_archive_legacy/archive.nim | 323 +++ waku/waku_archive_legacy/archive_metrics.nim | 23 + waku/waku_archive_legacy/common.nim | 87 + waku/waku_archive_legacy/driver.nim | 119 + waku/waku_archive_legacy/driver/builder.nim | 125 ++ .../driver/postgres_driver.nim | 11 + .../driver/postgres_driver/migrations.nim | 89 + .../postgres_driver/partitions_manager.nim | 102 + .../postgres_driver/postgres_driver.nim | 1159 ++++++++++ .../postgres_driver/postgres_healthcheck.nim | 41 + .../driver/queue_driver.nim | 8 + .../driver/queue_driver/index.nim | 91 + .../driver/queue_driver/queue_driver.nim | 363 ++++ .../driver/sqlite_driver.nim | 8 + .../driver/sqlite_driver/cursor.nim | 5 +- .../driver/sqlite_driver/migrations.nim | 74 + .../driver/sqlite_driver/queries.nim | 744 +++++++ .../driver/sqlite_driver/sqlite_driver.nim | 225 ++ waku/waku_archive_legacy/retention_policy.nim | 16 + .../retention_policy/builder.nim | 88 + .../retention_policy_capacity.nim | 68 + .../retention_policy_size.nim | 27 + .../retention_policy_time.nim | 40 + 71 files changed, 12530 insertions(+), 2093 deletions(-) create mode 100644 migrations/message_store_postgres/content_script_version_6.nim create mode 100644 tests/testlib/postgres_legacy.nim create mode 100644 tests/waku_archive_legacy/archive_utils.nim create mode 100644 tests/waku_archive_legacy/test_all.nim create mode 100644 tests/waku_archive_legacy/test_driver_postgres.nim create mode 100644 tests/waku_archive_legacy/test_driver_postgres_query.nim create mode 100644 tests/waku_archive_legacy/test_driver_queue.nim create mode 100644 tests/waku_archive_legacy/test_driver_queue_index.nim create mode 100644 tests/waku_archive_legacy/test_driver_queue_pagination.nim create mode 100644 tests/waku_archive_legacy/test_driver_queue_query.nim create mode 100644 tests/waku_archive_legacy/test_driver_sqlite.nim create mode 100644 tests/waku_archive_legacy/test_driver_sqlite_query.nim create mode 100644 tests/waku_archive_legacy/test_retention_policy.nim create mode 100644 tests/waku_archive_legacy/test_waku_archive.nim create mode 100644 waku/waku_archive_legacy.nim create mode 100644 waku/waku_archive_legacy/archive.nim create mode 100644 waku/waku_archive_legacy/archive_metrics.nim create mode 100644 waku/waku_archive_legacy/common.nim create mode 100644 waku/waku_archive_legacy/driver.nim create mode 100644 waku/waku_archive_legacy/driver/builder.nim create mode 100644 waku/waku_archive_legacy/driver/postgres_driver.nim create mode 100644 waku/waku_archive_legacy/driver/postgres_driver/migrations.nim create mode 100644 waku/waku_archive_legacy/driver/postgres_driver/partitions_manager.nim create mode 100644 waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim create mode 100644 waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim create mode 100644 waku/waku_archive_legacy/driver/queue_driver.nim create mode 100644 waku/waku_archive_legacy/driver/queue_driver/index.nim create mode 100644 waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim create mode 100644 waku/waku_archive_legacy/driver/sqlite_driver.nim rename waku/{waku_archive => waku_archive_legacy}/driver/sqlite_driver/cursor.nim (67%) create mode 100644 waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim create mode 100644 waku/waku_archive_legacy/driver/sqlite_driver/queries.nim create mode 100644 waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim create mode 100644 waku/waku_archive_legacy/retention_policy.nim create mode 100644 waku/waku_archive_legacy/retention_policy/builder.nim create mode 100644 waku/waku_archive_legacy/retention_policy/retention_policy_capacity.nim create mode 100644 waku/waku_archive_legacy/retention_policy/retention_policy_size.nim create mode 100644 waku/waku_archive_legacy/retention_policy/retention_policy_time.nim diff --git a/apps/chat2/chat2.nim b/apps/chat2/chat2.nim index 573dd561f..dd2694cf7 100644 --- a/apps/chat2/chat2.nim +++ b/apps/chat2/chat2.nim @@ -44,6 +44,7 @@ import factory/builder, common/utils/nat, waku_relay, + waku_store/common, ], ./config_chat2 @@ -468,22 +469,30 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} = # We have a viable storenode. Let's query it for historical messages. echo "Connecting to storenode: " & $(storenode.get()) - node.mountLegacyStoreClient() - node.peerManager.addServicePeer(storenode.get(), WakuLegacyStoreCodec) + node.mountStoreClient() + node.peerManager.addServicePeer(storenode.get(), WakuStoreCodec) - proc storeHandler(response: HistoryResponse) {.gcsafe.} = + proc storeHandler(response: StoreQueryResponse) {.gcsafe.} = for msg in response.messages: + let payload = + if msg.message.isSome(): + msg.message.get().payload + else: + newSeq[byte](0) + let - pb = Chat2Message.init(msg.payload) + pb = Chat2Message.init(payload) chatLine = if pb.isOk: pb[].toString() else: - string.fromBytes(msg.payload) + string.fromBytes(payload) echo &"{chatLine}" info "Hit store handler" - let queryRes = await node.query(HistoryQuery(contentTopics: @[chat.contentTopic])) + let queryRes = await node.query( + StoreQueryRequest(contentTopics: @[chat.contentTopic]), storenode.get() + ) if queryRes.isOk(): storeHandler(queryRes.value) diff --git a/migrations/message_store_postgres/content_script_version_6.nim b/migrations/message_store_postgres/content_script_version_6.nim new file mode 100644 index 000000000..126ec6da1 --- /dev/null +++ b/migrations/message_store_postgres/content_script_version_6.nim @@ -0,0 +1,12 @@ +const ContentScriptVersion_6* = + """ +-- we can drop the timestamp column because this data is also kept in the storedAt column +ALTER TABLE messages DROP COLUMN timestamp; + +-- from now on we are only interested in the message timestamp +ALTER TABLE messages RENAME COLUMN storedAt TO timestamp; + +-- Update to new version +UPDATE version SET version = 6 WHERE version = 5; + +""" diff --git a/migrations/message_store_postgres/pg_migration_manager.nim b/migrations/message_store_postgres/pg_migration_manager.nim index 86f6fcb27..e90a51fc2 100644 --- a/migrations/message_store_postgres/pg_migration_manager.nim +++ b/migrations/message_store_postgres/pg_migration_manager.nim @@ -1,6 +1,6 @@ import content_script_version_1, content_script_version_2, content_script_version_3, - content_script_version_4, content_script_version_5 + content_script_version_4, content_script_version_5, content_script_version_6 type MigrationScript* = object version*: int @@ -16,6 +16,7 @@ const PgMigrationScripts* = MigrationScript(version: 3, scriptContent: ContentScriptVersion_3), MigrationScript(version: 4, scriptContent: ContentScriptVersion_4), MigrationScript(version: 5, scriptContent: ContentScriptVersion_5), + MigrationScript(version: 6, scriptContent: ContentScriptVersion_6), ] proc getMigrationScripts*(currentVersion: int64, targetVersion: int64): seq[string] = diff --git a/tests/all_tests_waku.nim b/tests/all_tests_waku.nim index f5caf08a1..004f3e58a 100644 --- a/tests/all_tests_waku.nim +++ b/tests/all_tests_waku.nim @@ -18,7 +18,15 @@ import ./waku_archive/test_driver_sqlite, ./waku_archive/test_retention_policy, ./waku_archive/test_waku_archive, - ./waku_archive/test_partition_manager + ./waku_archive/test_partition_manager, + ./waku_archive_legacy/test_driver_queue_index, + ./waku_archive_legacy/test_driver_queue_pagination, + ./waku_archive_legacy/test_driver_queue_query, + ./waku_archive_legacy/test_driver_queue, + ./waku_archive_legacy/test_driver_sqlite_query, + ./waku_archive_legacy/test_driver_sqlite, + ./waku_archive_legacy/test_retention_policy, + ./waku_archive_legacy/test_waku_archive const os* {.strdefine.} = "" when os == "Linux" and @@ -28,6 +36,8 @@ when os == "Linux" and import ./waku_archive/test_driver_postgres_query, ./waku_archive/test_driver_postgres, + #./waku_archive_legacy/test_driver_postgres_query, + #./waku_archive_legacy/test_driver_postgres, ./factory/test_node_factory, ./wakunode_rest/test_rest_store diff --git a/tests/node/test_wakunode_legacy_store.nim b/tests/node/test_wakunode_legacy_store.nim index 7c672521e..5b0409d86 100644 --- a/tests/node/test_wakunode_legacy_store.nim +++ b/tests/node/test_wakunode_legacy_store.nim @@ -15,12 +15,12 @@ import waku_core, waku_store_legacy, waku_store_legacy/client, - waku_archive, - waku_archive/driver/sqlite_driver, + waku_archive_legacy, + waku_archive_legacy/driver/sqlite_driver, common/databases/db_sqlite, ], ../waku_store_legacy/store_utils, - ../waku_archive/archive_utils, + ../waku_archive_legacy/archive_utils, ../testlib/[common, wakucore, wakunode, testasync, futures, testutils] suite "Waku Store - End to End - Sorted Archive": @@ -73,7 +73,7 @@ suite "Waku Store - End to End - Sorted Archive": client = newTestWakuNode(clientKey, ValidIpAddress.init("0.0.0.0"), Port(0)) archiveDriver = newArchiveDriverWithMessages(pubsubTopic, archiveMessages) - let mountArchiveResult = server.mountArchive(archiveDriver) + let mountArchiveResult = server.mountLegacyArchive(archiveDriver) assert mountArchiveResult.isOk() await server.mountLegacyStore() @@ -445,7 +445,7 @@ suite "Waku Store - End to End - Sorted Archive": otherServer = newTestWakuNode(otherServerKey, ValidIpAddress.init("0.0.0.0"), Port(0)) mountOtherArchiveResult = - otherServer.mountArchive(otherArchiveDriverWithMessages) + otherServer.mountLegacyArchive(otherArchiveDriverWithMessages) assert mountOtherArchiveResult.isOk() await otherServer.mountLegacyStore() @@ -532,7 +532,7 @@ suite "Waku Store - End to End - Unsorted Archive": unsortedArchiveDriverWithMessages = newArchiveDriverWithMessages(pubsubTopic, unsortedArchiveMessages) mountUnsortedArchiveResult = - server.mountArchive(unsortedArchiveDriverWithMessages) + server.mountLegacyArchive(unsortedArchiveDriverWithMessages) assert mountUnsortedArchiveResult.isOk() @@ -687,7 +687,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": let archiveDriver = newSqliteArchiveDriver() .put(pubsubTopic, archiveMessages[0 ..< 6]) .put(pubsubTopicB, archiveMessages[6 ..< 10]) - let mountSortedArchiveResult = server.mountArchive(archiveDriver) + let mountSortedArchiveResult = server.mountLegacyArchive(archiveDriver) assert mountSortedArchiveResult.isOk() @@ -932,7 +932,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": ephemeralServer = newTestWakuNode(ephemeralServerKey, ValidIpAddress.init("0.0.0.0"), Port(0)) mountEphemeralArchiveResult = - ephemeralServer.mountArchive(ephemeralArchiveDriver) + ephemeralServer.mountLegacyArchive(ephemeralArchiveDriver) assert mountEphemeralArchiveResult.isOk() await ephemeralServer.mountLegacyStore() @@ -974,7 +974,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": mixedServerKey = generateSecp256k1Key() mixedServer = newTestWakuNode(mixedServerKey, ValidIpAddress.init("0.0.0.0"), Port(0)) - mountMixedArchiveResult = mixedServer.mountArchive(mixedArchiveDriver) + mountMixedArchiveResult = mixedServer.mountLegacyArchive(mixedArchiveDriver) assert mountMixedArchiveResult.isOk() await mixedServer.mountLegacyStore() @@ -1001,7 +1001,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": emptyServerKey = generateSecp256k1Key() emptyServer = newTestWakuNode(emptyServerKey, ValidIpAddress.init("0.0.0.0"), Port(0)) - mountEmptyArchiveResult = emptyServer.mountArchive(emptyArchiveDriver) + mountEmptyArchiveResult = emptyServer.mountLegacyArchive(emptyArchiveDriver) assert mountEmptyArchiveResult.isOk() await emptyServer.mountLegacyStore() @@ -1033,7 +1033,7 @@ suite "Waku Store - End to End - Archive with Multiple Topics": voluminousServer = newTestWakuNode(voluminousServerKey, ValidIpAddress.init("0.0.0.0"), Port(0)) mountVoluminousArchiveResult = - voluminousServer.mountArchive(voluminousArchiveDriverWithMessages) + voluminousServer.mountLegacyArchive(voluminousArchiveDriverWithMessages) assert mountVoluminousArchiveResult.isOk() await voluminousServer.mountLegacyStore() diff --git a/tests/testlib/postgres_legacy.nim b/tests/testlib/postgres_legacy.nim new file mode 100644 index 000000000..50988c6c8 --- /dev/null +++ b/tests/testlib/postgres_legacy.nim @@ -0,0 +1,27 @@ +import chronicles, chronos +import + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver as driver_module, + waku/waku_archive_legacy/driver/builder, + waku/waku_archive_legacy/driver/postgres_driver + +const storeMessageDbUrl = "postgres://postgres:test123@localhost:5432/postgres" + +proc newTestPostgresDriver*(): Future[Result[ArchiveDriver, string]] {. + async, deprecated +.} = + proc onErr(errMsg: string) {.gcsafe, closure.} = + error "error creating ArchiveDriver", error = errMsg + quit(QuitFailure) + + let + vacuum = false + migrate = true + maxNumConn = 50 + + let driverRes = + await ArchiveDriver.new(storeMessageDbUrl, vacuum, migrate, maxNumConn, onErr) + if driverRes.isErr(): + onErr("could not create archive driver: " & driverRes.error) + + return ok(driverRes.get()) diff --git a/tests/waku_archive/archive_utils.nim b/tests/waku_archive/archive_utils.nim index 454150dd9..affca9f78 100644 --- a/tests/waku_archive/archive_utils.nim +++ b/tests/waku_archive/archive_utils.nim @@ -23,26 +23,11 @@ proc newSqliteArchiveDriver*(): ArchiveDriver = proc newWakuArchive*(driver: ArchiveDriver): WakuArchive = WakuArchive.new(driver).get() -proc computeArchiveCursor*( - pubsubTopic: PubsubTopic, message: WakuMessage -): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - proc put*( driver: ArchiveDriver, pubsubTopic: PubSubTopic, msgList: seq[WakuMessage] ): ArchiveDriver = for msg in msgList: - let - msgDigest = computeDigest(msg) - msgHash = computeMessageHash(pubsubTopic, msg) - _ = waitFor driver.put(pubsubTopic, msg, msgDigest, msgHash, msg.timestamp) - # discard crashes + let _ = waitFor driver.put(computeMessageHash(pubsubTopic, msg), pubsubTopic, msg) return driver proc newArchiveDriverWithMessages*( diff --git a/tests/waku_archive/test_all.nim b/tests/waku_archive/test_all.nim index 9d45d99a1..9b97cc22d 100644 --- a/tests/waku_archive/test_all.nim +++ b/tests/waku_archive/test_all.nim @@ -9,5 +9,6 @@ import ./test_driver_queue, ./test_driver_sqlite_query, ./test_driver_sqlite, + ./test_partition_manager, ./test_retention_policy, ./test_waku_archive diff --git a/tests/waku_archive/test_driver_postgres.nim b/tests/waku_archive/test_driver_postgres.nim index ef03e491c..7b808c14d 100644 --- a/tests/waku_archive/test_driver_postgres.nim +++ b/tests/waku_archive/test_driver_postgres.nim @@ -12,15 +12,6 @@ import ../testlib/testasync, ../testlib/postgres -proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - suite "Postgres driver": ## Unique driver instance var driver {.threadvar.}: PostgresDriver @@ -60,11 +51,8 @@ suite "Postgres driver": let msg = fakeWakuMessage(contentTopic = contentTopic, meta = meta) - let computedDigest = computeDigest(msg) - let computedHash = computeMessageHash(DefaultPubsubTopic, msg) - let putRes = await driver.put( - DefaultPubsubTopic, msg, computedDigest, computedHash, msg.timestamp + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) assert putRes.isOk(), putRes.error @@ -72,12 +60,10 @@ suite "Postgres driver": assert storedMsg.len == 1 - let (pubsubTopic, actualMsg, digest, _, hash) = storedMsg[0] + let (_, pubsubTopic, actualMsg) = storedMsg[0] assert actualMsg.contentTopic == contentTopic assert pubsubTopic == DefaultPubsubTopic - assert toHex(computedDigest.data) == toHex(digest) assert toHex(actualMsg.payload) == toHex(msg.payload) - assert toHex(computedHash) == toHex(hash) assert toHex(actualMsg.meta) == toHex(msg.meta) asyncTest "Insert and query message": @@ -88,24 +74,14 @@ suite "Postgres driver": let msg1 = fakeWakuMessage(contentTopic = contentTopic1) - var putRes = await driver.put( - pubsubTopic1, - msg1, - computeDigest(msg1), - computeMessageHash(pubsubTopic1, msg1), - msg1.timestamp, - ) + var putRes = + await driver.put(computeMessageHash(pubsubTopic1, msg1), pubsubTopic1, msg1) assert putRes.isOk(), putRes.error let msg2 = fakeWakuMessage(contentTopic = contentTopic2) - putRes = await driver.put( - pubsubTopic2, - msg2, - computeDigest(msg2), - computeMessageHash(pubsubTopic2, msg2), - msg2.timestamp, - ) + putRes = + await driver.put(computeMessageHash(pubsubTopic2, msg2), pubsubTopic2, msg2) assert putRes.isOk(), putRes.error let countMessagesRes = await driver.getMessagesCount() @@ -113,49 +89,49 @@ suite "Postgres driver": assert countMessagesRes.isOk(), $countMessagesRes.error assert countMessagesRes.get() == 2 - var messagesRes = await driver.getMessages(contentTopic = @[contentTopic1]) + var messagesRes = await driver.getMessages(contentTopics = @[contentTopic1]) assert messagesRes.isOk(), $messagesRes.error assert messagesRes.get().len == 1 # Get both content topics, check ordering messagesRes = - await driver.getMessages(contentTopic = @[contentTopic1, contentTopic2]) + await driver.getMessages(contentTopics = @[contentTopic1, contentTopic2]) assert messagesRes.isOk(), messagesRes.error assert messagesRes.get().len == 2 - assert messagesRes.get()[0][1].contentTopic == contentTopic1 + assert messagesRes.get()[0][2].contentTopic == contentTopic1 # Descending order messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], ascendingOrder = false + contentTopics = @[contentTopic1, contentTopic2], ascendingOrder = false ) assert messagesRes.isOk(), messagesRes.error assert messagesRes.get().len == 2 - assert messagesRes.get()[0][1].contentTopic == contentTopic2 + assert messagesRes.get()[0][2].contentTopic == contentTopic2 # cursor # Get both content topics messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], - cursor = some(computeTestCursor(pubsubTopic1, messagesRes.get()[1][1])), + contentTopics = @[contentTopic1, contentTopic2], + cursor = some(computeMessageHash(pubsubTopic1, messagesRes.get()[1][2])), ) assert messagesRes.isOk() assert messagesRes.get().len == 1 # Get both content topics but one pubsub topic messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], pubsubTopic = some(pubsubTopic1) + contentTopics = @[contentTopic1, contentTopic2], pubsubTopic = some(pubsubTopic1) ) assert messagesRes.isOk(), messagesRes.error assert messagesRes.get().len == 1 - assert messagesRes.get()[0][1].contentTopic == contentTopic1 + assert messagesRes.get()[0][2].contentTopic == contentTopic1 # Limit messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], maxPageSize = 1 + contentTopics = @[contentTopic1, contentTopic2], maxPageSize = 1 ) assert messagesRes.isOk(), messagesRes.error assert messagesRes.get().len == 1 @@ -172,11 +148,7 @@ suite "Postgres driver": raiseAssert "could not get num mgs correctly: " & $error var putRes = await driver.put( - DefaultPubsubTopic, - msg1, - computeDigest(msg1), - computeMessageHash(DefaultPubsubTopic, msg1), - msg1.timestamp, + computeMessageHash(DefaultPubsubTopic, msg1), DefaultPubsubTopic, msg1 ) assert putRes.isOk(), putRes.error @@ -187,11 +159,7 @@ suite "Postgres driver": "wrong number of messages: " & $newNumMsgs putRes = await driver.put( - DefaultPubsubTopic, - msg2, - computeDigest(msg2), - computeMessageHash(DefaultPubsubTopic, msg2), - msg2.timestamp, + computeMessageHash(DefaultPubsubTopic, msg2), DefaultPubsubTopic, msg2 ) assert putRes.isOk() diff --git a/tests/waku_archive/test_driver_postgres_query.nim b/tests/waku_archive/test_driver_postgres_query.nim index d429df7ac..15c3e2c97 100644 --- a/tests/waku_archive/test_driver_postgres_query.nim +++ b/tests/waku_archive/test_driver_postgres_query.nim @@ -27,30 +27,21 @@ logScope: # Initialize the random number generator common.randomize() -proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - suite "Postgres driver - queries": ## Unique driver instance var driver {.threadvar.}: PostgresDriver asyncSetup: let driverRes = await newTestPostgresDriver() - if driverRes.isErr(): - assert false, driverRes.error + + assert driverRes.isOk(), $driverRes.error driver = PostgresDriver(driverRes.get()) asyncTeardown: let resetRes = await driver.reset() - if resetRes.isErr(): - assert false, resetRes.error + + assert resetRes.isOk(), $resetRes.error (await driver.close()).expect("driver to close") @@ -75,15 +66,10 @@ suite "Postgres driver - queries": debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() + let putRes = await driver.put( + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg + ) + assert putRes.isOk(), $putRes.error ## When let res = await driver.getMessages(maxPageSize = 5, ascendingOrder = true) @@ -91,7 +77,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[0 .. 4] @@ -118,23 +104,19 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3] @@ -173,23 +155,19 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3] @@ -216,23 +194,19 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = false ) ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[6 .. 7].reversed() @@ -261,17 +235,13 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When var res = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], + contentTopics = @[contentTopic1, contentTopic2], pubsubTopic = some(DefaultPubsubTopic), maxPageSize = 2, ascendingOrder = true, @@ -281,14 +251,14 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - var filteredMessages = res.tryGet().mapIt(it[1]) + var filteredMessages = res.tryGet().mapIt(it[2]) check filteredMessages == expected[2 .. 3] ## When ## This is very similar to the previous one but we enforce to use the prepared ## statement by querying one single content topic res = await driver.getMessages( - contentTopic = @[contentTopic1], + contentTopics = @[contentTopic1], pubsubTopic = some(DefaultPubsubTopic), maxPageSize = 2, ascendingOrder = true, @@ -298,7 +268,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - filteredMessages = res.tryGet().mapIt(it[1]) + filteredMessages = res.tryGet().mapIt(it[2]) check filteredMessages == @[expected[2]] asyncTest "single content topic - no results": @@ -321,23 +291,19 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 @@ -349,17 +315,13 @@ suite "Postgres driver - queries": let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[DefaultContentTopic], + contentTopics = @[DefaultContentTopic], maxPageSize = pageSize, ascendingOrder = true, ) @@ -367,7 +329,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 40 @@ -413,11 +375,7 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages( @@ -428,7 +386,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -474,11 +432,7 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages(maxPageSize = 2, ascendingOrder = true) @@ -487,7 +441,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[0 .. 1] @@ -533,15 +487,11 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true, @@ -551,7 +501,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -579,15 +529,11 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = await driver.getMessages( @@ -597,7 +543,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[5 .. 6] @@ -625,15 +571,11 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = await driver.getMessages( @@ -643,7 +585,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3].reversed() @@ -669,21 +611,16 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) - let cursor = ArchiveCursor(hash: fakeCursor) + let cursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) ## When let res = await driver.getMessages( includeData = true, - contentTopicSeq = @[DefaultContentTopic], + contentTopics = @[DefaultContentTopic], pubsubTopic = none(PubsubTopic), cursor = some(cursor), startTime = none(Timestamp), @@ -694,10 +631,10 @@ suite "Postgres driver - queries": ) ## Then - assert res.isOk(), res.error + assert res.isErr(), $res.value check: - res.value.len == 0 + res.error == "cursor not found" asyncTest "content topic and cursor": ## Given @@ -723,19 +660,15 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), maxPageSize = 10, ascendingOrder = true, @@ -744,7 +677,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[5 .. 6] @@ -772,19 +705,15 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[6]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), maxPageSize = 10, ascendingOrder = false, @@ -793,7 +722,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 5].reversed() @@ -864,13 +793,9 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeTestCursor(expected[5][0], expected[5][1]) + let cursor = computeMessageHash(expected[5][0], expected[5][1]) ## When let res = await driver.getMessages( @@ -884,7 +809,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[6 .. 7] @@ -955,13 +880,9 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeTestCursor(expected[6][0], expected[6][1]) + let cursor = computeMessageHash(expected[6][0], expected[6][1]) ## When let res = await driver.getMessages( @@ -975,7 +896,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5].reversed() @@ -1003,11 +924,7 @@ suite "Postgres driver - queries": let hashes = messages.mapIt(computeMessageHash(DefaultPubsubTopic, it)) for (msg, hash) in messages.zip(hashes): - require ( - await driver.put( - DefaultPubsubTopic, msg, computeDigest(msg), hash, msg.timestamp - ) - ).isOk() + require (await driver.put(hash, DefaultPubsubTopic, msg)).isOk() ## When let res = await driver.getMessages(hashes = hashes, ascendingOrder = false) @@ -1016,7 +933,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.reversed() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages @@ -1044,11 +961,7 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -1060,7 +973,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6] @@ -1088,11 +1001,7 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -1104,7 +1013,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[0 .. 4] @@ -1177,11 +1086,7 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages( @@ -1195,7 +1100,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[2 .. 4] @@ -1224,17 +1129,13 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(45, timeOrigin)), endTime = some(ts(15, timeOrigin)), maxPageSize = 2, @@ -1243,7 +1144,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 @@ -1271,17 +1172,13 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true, @@ -1289,7 +1186,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6] @@ -1320,17 +1217,13 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = false, @@ -1338,7 +1231,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6].reversed() @@ -1370,19 +1263,15 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[3]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[3]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), startTime = some(ts(15, timeOrigin)), maxPageSize = 10, @@ -1391,7 +1280,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[4 .. 9] @@ -1423,19 +1312,15 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[6]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), startTime = some(ts(15, timeOrigin)), maxPageSize = 10, @@ -1444,7 +1329,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[3 .. 4].reversed() @@ -1508,17 +1393,13 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[1][1]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[1][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(0, timeOrigin)), @@ -1530,7 +1411,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[3 .. 4] @@ -1593,17 +1474,13 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeTestCursor(expected[7][0], expected[7][1]) + let cursor = computeMessageHash(expected[7][0], expected[7][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1615,7 +1492,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5].reversed() @@ -1679,17 +1556,13 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeTestCursor(expected[1][0], expected[1][1]) + let cursor = computeMessageHash(expected[1][0], expected[1][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1702,7 +1575,7 @@ suite "Postgres driver - queries": assert res.isOk(), res.error let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -1766,17 +1639,13 @@ suite "Postgres driver - queries": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeTestCursor(expected[1][0], expected[1][1]) + let cursor = computeMessageHash(expected[1][0], expected[1][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1788,7 +1657,7 @@ suite "Postgres driver - queries": ## Then assert res.isOk(), res.error - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 @@ -1816,11 +1685,7 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -1867,11 +1732,7 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -1908,11 +1769,7 @@ suite "Postgres driver - queries": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() diff --git a/tests/waku_archive/test_driver_queue.nim b/tests/waku_archive/test_driver_queue.nim index 6ef56520c..16c0163c7 100644 --- a/tests/waku_archive/test_driver_queue.nim +++ b/tests/waku_archive/test_driver_queue.nim @@ -19,13 +19,11 @@ proc genIndexedWakuMessage(i: int8): (Index, WakuMessage) = let message = WakuMessage(payload: @[byte i], timestamp: Timestamp(i)) - topic = "test-pubsub-topic" + pubsubTopic = "test-pubsub-topic" cursor = Index( - receiverTime: Timestamp(i), - senderTime: Timestamp(i), - digest: MessageDigest(data: data), - pubsubTopic: topic, - hash: computeMessageHash(topic, message), + time: Timestamp(i), + hash: computeMessageHash(pubsubTopic, message), + pubsubTopic: pubsubTopic, ) (cursor, message) @@ -72,7 +70,7 @@ procSuite "Sorted driver queue": # Attempt to add message with older value than oldest in queue should fail let - oldestTimestamp = driver.first().get().senderTime + oldestTimestamp = driver.first().get().time (index, message) = genIndexedWakuMessage(oldestTimestamp.int8 - 1) addRes = driver.add(index, message) @@ -121,7 +119,7 @@ procSuite "Sorted driver queue": let first = firstRes.tryGet() check: - first.senderTime == Timestamp(1) + first.time == Timestamp(1) test "get first item from empty queue should fail": ## Given @@ -152,7 +150,7 @@ procSuite "Sorted driver queue": let last = lastRes.tryGet() check: - last.senderTime == Timestamp(5) + last.time == Timestamp(5) test "get last item from empty queue should fail": ## Given diff --git a/tests/waku_archive/test_driver_queue_index.nim b/tests/waku_archive/test_driver_queue_index.nim index 214a67d22..c383a676c 100644 --- a/tests/waku_archive/test_driver_queue_index.nim +++ b/tests/waku_archive/test_driver_queue_index.nim @@ -7,20 +7,6 @@ var rng = initRand() ## Helpers -proc getTestTimestamp(offset = 0): Timestamp = - let now = getNanosecondTime(epochTime() + float(offset)) - Timestamp(now) - -proc hashFromStr(input: string): MDigest[256] = - var ctx: sha256 - - ctx.init() - ctx.update(input.toBytes()) - let hashed = ctx.finish() - ctx.clear() - - return hashed - proc randomHash(): WakuMessageHash = var hash: WakuMessageHash @@ -33,187 +19,29 @@ proc randomHash(): WakuMessageHash = suite "Queue Driver - index": ## Test vars let - smallIndex1 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - hash: randomHash(), - ) - smallIndex2 = Index( - digest: hashFromStr("1234567"), # digest is less significant than senderTime - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - hash: randomHash(), - ) - largeIndex1 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(9000), - hash: randomHash(), - ) # only senderTime differ from smallIndex1 - largeIndex2 = Index( - digest: hashFromStr("12345"), # only digest differs from smallIndex1 - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - hash: randomHash(), - ) - eqIndex1 = Index( - digest: hashFromStr("0003"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(54321), - hash: randomHash(), - ) - eqIndex2 = Index( - digest: hashFromStr("0003"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(54321), - hash: randomHash(), - ) - eqIndex3 = Index( - digest: hashFromStr("0003"), - receiverTime: getNanosecondTime(9999), - # receiverTime difference should have no effect on comparisons - senderTime: getNanosecondTime(54321), - hash: randomHash(), - ) - diffPsTopic = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - pubsubTopic: "zzzz", - hash: randomHash(), - ) - noSenderTime1 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(1100), - senderTime: getNanosecondTime(0), - pubsubTopic: "zzzz", - hash: randomHash(), - ) - noSenderTime2 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(10000), - senderTime: getNanosecondTime(0), - pubsubTopic: "zzzz", - hash: randomHash(), - ) - noSenderTime3 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(1200), - senderTime: getNanosecondTime(0), - pubsubTopic: "aaaa", - hash: randomHash(), - ) - noSenderTime4 = Index( - digest: hashFromStr("0"), - receiverTime: getNanosecondTime(1200), - senderTime: getNanosecondTime(0), - pubsubTopic: "zzzz", - hash: randomHash(), - ) + hash = randomHash() + eqIndex1 = Index(time: getNanosecondTime(54321), hash: hash) + eqIndex2 = Index(time: getNanosecondTime(54321), hash: hash) + eqIndex3 = Index(time: getNanosecondTime(54321), hash: randomHash()) + eqIndex4 = Index(time: getNanosecondTime(65432), hash: hash) test "Index comparison": - # Index comparison with senderTime diff - check: - cmp(smallIndex1, largeIndex1) < 0 - cmp(smallIndex2, largeIndex1) < 0 - - # Index comparison with digest diff - check: - cmp(smallIndex1, smallIndex2) < 0 - cmp(smallIndex1, largeIndex2) < 0 - cmp(smallIndex2, largeIndex2) > 0 - cmp(largeIndex1, largeIndex2) > 0 - - # Index comparison when equal check: + # equality cmp(eqIndex1, eqIndex2) == 0 + cmp(eqIndex1, eqIndex3) != 0 + cmp(eqIndex1, eqIndex4) != 0 - # pubsubTopic difference - check: - cmp(smallIndex1, diffPsTopic) < 0 + # ordering + cmp(eqIndex3, eqIndex4) < 0 + cmp(eqIndex4, eqIndex3) > 0 # Test symmetry - # receiverTime diff plays no role when senderTime set - check: - cmp(eqIndex1, eqIndex3) == 0 - - # receiverTime diff plays no role when digest/pubsubTopic equal - check: - cmp(noSenderTime1, noSenderTime2) == 0 - - # sort on receiverTime with no senderTimestamp and unequal pubsubTopic - check: - cmp(noSenderTime1, noSenderTime3) < 0 - - # sort on receiverTime with no senderTimestamp and unequal digest - check: - cmp(noSenderTime1, noSenderTime4) < 0 - - # sort on receiverTime if no senderTimestamp on only one side - check: - cmp(smallIndex1, noSenderTime1) < 0 - cmp(noSenderTime1, smallIndex1) > 0 # Test symmetry - cmp(noSenderTime2, eqIndex3) < 0 - cmp(eqIndex3, noSenderTime2) > 0 # Test symmetry + cmp(eqIndex2, eqIndex4) < 0 + cmp(eqIndex4, eqIndex2) > 0 # Test symmetry test "Index equality": - # Exactly equal check: eqIndex1 == eqIndex2 - - # Receiver time plays no role, even without sender time - check: - eqIndex1 == eqIndex3 - noSenderTime1 == noSenderTime2 # only receiver time differs, indices are equal - noSenderTime1 != noSenderTime3 # pubsubTopics differ - noSenderTime1 != noSenderTime4 # digests differ - - # Unequal sender time - check: - smallIndex1 != largeIndex1 - - # Unequal digest - check: - smallIndex1 != smallIndex2 - - # Unequal hash and digest - check: - smallIndex1 != eqIndex1 - - # Unequal pubsubTopic - check: - smallIndex1 != diffPsTopic - - test "Index computation should not be empty": - ## Given - let ts = getTestTimestamp() - let wm = WakuMessage(payload: @[byte 1, 2, 3], timestamp: ts) - - ## When - let ts2 = getTestTimestamp() + 10 - let index = Index.compute(wm, ts2, DefaultContentTopic) - - ## Then - check: - index.digest.data.len != 0 - index.digest.data.len == 32 # sha2 output length in bytes - index.receiverTime == ts2 # the receiver timestamp should be a non-zero value - index.senderTime == ts - index.pubsubTopic == DefaultContentTopic - - test "Index digest of two identical messsage should be the same": - ## Given - let topic = ContentTopic("test-content-topic") - let - wm1 = WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic) - wm2 = WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic) - - ## When - let ts = getTestTimestamp() - let - index1 = Index.compute(wm1, ts, DefaultPubsubTopic) - index2 = Index.compute(wm2, ts, DefaultPubsubTopic) - - ## Then - check: - index1.digest == index2.digest + eqIndex1 == eqIndex4 + eqIndex2 != eqIndex3 + eqIndex4 != eqIndex3 diff --git a/tests/waku_archive/test_driver_queue_pagination.nim b/tests/waku_archive/test_driver_queue_pagination.nim index 1545a4aab..dec3ccdee 100644 --- a/tests/waku_archive/test_driver_queue_pagination.nim +++ b/tests/waku_archive/test_driver_queue_pagination.nim @@ -23,10 +23,9 @@ proc getTestQueueDriver(numMessages: int): QueueDriver = let msg = WakuMessage(payload: @[byte i], timestamp: Timestamp(i)) let index = Index( - receiverTime: Timestamp(i), - senderTime: Timestamp(i), - digest: MessageDigest(data: data), + time: Timestamp(i), hash: computeMessageHash(DefaultPubsubTopic, msg), + pubsubTopic: DefaultPubsubTopic, ) discard testQueueDriver.add(index, msg) @@ -50,7 +49,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 2 data == msgList[4 .. 5] @@ -66,7 +65,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 2 data == msgList[0 .. 1] @@ -82,7 +81,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 10 data == msgList[0 .. 9] @@ -99,7 +98,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 0 @@ -114,7 +113,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 6 data == msgList[4 .. 9] @@ -130,7 +129,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: uint(data.len) <= MaxPageSize @@ -145,19 +144,14 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 0 test "Forward pagination - invalid cursor": ## Given let msg = fakeWakuMessage(payload = @[byte 10]) - let index = ArchiveCursor( - pubsubTopic: DefaultPubsubTopic, - senderTime: msg.timestamp, - storeTime: msg.timestamp, - digest: computeDigest(msg), - ).toIndex() + let index = Index(hash: computeMessageHash(DefaultPubsubTopic, msg)) let pageSize: uint = 10 @@ -184,7 +178,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 1 @@ -200,7 +194,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 0 @@ -220,7 +214,7 @@ procSuite "Queue driver - pagination": ) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.mapIt(it.timestamp.int) == @[0, 2, 4] @@ -235,7 +229,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data == msgList[1 .. 2].reversed @@ -251,7 +245,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 0 @@ -266,7 +260,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 2 data == msgList[8 .. 9].reversed @@ -282,7 +276,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 10 data == msgList[0 .. 9].reversed @@ -298,7 +292,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data == msgList[0 .. 2].reversed @@ -313,7 +307,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: uint(data.len) <= MaxPageSize @@ -328,19 +322,14 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 0 test "Backward pagination - invalid cursor": ## Given let msg = fakeWakuMessage(payload = @[byte 10]) - let index = ArchiveCursor( - pubsubTopic: DefaultPubsubTopic, - senderTime: msg.timestamp, - storeTime: msg.timestamp, - digest: computeDigest(msg), - ).toIndex() + let index = Index(hash: computeMessageHash(DefaultPubsubTopic, msg)) let pageSize: uint = 2 @@ -367,7 +356,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 1 @@ -383,7 +372,7 @@ procSuite "Queue driver - pagination": let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.len == 0 @@ -403,6 +392,6 @@ procSuite "Queue driver - pagination": ) ## Then - let data = page.tryGet().mapIt(it[1]) + let data = page.tryGet().mapIt(it[2]) check: data.mapIt(it.timestamp.int) == @[5, 7, 9].reversed diff --git a/tests/waku_archive/test_driver_queue_query.nim b/tests/waku_archive/test_driver_queue_query.nim index d30b96c02..34d8087c8 100644 --- a/tests/waku_archive/test_driver_queue_query.nim +++ b/tests/waku_archive/test_driver_queue_query.nim @@ -22,15 +22,6 @@ common.randomize() proc newTestSqliteDriver(): ArchiveDriver = QueueDriver.new(capacity = 50) -proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - suite "Queue driver - query by content topic": test "no content topic": ## Given @@ -56,11 +47,7 @@ suite "Queue driver - query by content topic": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() @@ -71,7 +58,7 @@ suite "Queue driver - query by content topic": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[0 .. 4] @@ -102,24 +89,20 @@ suite "Queue driver - query by content topic": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3] @@ -150,24 +133,20 @@ suite "Queue driver - query by content topic": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = false ) ## Then check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[6 .. 7].reversed() @@ -200,17 +179,13 @@ suite "Queue driver - query by content topic": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], + contentTopics = @[contentTopic1, contentTopic2], maxPageSize = 2, ascendingOrder = true, ) @@ -219,7 +194,7 @@ suite "Queue driver - query by content topic": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3] @@ -247,24 +222,20 @@ suite "Queue driver - query by content topic": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 @@ -280,17 +251,13 @@ suite "Queue driver - query by content topic": for t in 0 ..< 40: let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[DefaultContentTopic], + contentTopics = @[DefaultContentTopic], maxPageSize = pageSize, ascendingOrder = true, ) @@ -299,7 +266,7 @@ suite "Queue driver - query by content topic": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 40 @@ -351,9 +318,7 @@ suite "SQLite driver - query by pubsub topic": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() ## When @@ -366,7 +331,7 @@ suite "SQLite driver - query by pubsub topic": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -417,9 +382,7 @@ suite "SQLite driver - query by pubsub topic": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() ## When @@ -430,7 +393,7 @@ suite "SQLite driver - query by pubsub topic": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[0 .. 1] @@ -481,14 +444,12 @@ suite "SQLite driver - query by pubsub topic": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true, @@ -499,7 +460,7 @@ suite "SQLite driver - query by pubsub topic": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -532,15 +493,11 @@ suite "Queue driver - query by cursor": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = waitFor driver.getMessages( @@ -551,7 +508,7 @@ suite "Queue driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[5 .. 6] @@ -583,15 +540,11 @@ suite "Queue driver - query by cursor": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = waitFor driver.getMessages( @@ -602,7 +555,7 @@ suite "Queue driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3].reversed() @@ -632,21 +585,16 @@ suite "Queue driver - query by cursor": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() - let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) - let cursor = ArchiveCursor(hash: fakeCursor) + let cursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) ## When let res = waitFor driver.getMessages( includeData = true, - contentTopic = @[DefaultContentTopic], + contentTopics = @[DefaultContentTopic], pubsubTopic = none(PubsubTopic), cursor = some(cursor), startTime = none(Timestamp), @@ -689,19 +637,15 @@ suite "Queue driver - query by cursor": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), maxPageSize = 10, ascendingOrder = true, @@ -711,7 +655,7 @@ suite "Queue driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[5 .. 6] @@ -743,19 +687,15 @@ suite "Queue driver - query by cursor": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[6]) ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), maxPageSize = 10, ascendingOrder = false, @@ -765,7 +705,7 @@ suite "Queue driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 5].reversed() @@ -841,12 +781,10 @@ suite "Queue driver - query by cursor": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() - let cursor = computeTestCursor(expected[5][0], expected[5][1]) + let cursor = computeMessageHash(expected[5][0], expected[5][1]) ## When let res = waitFor driver.getMessages( @@ -861,7 +799,7 @@ suite "Queue driver - query by cursor": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[6 .. 7] @@ -937,12 +875,10 @@ suite "Queue driver - query by cursor": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() - let cursor = computeTestCursor(expected[6][0], expected[6][1]) + let cursor = computeMessageHash(expected[6][0], expected[6][1]) ## When let res = waitFor driver.getMessages( @@ -957,7 +893,7 @@ suite "Queue driver - query by cursor": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5].reversed() @@ -990,11 +926,7 @@ suite "Queue driver - query by time range": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() @@ -1007,7 +939,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6] @@ -1039,11 +971,7 @@ suite "Queue driver - query by time range": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() @@ -1056,7 +984,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[0 .. 4] @@ -1134,9 +1062,7 @@ suite "Queue driver - query by time range": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() ## When @@ -1152,7 +1078,7 @@ suite "Queue driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[2 .. 4] @@ -1185,17 +1111,13 @@ suite "Queue driver - query by time range": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(45, timeOrigin)), endTime = some(ts(15, timeOrigin)), maxPageSize = 2, @@ -1205,7 +1127,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 @@ -1237,17 +1159,13 @@ suite "Queue driver - query by time range": for msg in messages: let retFut = await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true, @@ -1256,7 +1174,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6] @@ -1291,17 +1209,13 @@ suite "Queue driver - query by time range": for msg in messages: let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = false, @@ -1310,7 +1224,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6].reversed() @@ -1346,19 +1260,15 @@ suite "Queue driver - query by time range": for msg in messages: let retFut = await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[3]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[3]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), startTime = some(ts(15, timeOrigin)), maxPageSize = 10, @@ -1368,7 +1278,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[4 .. 9] @@ -1404,19 +1314,15 @@ suite "Queue driver - query by time range": for msg in messages: let retFut = await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) require retFut.isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[6]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), startTime = some(ts(15, timeOrigin)), maxPageSize = 10, @@ -1426,7 +1332,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[3 .. 4].reversed() @@ -1495,16 +1401,14 @@ suite "Queue driver - query by time range": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() - let cursor = computeTestCursor(DefaultPubsubTopic, expected[1][1]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[1][1]) ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(0, timeOrigin)), @@ -1517,7 +1421,7 @@ suite "Queue driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[3 .. 4] @@ -1585,16 +1489,14 @@ suite "Queue driver - query by time range": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() - let cursor = computeTestCursor(expected[7][0], expected[7][1]) + let cursor = computeMessageHash(expected[7][0], expected[7][1]) ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1607,7 +1509,7 @@ suite "Queue driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5].reversed() @@ -1676,16 +1578,14 @@ suite "Queue driver - query by time range": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() - let cursor = computeTestCursor(expected[1][0], expected[1][1]) + let cursor = computeMessageHash(expected[1][0], expected[1][1]) ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1699,7 +1599,7 @@ suite "Queue driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -1768,16 +1668,14 @@ suite "Queue driver - query by time range": for row in messages: let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) + let retFut = waitFor driver.put(computeMessageHash(topic, msg), topic, msg) require retFut.isOk() - let cursor = computeTestCursor(expected[1][0], expected[1][1]) + let cursor = computeMessageHash(expected[1][0], expected[1][1]) ## When let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1790,7 +1688,7 @@ suite "Queue driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 diff --git a/tests/waku_archive/test_driver_sqlite.nim b/tests/waku_archive/test_driver_sqlite.nim index a412ab7b7..3ceae595d 100644 --- a/tests/waku_archive/test_driver_sqlite.nim +++ b/tests/waku_archive/test_driver_sqlite.nim @@ -9,7 +9,6 @@ import waku_core, ], ../waku_archive/archive_utils, - ../testlib/common, ../testlib/wakucore suite "SQLite driver": @@ -42,9 +41,7 @@ suite "SQLite driver": let msgHash = computeMessageHash(DefaultPubsubTopic, msg) ## When - let putRes = waitFor driver.put( - DefaultPubsubTopic, msg, computeDigest(msg), msgHash, msg.timestamp - ) + let putRes = waitFor driver.put(msgHash, DefaultPubsubTopic, msg) ## Then check: @@ -54,7 +51,7 @@ suite "SQLite driver": check: storedMsg.len == 1 storedMsg.all do(item: auto) -> bool: - let (pubsubTopic, actualMsg, _, _, hash) = item + let (hash, pubsubTopic, actualMsg) = item actualMsg.contentTopic == contentTopic and pubsubTopic == DefaultPubsubTopic and hash == msgHash and msg.meta == actualMsg.meta diff --git a/tests/waku_archive/test_driver_sqlite_query.nim b/tests/waku_archive/test_driver_sqlite_query.nim index 026cab217..fc00a3be8 100644 --- a/tests/waku_archive/test_driver_sqlite_query.nim +++ b/tests/waku_archive/test_driver_sqlite_query.nim @@ -47,11 +47,7 @@ suite "SQLite driver - query by content topic": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -62,7 +58,7 @@ suite "SQLite driver - query by content topic": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[0 .. 4] @@ -94,24 +90,20 @@ suite "SQLite driver - query by content topic": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3] @@ -155,24 +147,20 @@ suite "SQLite driver - query by content topic": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3] @@ -204,24 +192,20 @@ suite "SQLite driver - query by content topic": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = false ) ## Then check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[6 .. 7].reversed() @@ -255,17 +239,13 @@ suite "SQLite driver - query by content topic": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], + contentTopics = @[contentTopic1, contentTopic2], maxPageSize = 2, ascendingOrder = true, ) @@ -274,7 +254,7 @@ suite "SQLite driver - query by content topic": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3] @@ -303,24 +283,20 @@ suite "SQLite driver - query by content topic": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + contentTopics = @[contentTopic], maxPageSize = 2, ascendingOrder = true ) ## Then check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 @@ -337,17 +313,13 @@ suite "SQLite driver - query by content topic": let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[DefaultContentTopic], + contentTopics = @[DefaultContentTopic], maxPageSize = pageSize, ascendingOrder = true, ) @@ -356,7 +328,7 @@ suite "SQLite driver - query by content topic": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 40 @@ -408,11 +380,7 @@ suite "SQLite driver - query by pubsub topic": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages( @@ -424,7 +392,7 @@ suite "SQLite driver - query by pubsub topic": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -475,11 +443,7 @@ suite "SQLite driver - query by pubsub topic": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages(maxPageSize = 2, ascendingOrder = true) @@ -489,7 +453,7 @@ suite "SQLite driver - query by pubsub topic": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[0 .. 1] @@ -540,15 +504,11 @@ suite "SQLite driver - query by pubsub topic": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true, @@ -559,7 +519,7 @@ suite "SQLite driver - query by pubsub topic": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -593,15 +553,11 @@ suite "SQLite driver - query by cursor": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = await driver.getMessages( @@ -612,7 +568,7 @@ suite "SQLite driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[5 .. 6] @@ -645,15 +601,11 @@ suite "SQLite driver - query by cursor": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = await driver.getMessages( @@ -664,7 +616,7 @@ suite "SQLite driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 3].reversed() @@ -695,21 +647,16 @@ suite "SQLite driver - query by cursor": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) - let cursor = ArchiveCursor(hash: fakeCursor) + let cursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) ## When let res = await driver.getMessages( includeData = true, - contentTopic = @[DefaultContentTopic], + contentTopics = @[DefaultContentTopic], pubsubTopic = none(PubsubTopic), cursor = some(cursor), startTime = none(Timestamp), @@ -753,19 +700,15 @@ suite "SQLite driver - query by cursor": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[4]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), maxPageSize = 10, ascendingOrder = true, @@ -775,7 +718,7 @@ suite "SQLite driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[5 .. 6] @@ -808,19 +751,15 @@ suite "SQLite driver - query by cursor": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[6]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[6]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), maxPageSize = 10, ascendingOrder = false, @@ -830,7 +769,7 @@ suite "SQLite driver - query by cursor": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 5].reversed() @@ -906,13 +845,9 @@ suite "SQLite driver - query by cursor": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeArchiveCursor(expected[5][0], expected[5][1]) + let cursor = computeMessageHash(expected[5][0], expected[5][1]) ## When let res = await driver.getMessages( @@ -927,7 +862,7 @@ suite "SQLite driver - query by cursor": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[6 .. 7] @@ -1003,13 +938,9 @@ suite "SQLite driver - query by cursor": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeArchiveCursor(expected[6][0], expected[6][1]) + let cursor = computeMessageHash(expected[6][0], expected[6][1]) ## When let res = await driver.getMessages( @@ -1024,7 +955,7 @@ suite "SQLite driver - query by cursor": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5].reversed() @@ -1058,11 +989,7 @@ suite "SQLite driver - query by time range": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -1075,7 +1002,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6] @@ -1108,11 +1035,7 @@ suite "SQLite driver - query by time range": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -1125,7 +1048,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[0 .. 4] @@ -1203,11 +1126,7 @@ suite "SQLite driver - query by time range": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() ## When let res = await driver.getMessages( @@ -1222,7 +1141,7 @@ suite "SQLite driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[2 .. 4] @@ -1256,17 +1175,13 @@ suite "SQLite driver - query by time range": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(45, timeOrigin)), endTime = some(ts(15, timeOrigin)), maxPageSize = 2, @@ -1276,7 +1191,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 @@ -1309,17 +1224,13 @@ suite "SQLite driver - query by time range": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true, @@ -1328,7 +1239,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6] @@ -1364,17 +1275,13 @@ suite "SQLite driver - query by time range": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = false, @@ -1383,7 +1290,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[2 .. 6].reversed() @@ -1420,19 +1327,15 @@ suite "SQLite driver - query by time range": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[3]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[3]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), startTime = some(ts(15, timeOrigin)), maxPageSize = 10, @@ -1442,7 +1345,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[4 .. 9] @@ -1479,19 +1382,15 @@ suite "SQLite driver - query by time range": for msg in messages: require ( await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[6]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[6]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], cursor = some(cursor), startTime = some(ts(15, timeOrigin)), maxPageSize = 10, @@ -1501,7 +1400,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expected[3 .. 4].reversed() @@ -1570,17 +1469,13 @@ suite "SQLite driver - query by time range": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[1][1]) + let cursor = computeMessageHash(DefaultPubsubTopic, expected[1][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(0, timeOrigin)), @@ -1593,7 +1488,7 @@ suite "SQLite driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[3 .. 4] @@ -1661,17 +1556,13 @@ suite "SQLite driver - query by time range": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeArchiveCursor(expected[7][0], expected[7][1]) + let cursor = computeMessageHash(expected[7][0], expected[7][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1684,7 +1575,7 @@ suite "SQLite driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5].reversed() @@ -1753,17 +1644,13 @@ suite "SQLite driver - query by time range": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeArchiveCursor(expected[1][0], expected[1][1]) + let cursor = computeMessageHash(expected[1][0], expected[1][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1777,7 +1664,7 @@ suite "SQLite driver - query by time range": res.isOk() let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages == expectedMessages[4 .. 5] @@ -1846,17 +1733,13 @@ suite "SQLite driver - query by time range": for row in messages: let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() + require (await driver.put(computeMessageHash(topic, msg), topic, msg)).isOk() - let cursor = computeArchiveCursor(expected[1][0], expected[1][1]) + let cursor = computeMessageHash(expected[1][0], expected[1][1]) ## When let res = await driver.getMessages( - contentTopic = @[contentTopic], + contentTopics = @[contentTopic], pubsubTopic = some(pubsubTopic), cursor = some(cursor), startTime = some(ts(35, timeOrigin)), @@ -1869,7 +1752,7 @@ suite "SQLite driver - query by time range": check: res.isOk() - let filteredMessages = res.tryGet().mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[2]) check: filteredMessages.len == 0 diff --git a/tests/waku_archive/test_retention_policy.nim b/tests/waku_archive/test_retention_policy.nim index f3d7ebf51..4686dda7e 100644 --- a/tests/waku_archive/test_retention_policy.nim +++ b/tests/waku_archive/test_retention_policy.nim @@ -13,7 +13,6 @@ import waku_archive/retention_policy/retention_policy_size, ], ../waku_archive/archive_utils, - ../testlib/common, ../testlib/wakucore suite "Waku Archive - Retention policy": @@ -35,18 +34,13 @@ suite "Waku Archive - Retention policy": payload = @[byte i], contentTopic = DefaultContentTopic, ts = Timestamp(i) ) putFutures.add( - driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) + driver.put(computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg) ) discard waitFor allFinished(putFutures) - require (waitFor retentionPolicy.execute(driver)).isOk() + let res = waitFor retentionPolicy.execute(driver) + assert res.isOk(), $res.error ## Then let numMessages = (waitFor driver.getMessagesCount()).tryGet() @@ -88,13 +82,7 @@ suite "Waku Archive - Retention policy": payload = @[byte i], contentTopic = DefaultContentTopic, ts = Timestamp(i) ) putFutures.add( - driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) + driver.put(computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg) ) # waitFor is used to synchronously wait for the futures to complete. @@ -150,11 +138,7 @@ suite "Waku Archive - Retention policy": for msg in messages: require ( waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() require (waitFor retentionPolicy.execute(driver)).isOk() @@ -164,7 +148,7 @@ suite "Waku Archive - Retention policy": check: storedMsg.len == capacity storedMsg.all do(item: auto) -> bool: - let (pubsubTopic, msg, _, _, _) = item + let (_, pubsubTopic, msg) = item msg.contentTopic == contentTopic and pubsubTopic == DefaultPubsubTopic ## Cleanup diff --git a/tests/waku_archive/test_waku_archive.nim b/tests/waku_archive/test_waku_archive.nim index de7034a1b..9e1b927e0 100644 --- a/tests/waku_archive/test_waku_archive.nim +++ b/tests/waku_archive/test_waku_archive.nim @@ -1,11 +1,6 @@ {.used.} -import - std/[options, sequtils], - testutils/unittests, - chronicles, - chronos, - libp2p/crypto/crypto +import std/[options, sequtils], testutils/unittests, chronos, libp2p/crypto/crypto import waku/[ @@ -17,7 +12,6 @@ import waku_archive, ], ../waku_archive/archive_utils, - ../testlib/common, ../testlib/wakucore suite "Waku Archive - message handling": @@ -60,7 +54,7 @@ suite "Waku Archive - message handling": check: (waitFor driver.getMessagesCount()).tryGet() == 2 - test "it should archive a message with no sender timestamp": + test "it should not archive a message with no sender timestamp": ## Setup let driver = newSqliteArchiveDriver() let archive = newWakuArchive(driver) @@ -74,7 +68,7 @@ suite "Waku Archive - message handling": ## Then check: - (waitFor driver.getMessagesCount()).tryGet() == 1 + (waitFor driver.getMessagesCount()).tryGet() == 0 test "it should not archive a message with a sender time variance greater than max time variance (future)": ## Setup @@ -160,11 +154,7 @@ procSuite "Waku Archive - find messages": for msg in msgListA: require ( waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() @@ -250,13 +240,11 @@ procSuite "Waku Archive - find messages": let queryRes = waitFor archive.findMessages(req) ## Then - check: - queryRes.isErr() + assert queryRes.isOk(), $queryRes.error - let error = queryRes.tryError() + let response = queryRes.tryGet() check: - error.kind == ArchiveErrorKind.INVALID_QUERY - error.cause == "too many content topics" + response.messages.len() == 0 test "handle query with pubsub topic filter": ## Setup @@ -394,8 +382,8 @@ procSuite "Waku Archive - find messages": ## Then check: - cursors[0] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[3])) - cursors[1] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[7])) + cursors[0] == some(computeMessageHash(DefaultPubsubTopic, msgListA[3])) + cursors[1] == some(computeMessageHash(DefaultPubsubTopic, msgListA[7])) cursors[2] == none(ArchiveCursor) check: @@ -428,8 +416,8 @@ procSuite "Waku Archive - find messages": ## Then check: - cursors[0] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[6])) - cursors[1] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[2])) + cursors[0] == some(computeMessageHash(DefaultPubsubTopic, msgListA[6])) + cursors[1] == some(computeMessageHash(DefaultPubsubTopic, msgListA[2])) cursors[2] == none(ArchiveCursor) check: @@ -460,11 +448,7 @@ procSuite "Waku Archive - find messages": for msg in msgList: require ( waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, + computeMessageHash(DefaultPubsubTopic, msg), DefaultPubsubTopic, msg ) ).isOk() diff --git a/tests/waku_archive_legacy/archive_utils.nim b/tests/waku_archive_legacy/archive_utils.nim new file mode 100644 index 000000000..5fb17614d --- /dev/null +++ b/tests/waku_archive_legacy/archive_utils.nim @@ -0,0 +1,53 @@ +{.used.} + +import std/options, results, chronos, libp2p/crypto/crypto + +import + waku/[ + node/peer_manager, + waku_core, + waku_archive_legacy, + waku_archive_legacy/common, + waku_archive_legacy/driver/sqlite_driver, + common/databases/db_sqlite, + ], + ../testlib/[wakucore] + +proc newSqliteDatabase*(path: Option[string] = string.none()): SqliteDatabase = + SqliteDatabase.new(path.get(":memory:")).tryGet() + +proc newSqliteArchiveDriver*(): ArchiveDriver = + let database = newSqliteDatabase() + SqliteDriver.new(database).tryGet() + +proc newWakuArchive*(driver: ArchiveDriver): WakuArchive = + WakuArchive.new(driver).get() + +proc computeArchiveCursor*( + pubsubTopic: PubsubTopic, message: WakuMessage +): ArchiveCursor = + ArchiveCursor( + pubsubTopic: pubsubTopic, + senderTime: message.timestamp, + storeTime: message.timestamp, + digest: computeDigest(message), + hash: computeMessageHash(pubsubTopic, message), + ) + +proc put*( + driver: ArchiveDriver, pubsubTopic: PubSubTopic, msgList: seq[WakuMessage] +): ArchiveDriver = + for msg in msgList: + let + msgDigest = computeDigest(msg) + msgHash = computeMessageHash(pubsubTopic, msg) + _ = waitFor driver.put(pubsubTopic, msg, msgDigest, msgHash, msg.timestamp) + # discard crashes + return driver + +proc newArchiveDriverWithMessages*( + pubsubTopic: PubSubTopic, msgList: seq[WakuMessage] +): ArchiveDriver = + var driver = newSqliteArchiveDriver() + driver = driver.put(pubsubTopic, msgList) + return driver diff --git a/tests/waku_archive_legacy/test_all.nim b/tests/waku_archive_legacy/test_all.nim new file mode 100644 index 000000000..9d45d99a1 --- /dev/null +++ b/tests/waku_archive_legacy/test_all.nim @@ -0,0 +1,13 @@ +{.used.} + +import + ./test_driver_postgres_query, + ./test_driver_postgres, + ./test_driver_queue_index, + ./test_driver_queue_pagination, + ./test_driver_queue_query, + ./test_driver_queue, + ./test_driver_sqlite_query, + ./test_driver_sqlite, + ./test_retention_policy, + ./test_waku_archive diff --git a/tests/waku_archive_legacy/test_driver_postgres.nim b/tests/waku_archive_legacy/test_driver_postgres.nim new file mode 100644 index 000000000..b83897a33 --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_postgres.nim @@ -0,0 +1,201 @@ +{.used.} + +import std/[sequtils, options], testutils/unittests, chronos +import + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver/postgres_driver, + waku/waku_core, + waku/waku_core/message/digest, + ../testlib/wakucore, + ../testlib/testasync, + ../testlib/postgres_legacy + +proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = + ArchiveCursor( + pubsubTopic: pubsubTopic, + senderTime: message.timestamp, + storeTime: message.timestamp, + digest: computeDigest(message), + hash: computeMessageHash(pubsubTopic, message), + ) + +suite "Postgres driver": + ## Unique driver instance + var driver {.threadvar.}: PostgresDriver + + asyncSetup: + let driverRes = await newTestPostgresDriver() + if driverRes.isErr(): + assert false, driverRes.error + + driver = PostgresDriver(driverRes.get()) + + asyncTeardown: + let resetRes = await driver.reset() + if resetRes.isErr(): + assert false, resetRes.error + + (await driver.close()).expect("driver to close") + + asyncTest "Asynchronous queries": + var futures = newSeq[Future[ArchiveDriverResult[void]]](0) + + let beforeSleep = now() + for _ in 1 .. 100: + futures.add(driver.sleep(1)) + + await allFutures(futures) + + let diff = now() - beforeSleep + # Actually, the diff randomly goes between 1 and 2 seconds. + # although in theory it should spend 1s because we establish 100 + # connections and we spawn 100 tasks that spend ~1s each. + assert diff < 20_000_000_000 + + asyncTest "Insert a message": + const contentTopic = "test-content-topic" + const meta = "test meta" + + let msg = fakeWakuMessage(contentTopic = contentTopic, meta = meta) + + let computedDigest = computeDigest(msg) + let computedHash = computeMessageHash(DefaultPubsubTopic, msg) + + let putRes = await driver.put( + DefaultPubsubTopic, msg, computedDigest, computedHash, msg.timestamp + ) + assert putRes.isOk(), putRes.error + + let storedMsg = (await driver.getAllMessages()).tryGet() + + assert storedMsg.len == 1 + + let (pubsubTopic, actualMsg, digest, _, hash) = storedMsg[0] + assert actualMsg.contentTopic == contentTopic + assert pubsubTopic == DefaultPubsubTopic + assert toHex(computedDigest.data) == toHex(digest) + assert toHex(actualMsg.payload) == toHex(msg.payload) + assert toHex(computedHash) == toHex(hash) + assert toHex(actualMsg.meta) == toHex(msg.meta) + + asyncTest "Insert and query message": + const contentTopic1 = "test-content-topic-1" + const contentTopic2 = "test-content-topic-2" + const pubsubTopic1 = "pubsubtopic-1" + const pubsubTopic2 = "pubsubtopic-2" + + let msg1 = fakeWakuMessage(contentTopic = contentTopic1) + + var putRes = await driver.put( + pubsubTopic1, + msg1, + computeDigest(msg1), + computeMessageHash(pubsubTopic1, msg1), + msg1.timestamp, + ) + assert putRes.isOk(), putRes.error + + let msg2 = fakeWakuMessage(contentTopic = contentTopic2) + + putRes = await driver.put( + pubsubTopic2, + msg2, + computeDigest(msg2), + computeMessageHash(pubsubTopic2, msg2), + msg2.timestamp, + ) + assert putRes.isOk(), putRes.error + + let countMessagesRes = await driver.getMessagesCount() + + assert countMessagesRes.isOk(), $countMessagesRes.error + assert countMessagesRes.get() == 2 + + var messagesRes = await driver.getMessages(contentTopic = @[contentTopic1]) + + assert messagesRes.isOk(), $messagesRes.error + assert messagesRes.get().len == 1 + + # Get both content topics, check ordering + messagesRes = + await driver.getMessages(contentTopic = @[contentTopic1, contentTopic2]) + assert messagesRes.isOk(), messagesRes.error + + assert messagesRes.get().len == 2 + assert messagesRes.get()[0][1].contentTopic == contentTopic1 + + # Descending order + messagesRes = await driver.getMessages( + contentTopic = @[contentTopic1, contentTopic2], ascendingOrder = false + ) + assert messagesRes.isOk(), messagesRes.error + + assert messagesRes.get().len == 2 + assert messagesRes.get()[0][1].contentTopic == contentTopic2 + + # cursor + # Get both content topics + messagesRes = await driver.getMessages( + contentTopic = @[contentTopic1, contentTopic2], + cursor = some(computeTestCursor(pubsubTopic1, messagesRes.get()[1][1])), + ) + assert messagesRes.isOk() + assert messagesRes.get().len == 1 + + # Get both content topics but one pubsub topic + messagesRes = await driver.getMessages( + contentTopic = @[contentTopic1, contentTopic2], pubsubTopic = some(pubsubTopic1) + ) + assert messagesRes.isOk(), messagesRes.error + + assert messagesRes.get().len == 1 + assert messagesRes.get()[0][1].contentTopic == contentTopic1 + + # Limit + messagesRes = await driver.getMessages( + contentTopic = @[contentTopic1, contentTopic2], maxPageSize = 1 + ) + assert messagesRes.isOk(), messagesRes.error + assert messagesRes.get().len == 1 + + asyncTest "Insert true duplicated messages": + # Validates that two completely equal messages can not be stored. + + let now = now() + + let msg1 = fakeWakuMessage(ts = now) + let msg2 = fakeWakuMessage(ts = now) + + let initialNumMsgs = (await driver.getMessagesCount()).valueOr: + raiseAssert "could not get num mgs correctly: " & $error + + var putRes = await driver.put( + DefaultPubsubTopic, + msg1, + computeDigest(msg1), + computeMessageHash(DefaultPubsubTopic, msg1), + msg1.timestamp, + ) + assert putRes.isOk(), putRes.error + + var newNumMsgs = (await driver.getMessagesCount()).valueOr: + raiseAssert "could not get num mgs correctly: " & $error + + assert newNumMsgs == (initialNumMsgs + 1.int64), + "wrong number of messages: " & $newNumMsgs + + putRes = await driver.put( + DefaultPubsubTopic, + msg2, + computeDigest(msg2), + computeMessageHash(DefaultPubsubTopic, msg2), + msg2.timestamp, + ) + + assert putRes.isOk() + + newNumMsgs = (await driver.getMessagesCount()).valueOr: + raiseAssert "could not get num mgs correctly: " & $error + + assert newNumMsgs == (initialNumMsgs + 1.int64), + "wrong number of messages: " & $newNumMsgs diff --git a/tests/waku_archive_legacy/test_driver_postgres_query.nim b/tests/waku_archive_legacy/test_driver_postgres_query.nim new file mode 100644 index 000000000..088943452 --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_postgres_query.nim @@ -0,0 +1,1931 @@ +{.used.} + +import + std/[options, sequtils, strformat, random, algorithm], + testutils/unittests, + chronos, + chronicles +import + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver as driver_module, + waku/waku_archive_legacy/driver/postgres_driver, + waku/waku_core, + waku/waku_core/message/digest, + ../testlib/common, + ../testlib/wakucore, + ../testlib/testasync, + ../testlib/postgres_legacy + +logScope: + topics = "test archive postgres driver" + +## This whole file is copied from the 'test_driver_sqlite_query.nim' file +## and it tests the same use cases but using the postgres driver. + +# Initialize the random number generator +common.randomize() + +proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = + ArchiveCursor( + pubsubTopic: pubsubTopic, + senderTime: message.timestamp, + storeTime: message.timestamp, + digest: computeDigest(message), + hash: computeMessageHash(pubsubTopic, message), + ) + +suite "Postgres driver - queries": + ## Unique driver instance + var driver {.threadvar.}: PostgresDriver + + asyncSetup: + let driverRes = await newTestPostgresDriver() + if driverRes.isErr(): + assert false, driverRes.error + + driver = PostgresDriver(driverRes.get()) + + asyncTeardown: + let resetRes = await driver.reset() + if resetRes.isErr(): + assert false, resetRes.error + + (await driver.close()).expect("driver to close") + + asyncTest "no content topic": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages(maxPageSize = 5, ascendingOrder = true) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[0 .. 4] + + asyncTest "single content topic": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3] + + asyncTest "single content topic with meta field": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), + fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" + ), + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" + ), + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" + ), + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" + ), + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" + ), + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3] + + asyncTest "single content topic - descending order": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[6 .. 7].reversed() + + asyncTest "multiple content topic": + ## Given + const contentTopic1 = "test-content-topic-1" + const contentTopic2 = "test-content-topic-2" + const contentTopic3 = "test-content-topic-3" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + var res = await driver.getMessages( + contentTopic = @[contentTopic1, contentTopic2], + pubsubTopic = some(DefaultPubsubTopic), + maxPageSize = 2, + ascendingOrder = true, + startTime = some(ts(00)), + endTime = some(ts(40)), + ) + + ## Then + assert res.isOk(), res.error + var filteredMessages = res.tryGet().mapIt(it[1]) + check filteredMessages == expected[2 .. 3] + + ## When + ## This is very similar to the previous one but we enforce to use the prepared + ## statement by querying one single content topic + res = await driver.getMessages( + contentTopic = @[contentTopic1], + pubsubTopic = some(DefaultPubsubTopic), + maxPageSize = 2, + ascendingOrder = true, + startTime = some(ts(00)), + endTime = some(ts(40)), + ) + + ## Then + assert res.isOk(), res.error + filteredMessages = res.tryGet().mapIt(it[1]) + check filteredMessages == @[expected[2]] + + asyncTest "single content topic - no results": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + asyncTest "content topic and max page size - not enough messages stored": + ## Given + const pageSize: uint = 50 + + for t in 0 ..< 40: + let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[DefaultContentTopic], + maxPageSize = pageSize, + ascendingOrder = true, + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 40 + + asyncTest "pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages( + pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true + ) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + asyncTest "no pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages(maxPageSize = 2, ascendingOrder = true) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[0 .. 1] + + asyncTest "content topic and pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + maxPageSize = 2, + ascendingOrder = true, + ) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + asyncTest "only cursor": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = await driver.getMessages( + cursor = some(cursor), maxPageSize = 2, ascendingOrder = true + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[5 .. 6] + + asyncTest "only cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = await driver.getMessages( + cursor = some(cursor), maxPageSize = 2, ascendingOrder = false + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3].reversed() + + asyncTest "only cursor - invalid": + ## Given + const contentTopic = "test-content-topic" + + var messages = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) + let cursor = ArchiveCursor(hash: fakeCursor) + + ## When + let res = await driver.getMessages( + includeData = true, + contentTopicSeq = @[DefaultContentTopic], + pubsubTopic = none(PubsubTopic), + cursor = some(cursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hashes = @[], + maxPageSize = 5, + ascendingOrder = true, + ) + + ## Then + assert res.isOk(), res.error + + check: + res.value.len == 0 + + asyncTest "content topic and cursor": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[5 .. 6] + + asyncTest "content topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 5].reversed() + + asyncTest "pubsub topic and cursor": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeTestCursor(expected[5][0], expected[5][1]) + + ## When + let res = await driver.getMessages( + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[6 .. 7] + + asyncTest "pubsub topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeTestCursor(expected[6][0], expected[6][1]) + + ## When + let res = await driver.getMessages( + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5].reversed() + + asyncTest "only hashes - descending order": + ## Given + let timeOrigin = now() + var expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + let hashes = messages.mapIt(computeMessageHash(DefaultPubsubTopic, it)) + + for (msg, hash) in messages.zip(hashes): + require ( + await driver.put( + DefaultPubsubTopic, msg, computeDigest(msg), hash, msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages(hashes = hashes, ascendingOrder = false) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.reversed() + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages + + asyncTest "start time only": + ## Given + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6] + + asyncTest "end time only": + ## Given + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + endTime = some(ts(45, timeOrigin)), maxPageSize = 10, ascendingOrder = true + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[0 .. 4] + + asyncTest "start time and end time": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages( + startTime = some(ts(15, timeOrigin)), + endTime = some(ts(45, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[2 .. 4] + + asyncTest "invalid time range - no results": + ## Given + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(45, timeOrigin)), + endTime = some(ts(15, timeOrigin)), + maxPageSize = 2, + ascendingOrder = true, + ) + + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + asyncTest "time range start and content topic": + ## Given + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6] + + asyncTest "time range start and content topic - descending order": + ## Given + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6].reversed() + + asyncTest "time range start, single content topic and cursor": + ## Given + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[3]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[4 .. 9] + + asyncTest "time range start, single content topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[3 .. 4].reversed() + + asyncTest "time range, content topic, pubsub topic and cursor": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let timeOrigin = now() + let expected = + @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[1][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(0, timeOrigin)), + endTime = some(ts(45, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[3 .. 4] + + asyncTest "time range, content topic, pubsub topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeTestCursor(expected[7][0], expected[7][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5].reversed() + + asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeTestCursor(expected[1][0], expected[1][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + assert res.isOk(), res.error + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range, descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeTestCursor(expected[1][0], expected[1][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + assert res.isOk(), res.error + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + asyncTest "Get oldest and newest message timestamp": + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let oldestTime = ts(00, timeOrigin) + let newestTime = ts(100, timeOrigin) + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = oldestTime), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = newestTime), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## just keep the second resolution. + ## Notice that the oldest timestamps considers the minimum partition timestamp, which + ## is expressed in seconds. + let oldestPartitionTimestamp = + Timestamp(float(oldestTime) / 1_000_000_000) * 1_000_000_000 + + var res = await driver.getOldestMessageTimestamp() + assert res.isOk(), res.error + + ## We give certain margin of error. The oldest timestamp is obtained from + ## the oldest partition timestamp and there might be at most one second of difference + ## between the time created in the test and the oldest-partition-timestamp created within + ## the driver logic. + assert abs(res.get() - oldestPartitionTimestamp) < (2 * 1_000_000_000), + fmt"Failed to retrieve the latest timestamp {res.get()} != {oldestPartitionTimestamp}" + + res = await driver.getNewestMessageTimestamp() + assert res.isOk(), res.error + assert res.get() == newestTime, "Failed to retrieve the newest timestamp" + + asyncTest "Delete messages older than certain timestamp": + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let targetTime = ts(40, timeOrigin) + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = targetTime), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + var res = await driver.getMessagesCount() + assert res.isOk(), res.error + assert res.get() == 7, "Failed to retrieve the initial number of messages" + + let deleteRes = await driver.deleteMessagesOlderThanTimestamp(targetTime) + assert deleteRes.isOk(), deleteRes.error + + res = await driver.getMessagesCount() + assert res.isOk(), res.error + assert res.get() == 3, "Failed to retrieve the # of messages after deletion" + + asyncTest "Keep last n messages": + const contentTopic = "test-content-topic" + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + var res = await driver.getMessagesCount() + assert res.isOk(), res.error + assert res.get() == 7, "Failed to retrieve the initial number of messages" + + let deleteRes = await driver.deleteOldestMessagesNotWithinLimit(2) + assert deleteRes.isOk(), deleteRes.error + + res = await driver.getMessagesCount() + assert res.isOk(), res.error + assert res.get() == 2, "Failed to retrieve the # of messages after deletion" + + asyncTest "Exists table": + var existsRes = await driver.existsTable("version") + assert existsRes.isOk(), existsRes.error + check existsRes.get() == true diff --git a/tests/waku_archive_legacy/test_driver_queue.nim b/tests/waku_archive_legacy/test_driver_queue.nim new file mode 100644 index 000000000..c69e5aa6a --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_queue.nim @@ -0,0 +1,182 @@ +{.used.} + +import std/options, stew/results, testutils/unittests +import + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver/queue_driver/queue_driver {.all.}, + waku/waku_archive_legacy/driver/queue_driver/index, + waku/waku_core + +# Helper functions + +proc genIndexedWakuMessage(i: int8): (Index, WakuMessage) = + ## Use i to generate an Index WakuMessage + var data {.noinit.}: array[32, byte] + for x in data.mitems: + x = i.byte + + let + message = WakuMessage(payload: @[byte i], timestamp: Timestamp(i)) + topic = "test-pubsub-topic" + cursor = Index( + receiverTime: Timestamp(i), + senderTime: Timestamp(i), + digest: MessageDigest(data: data), + pubsubTopic: topic, + hash: computeMessageHash(topic, message), + ) + + (cursor, message) + +proc getPrepopulatedTestQueue(unsortedSet: auto, capacity: int): QueueDriver = + let driver = QueueDriver.new(capacity) + + for i in unsortedSet: + let (index, message) = genIndexedWakuMessage(i.int8) + discard driver.add(index, message) + + driver + +procSuite "Sorted driver queue": + test "queue capacity - add a message over the limit": + ## Given + let capacity = 5 + let driver = QueueDriver.new(capacity) + + ## When + # Fill up the queue + for i in 1 .. capacity: + let (index, message) = genIndexedWakuMessage(i.int8) + require(driver.add(index, message).isOk()) + + # Add one more. Capacity should not be exceeded + let (index, message) = genIndexedWakuMessage(capacity.int8 + 1) + require(driver.add(index, message).isOk()) + + ## Then + check: + driver.len == capacity + + test "queue capacity - add message older than oldest in the queue": + ## Given + let capacity = 5 + let driver = QueueDriver.new(capacity) + + ## When + # Fill up the queue + for i in 1 .. capacity: + let (index, message) = genIndexedWakuMessage(i.int8) + require(driver.add(index, message).isOk()) + + # Attempt to add message with older value than oldest in queue should fail + let + oldestTimestamp = driver.first().get().senderTime + (index, message) = genIndexedWakuMessage(oldestTimestamp.int8 - 1) + addRes = driver.add(index, message) + + ## Then + check: + addRes.isErr() + addRes.error() == "too_old" + + check: + driver.len == capacity + + test "queue sort-on-insert": + ## Given + let + capacity = 5 + unsortedSet = [5, 1, 3, 2, 4] + let driver = getPrepopulatedTestQueue(unsortedSet, capacity) + + # Walk forward through the set and verify ascending order + var (prevSmaller, _) = genIndexedWakuMessage(min(unsortedSet).int8 - 1) + for i in driver.fwdIterator: + let (index, _) = i + check cmp(index, prevSmaller) > 0 + prevSmaller = index + + # Walk backward through the set and verify descending order + var (prevLarger, _) = genIndexedWakuMessage(max(unsortedSet).int8 + 1) + for i in driver.bwdIterator: + let (index, _) = i + check cmp(index, prevLarger) < 0 + prevLarger = index + + test "access first item from queue": + ## Given + let + capacity = 5 + unsortedSet = [5, 1, 3, 2, 4] + let driver = getPrepopulatedTestQueue(unsortedSet, capacity) + + ## When + let firstRes = driver.first() + + ## Then + check: + firstRes.isOk() + + let first = firstRes.tryGet() + check: + first.senderTime == Timestamp(1) + + test "get first item from empty queue should fail": + ## Given + let capacity = 5 + let driver = QueueDriver.new(capacity) + + ## When + let firstRes = driver.first() + + ## Then + check: + firstRes.isErr() + firstRes.error() == "Not found" + + test "access last item from queue": + ## Given + let + capacity = 5 + unsortedSet = [5, 1, 3, 2, 4] + let driver = getPrepopulatedTestQueue(unsortedSet, capacity) + + ## When + let lastRes = driver.last() + + ## Then + check: + lastRes.isOk() + + let last = lastRes.tryGet() + check: + last.senderTime == Timestamp(5) + + test "get last item from empty queue should fail": + ## Given + let capacity = 5 + let driver = QueueDriver.new(capacity) + + ## When + let lastRes = driver.last() + + ## Then + check: + lastRes.isErr() + lastRes.error() == "Not found" + + test "verify if queue contains an index": + ## Given + let + capacity = 5 + unsortedSet = [5, 1, 3, 2, 4] + let driver = getPrepopulatedTestQueue(unsortedSet, capacity) + + let + (existingIndex, _) = genIndexedWakuMessage(4) + (nonExistingIndex, _) = genIndexedWakuMessage(99) + + ## Then + check: + driver.contains(existingIndex) == true + driver.contains(nonExistingIndex) == false diff --git a/tests/waku_archive_legacy/test_driver_queue_index.nim b/tests/waku_archive_legacy/test_driver_queue_index.nim new file mode 100644 index 000000000..404dca8cb --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_queue_index.nim @@ -0,0 +1,219 @@ +{.used.} + +import std/[times, random], stew/byteutils, testutils/unittests, nimcrypto +import waku/waku_core, waku/waku_archive_legacy/driver/queue_driver/index + +var rng = initRand() + +## Helpers + +proc getTestTimestamp(offset = 0): Timestamp = + let now = getNanosecondTime(epochTime() + float(offset)) + Timestamp(now) + +proc hashFromStr(input: string): MDigest[256] = + var ctx: sha256 + + ctx.init() + ctx.update(input.toBytes()) + let hashed = ctx.finish() + ctx.clear() + + return hashed + +proc randomHash(): WakuMessageHash = + var hash: WakuMessageHash + + for i in 0 ..< hash.len: + let numb: byte = byte(rng.next()) + hash[i] = numb + + hash + +suite "Queue Driver - index": + ## Test vars + let + smallIndex1 = Index( + digest: hashFromStr("1234"), + receiverTime: getNanosecondTime(0), + senderTime: getNanosecondTime(1000), + hash: randomHash(), + ) + smallIndex2 = Index( + digest: hashFromStr("1234567"), # digest is less significant than senderTime + receiverTime: getNanosecondTime(0), + senderTime: getNanosecondTime(1000), + hash: randomHash(), + ) + largeIndex1 = Index( + digest: hashFromStr("1234"), + receiverTime: getNanosecondTime(0), + senderTime: getNanosecondTime(9000), + hash: randomHash(), + ) # only senderTime differ from smallIndex1 + largeIndex2 = Index( + digest: hashFromStr("12345"), # only digest differs from smallIndex1 + receiverTime: getNanosecondTime(0), + senderTime: getNanosecondTime(1000), + hash: randomHash(), + ) + eqIndex1 = Index( + digest: hashFromStr("0003"), + receiverTime: getNanosecondTime(0), + senderTime: getNanosecondTime(54321), + hash: randomHash(), + ) + eqIndex2 = Index( + digest: hashFromStr("0003"), + receiverTime: getNanosecondTime(0), + senderTime: getNanosecondTime(54321), + hash: randomHash(), + ) + eqIndex3 = Index( + digest: hashFromStr("0003"), + receiverTime: getNanosecondTime(9999), + # receiverTime difference should have no effect on comparisons + senderTime: getNanosecondTime(54321), + hash: randomHash(), + ) + diffPsTopic = Index( + digest: hashFromStr("1234"), + receiverTime: getNanosecondTime(0), + senderTime: getNanosecondTime(1000), + pubsubTopic: "zzzz", + hash: randomHash(), + ) + noSenderTime1 = Index( + digest: hashFromStr("1234"), + receiverTime: getNanosecondTime(1100), + senderTime: getNanosecondTime(0), + pubsubTopic: "zzzz", + hash: randomHash(), + ) + noSenderTime2 = Index( + digest: hashFromStr("1234"), + receiverTime: getNanosecondTime(10000), + senderTime: getNanosecondTime(0), + pubsubTopic: "zzzz", + hash: randomHash(), + ) + noSenderTime3 = Index( + digest: hashFromStr("1234"), + receiverTime: getNanosecondTime(1200), + senderTime: getNanosecondTime(0), + pubsubTopic: "aaaa", + hash: randomHash(), + ) + noSenderTime4 = Index( + digest: hashFromStr("0"), + receiverTime: getNanosecondTime(1200), + senderTime: getNanosecondTime(0), + pubsubTopic: "zzzz", + hash: randomHash(), + ) + + test "Index comparison": + # Index comparison with senderTime diff + check: + cmp(smallIndex1, largeIndex1) < 0 + cmp(smallIndex2, largeIndex1) < 0 + + # Index comparison with digest diff + check: + cmp(smallIndex1, smallIndex2) < 0 + cmp(smallIndex1, largeIndex2) < 0 + cmp(smallIndex2, largeIndex2) > 0 + cmp(largeIndex1, largeIndex2) > 0 + + # Index comparison when equal + check: + cmp(eqIndex1, eqIndex2) == 0 + + # pubsubTopic difference + check: + cmp(smallIndex1, diffPsTopic) < 0 + + # receiverTime diff plays no role when senderTime set + check: + cmp(eqIndex1, eqIndex3) == 0 + + # receiverTime diff plays no role when digest/pubsubTopic equal + check: + cmp(noSenderTime1, noSenderTime2) == 0 + + # sort on receiverTime with no senderTimestamp and unequal pubsubTopic + check: + cmp(noSenderTime1, noSenderTime3) < 0 + + # sort on receiverTime with no senderTimestamp and unequal digest + check: + cmp(noSenderTime1, noSenderTime4) < 0 + + # sort on receiverTime if no senderTimestamp on only one side + check: + cmp(smallIndex1, noSenderTime1) < 0 + cmp(noSenderTime1, smallIndex1) > 0 # Test symmetry + cmp(noSenderTime2, eqIndex3) < 0 + cmp(eqIndex3, noSenderTime2) > 0 # Test symmetry + + test "Index equality": + # Exactly equal + check: + eqIndex1 == eqIndex2 + + # Receiver time plays no role, even without sender time + check: + eqIndex1 == eqIndex3 + noSenderTime1 == noSenderTime2 # only receiver time differs, indices are equal + noSenderTime1 != noSenderTime3 # pubsubTopics differ + noSenderTime1 != noSenderTime4 # digests differ + + # Unequal sender time + check: + smallIndex1 != largeIndex1 + + # Unequal digest + check: + smallIndex1 != smallIndex2 + + # Unequal hash and digest + check: + smallIndex1 != eqIndex1 + + # Unequal pubsubTopic + check: + smallIndex1 != diffPsTopic + + test "Index computation should not be empty": + ## Given + let ts = getTestTimestamp() + let wm = WakuMessage(payload: @[byte 1, 2, 3], timestamp: ts) + + ## When + let ts2 = getTestTimestamp() + 10 + let index = Index.compute(wm, ts2, DefaultContentTopic) + + ## Then + check: + index.digest.data.len != 0 + index.digest.data.len == 32 # sha2 output length in bytes + index.receiverTime == ts2 # the receiver timestamp should be a non-zero value + index.senderTime == ts + index.pubsubTopic == DefaultContentTopic + + test "Index digest of two identical messsage should be the same": + ## Given + let topic = ContentTopic("test-content-topic") + let + wm1 = WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic) + wm2 = WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic) + + ## When + let ts = getTestTimestamp() + let + index1 = Index.compute(wm1, ts, DefaultPubsubTopic) + index2 = Index.compute(wm2, ts, DefaultPubsubTopic) + + ## Then + check: + index1.digest == index2.digest diff --git a/tests/waku_archive_legacy/test_driver_queue_pagination.nim b/tests/waku_archive_legacy/test_driver_queue_pagination.nim new file mode 100644 index 000000000..05d9759a2 --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_queue_pagination.nim @@ -0,0 +1,405 @@ +{.used.} + +import + std/[options, sequtils, algorithm], testutils/unittests, libp2p/protobuf/minprotobuf +import + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver/queue_driver/queue_driver {.all.}, + waku/waku_archive_legacy/driver/queue_driver/index, + waku/waku_core, + ../testlib/wakucore + +proc getTestQueueDriver(numMessages: int): QueueDriver = + let testQueueDriver = QueueDriver.new(numMessages) + + var data {.noinit.}: array[32, byte] + for x in data.mitems: + x = 1 + + for i in 0 ..< numMessages: + let msg = WakuMessage(payload: @[byte i], timestamp: Timestamp(i)) + + let index = Index( + receiverTime: Timestamp(i), + senderTime: Timestamp(i), + digest: MessageDigest(data: data), + hash: computeMessageHash(DefaultPubsubTopic, msg), + ) + + discard testQueueDriver.add(index, msg) + + return testQueueDriver + +procSuite "Queue driver - pagination": + let driver = getTestQueueDriver(10) + let + indexList: seq[Index] = toSeq(driver.fwdIterator()).mapIt(it[0]) + msgList: seq[WakuMessage] = toSeq(driver.fwdIterator()).mapIt(it[1]) + + test "Forward pagination - normal pagination": + ## Given + let + pageSize: uint = 2 + cursor: Option[Index] = some(indexList[3]) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 2 + data == msgList[4 .. 5] + + test "Forward pagination - initial pagination request with an empty cursor": + ## Given + let + pageSize: uint = 2 + cursor: Option[Index] = none(Index) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 2 + data == msgList[0 .. 1] + + test "Forward pagination - initial pagination request with an empty cursor to fetch the entire history": + ## Given + let + pageSize: uint = 13 + cursor: Option[Index] = none(Index) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 10 + data == msgList[0 .. 9] + + test "Forward pagination - empty msgList": + ## Given + let driver = getTestQueueDriver(0) + let + pageSize: uint = 2 + cursor: Option[Index] = none(Index) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 0 + + test "Forward pagination - page size larger than the remaining messages": + ## Given + let + pageSize: uint = 10 + cursor: Option[Index] = some(indexList[3]) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 6 + data == msgList[4 .. 9] + + test "Forward pagination - page size larger than the maximum allowed page size": + ## Given + let + pageSize: uint = MaxPageSize + 1 + cursor: Option[Index] = some(indexList[3]) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + uint(data.len) <= MaxPageSize + + test "Forward pagination - cursor pointing to the end of the message list": + ## Given + let + pageSize: uint = 10 + cursor: Option[Index] = some(indexList[9]) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 0 + + test "Forward pagination - invalid cursor": + ## Given + let msg = fakeWakuMessage(payload = @[byte 10]) + let index = ArchiveCursor( + pubsubTopic: DefaultPubsubTopic, + senderTime: msg.timestamp, + storeTime: msg.timestamp, + digest: computeDigest(msg), + ).toIndex() + + let + pageSize: uint = 10 + cursor: Option[Index] = some(index) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let error = page.tryError() + check: + error == QueueDriverErrorKind.INVALID_CURSOR + + test "Forward pagination - initial paging query over a message list with one message": + ## Given + let driver = getTestQueueDriver(1) + let + pageSize: uint = 10 + cursor: Option[Index] = none(Index) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 1 + + test "Forward pagination - pagination over a message list with one message": + ## Given + let driver = getTestQueueDriver(1) + let + pageSize: uint = 10 + cursor: Option[Index] = some(indexList[0]) + forward: bool = true + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 0 + + test "Forward pagination - with pradicate": + ## Given + let + pageSize: uint = 3 + cursor: Option[Index] = none(Index) + forward = true + + proc onlyEvenTimes(index: Index, msg: WakuMessage): bool = + msg.timestamp.int64 mod 2 == 0 + + ## When + let page = driver.getPage( + pageSize = pageSize, forward = forward, cursor = cursor, predicate = onlyEvenTimes + ) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.mapIt(it.timestamp.int) == @[0, 2, 4] + + test "Backward pagination - normal pagination": + ## Given + let + pageSize: uint = 2 + cursor: Option[Index] = some(indexList[3]) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data == msgList[1 .. 2].reversed + + test "Backward pagination - empty msgList": + ## Given + let driver = getTestQueueDriver(0) + let + pageSize: uint = 2 + cursor: Option[Index] = none(Index) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 0 + + test "Backward pagination - initial pagination request with an empty cursor": + ## Given + let + pageSize: uint = 2 + cursor: Option[Index] = none(Index) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 2 + data == msgList[8 .. 9].reversed + + test "Backward pagination - initial pagination request with an empty cursor to fetch the entire history": + ## Given + let + pageSize: uint = 13 + cursor: Option[Index] = none(Index) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 10 + data == msgList[0 .. 9].reversed + + test "Backward pagination - page size larger than the remaining messages": + ## Given + let + pageSize: uint = 5 + cursor: Option[Index] = some(indexList[3]) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data == msgList[0 .. 2].reversed + + test "Backward pagination - page size larger than the Maximum allowed page size": + ## Given + let + pageSize: uint = MaxPageSize + 1 + cursor: Option[Index] = some(indexList[3]) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + uint(data.len) <= MaxPageSize + + test "Backward pagination - cursor pointing to the begining of the message list": + ## Given + let + pageSize: uint = 5 + cursor: Option[Index] = some(indexList[0]) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 0 + + test "Backward pagination - invalid cursor": + ## Given + let msg = fakeWakuMessage(payload = @[byte 10]) + let index = ArchiveCursor( + pubsubTopic: DefaultPubsubTopic, + senderTime: msg.timestamp, + storeTime: msg.timestamp, + digest: computeDigest(msg), + ).toIndex() + + let + pageSize: uint = 2 + cursor: Option[Index] = some(index) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let error = page.tryError() + check: + error == QueueDriverErrorKind.INVALID_CURSOR + + test "Backward pagination - initial paging query over a message list with one message": + ## Given + let driver = getTestQueueDriver(1) + let + pageSize: uint = 10 + cursor: Option[Index] = none(Index) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 1 + + test "Backward pagination - paging query over a message list with one message": + ## Given + let driver = getTestQueueDriver(1) + let + pageSize: uint = 10 + cursor: Option[Index] = some(indexList[0]) + forward: bool = false + + ## When + let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.len == 0 + + test "Backward pagination - with predicate": + ## Given + let + pageSize: uint = 3 + cursor: Option[Index] = none(Index) + forward = false + + proc onlyOddTimes(index: Index, msg: WakuMessage): bool = + msg.timestamp.int64 mod 2 != 0 + + ## When + let page = driver.getPage( + pageSize = pageSize, forward = forward, cursor = cursor, predicate = onlyOddTimes + ) + + ## Then + let data = page.tryGet().mapIt(it[1]) + check: + data.mapIt(it.timestamp.int) == @[5, 7, 9].reversed diff --git a/tests/waku_archive_legacy/test_driver_queue_query.nim b/tests/waku_archive_legacy/test_driver_queue_query.nim new file mode 100644 index 000000000..6bd44059b --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_queue_query.nim @@ -0,0 +1,1795 @@ +{.used.} + +import + std/[options, sequtils, random, algorithm], testutils/unittests, chronos, chronicles +import + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver/queue_driver, + waku/waku_core, + waku/waku_core/message/digest, + ../testlib/common, + ../testlib/wakucore + +logScope: + topics = "test archive queue_driver" + +# Initialize the random number generator +common.randomize() + +proc newTestSqliteDriver(): ArchiveDriver = + QueueDriver.new(capacity = 50) + +proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = + ArchiveCursor( + pubsubTopic: pubsubTopic, + senderTime: message.timestamp, + storeTime: message.timestamp, + digest: computeDigest(message), + hash: computeMessageHash(pubsubTopic, message), + ) + +suite "Queue driver - query by content topic": + test "no content topic": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages(maxPageSize = 5, ascendingOrder = true) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[0 .. 4] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "single content topic": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "single content topic - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[6 .. 7].reversed() + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "multiple content topic": + ## Given + const contentTopic1 = "test-content-topic-1" + const contentTopic2 = "test-content-topic-2" + const contentTopic3 = "test-content-topic-3" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic1, contentTopic2], + maxPageSize = 2, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "single content topic - no results": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "content topic and max page size - not enough messages stored": + ## Given + const pageSize: uint = 50 + + let driver = newTestSqliteDriver() + + for t in 0 ..< 40: + let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[DefaultContentTopic], + maxPageSize = pageSize, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 40 + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + +suite "SQLite driver - query by pubsub topic": + test "pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "no pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages(maxPageSize = 2, ascendingOrder = true) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[0 .. 1] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "content topic and pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + maxPageSize = 2, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + +suite "Queue driver - query by cursor": + test "only cursor": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = waitFor driver.getMessages( + cursor = some(cursor), maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[5 .. 6] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "only cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = waitFor driver.getMessages( + cursor = some(cursor), maxPageSize = 2, ascendingOrder = false + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3].reversed() + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "only cursor - invalid": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + var messages = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) + let cursor = ArchiveCursor(hash: fakeCursor) + + ## When + let res = waitFor driver.getMessages( + includeData = true, + contentTopic = @[DefaultContentTopic], + pubsubTopic = none(PubsubTopic), + cursor = some(cursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hashes = @[], + maxPageSize = 5, + ascendingOrder = true, + ) + + ## Then + check: + res.isErr() + res.error == "invalid_cursor" + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "content topic and cursor": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[5 .. 6] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "content topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 5].reversed() + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "pubsub topic and cursor": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + let cursor = computeTestCursor(expected[5][0], expected[5][1]) + + ## When + let res = waitFor driver.getMessages( + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[6 .. 7] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "pubsub topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + let cursor = computeTestCursor(expected[6][0], expected[6][1]) + + ## When + let res = waitFor driver.getMessages( + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5].reversed() + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + +suite "Queue driver - query by time range": + test "start time only": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "end time only": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + endTime = some(ts(45, timeOrigin)), maxPageSize = 10, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[0 .. 4] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "start time and end time": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + startTime = some(ts(15, timeOrigin)), + endTime = some(ts(45, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[2 .. 4] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "invalid time range - no results": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(45, timeOrigin)), + endTime = some(ts(15, timeOrigin)), + maxPageSize = 2, + ascendingOrder = true, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + asynctest "time range start and content topic": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6] + + ## Cleanup + (await driver.close()).expect("driver to close") + + test "time range start and content topic - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6].reversed() + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + asynctest "time range start, single content topic and cursor": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[3]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[4 .. 9] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asynctest "time range start, single content topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + let retFut = await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + require retFut.isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[3 .. 4].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + + test "time range, content topic, pubsub topic and cursor": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + let cursor = computeTestCursor(DefaultPubsubTopic, expected[1][1]) + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(0, timeOrigin)), + endTime = some(ts(45, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[3 .. 4] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "time range, content topic, pubsub topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + let cursor = computeTestCursor(expected[7][0], expected[7][1]) + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5].reversed() + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + let cursor = computeTestCursor(expected[1][0], expected[1][1]) + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range, descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newTestSqliteDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + let retFut = waitFor driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + require retFut.isOk() + + let cursor = computeTestCursor(expected[1][0], expected[1][1]) + + ## When + let res = waitFor driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + ## Cleanup + (waitFor driver.close()).expect("driver to close") diff --git a/tests/waku_archive_legacy/test_driver_sqlite.nim b/tests/waku_archive_legacy/test_driver_sqlite.nim new file mode 100644 index 000000000..af043116f --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_sqlite.nim @@ -0,0 +1,60 @@ +{.used.} + +import std/sequtils, testutils/unittests, chronos +import + waku/common/databases/db_sqlite, + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver/sqlite_driver, + waku/waku_core, + ../waku_archive_legacy/archive_utils, + ../testlib/common, + ../testlib/wakucore + +suite "SQLite driver": + test "init driver and database": + ## Given + let database = newSqliteDatabase() + + ## When + let driverRes = SqliteDriver.new(database) + + ## Then + check: + driverRes.isOk() + + let driver: ArchiveDriver = driverRes.tryGet() + check: + not driver.isNil() + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "insert a message": + ## Given + const contentTopic = "test-content-topic" + const meta = "test meta" + + let driver = newSqliteArchiveDriver() + + let msg = fakeWakuMessage(contentTopic = contentTopic, meta = meta) + let msgHash = computeMessageHash(DefaultPubsubTopic, msg) + + ## When + let putRes = waitFor driver.put( + DefaultPubsubTopic, msg, computeDigest(msg), msgHash, msg.timestamp + ) + + ## Then + check: + putRes.isOk() + + let storedMsg = (waitFor driver.getAllMessages()).tryGet() + check: + storedMsg.len == 1 + storedMsg.all do(item: auto) -> bool: + let (pubsubTopic, actualMsg, _, _, hash) = item + actualMsg.contentTopic == contentTopic and pubsubTopic == DefaultPubsubTopic and + hash == msgHash and msg.meta == actualMsg.meta + + ## Cleanup + (waitFor driver.close()).expect("driver to close") diff --git a/tests/waku_archive_legacy/test_driver_sqlite_query.nim b/tests/waku_archive_legacy/test_driver_sqlite_query.nim new file mode 100644 index 000000000..ecf88e7c0 --- /dev/null +++ b/tests/waku_archive_legacy/test_driver_sqlite_query.nim @@ -0,0 +1,1875 @@ +{.used.} + +import + std/[options, sequtils, random, algorithm], testutils/unittests, chronos, chronicles + +import + waku/common/databases/db_sqlite, + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver/sqlite_driver, + waku/waku_core, + waku/waku_core/message/digest, + ../testlib/common, + ../testlib/wakucore, + ../waku_archive_legacy/archive_utils + +logScope: + topics = "test archive _driver" + +# Initialize the random number generator +common.randomize() + +suite "SQLite driver - query by content topic": + asyncTest "no content topic": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages(maxPageSize = 5, ascendingOrder = true) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[0 .. 4] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "single content topic": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "single content topic with meta field": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), + fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" + ), + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" + ), + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" + ), + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" + ), + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" + ), + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "single content topic - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[6 .. 7].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "multiple content topic": + ## Given + const contentTopic1 = "test-content-topic-1" + const contentTopic2 = "test-content-topic-2" + const contentTopic3 = "test-content-topic-3" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic1, contentTopic2], + maxPageSize = 2, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "single content topic - no results": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "content topic and max page size - not enough messages stored": + ## Given + const pageSize: uint = 50 + + let driver = newSqliteArchiveDriver() + + for t in 0 ..< 40: + let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[DefaultContentTopic], + maxPageSize = pageSize, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 40 + + ## Cleanup + (await driver.close()).expect("driver to close") + +suite "SQLite driver - query by pubsub topic": + asyncTest "pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages( + pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "no pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages(maxPageSize = 2, ascendingOrder = true) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[0 .. 1] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "content topic and pubsub topic": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + maxPageSize = 2, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + ## Cleanup + (await driver.close()).expect("driver to close") + +suite "SQLite driver - query by cursor": + asyncTest "only cursor": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = await driver.getMessages( + cursor = some(cursor), maxPageSize = 2, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[5 .. 6] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "only cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = await driver.getMessages( + cursor = some(cursor), maxPageSize = 2, ascendingOrder = false + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 3].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "only cursor - invalid": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + var messages = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) + let cursor = ArchiveCursor(hash: fakeCursor) + + ## When + let res = await driver.getMessages( + includeData = true, + contentTopic = @[DefaultContentTopic], + pubsubTopic = none(PubsubTopic), + cursor = some(cursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hashes = @[], + maxPageSize = 5, + ascendingOrder = true, + ) + + ## Then + check: + res.isErr() + res.error == "cursor not found" + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "content topic and cursor": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[5 .. 6] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "content topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let expected = + @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[6]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 5].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "pubsub topic and cursor": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeArchiveCursor(expected[5][0], expected[5][1]) + + ## When + let res = await driver.getMessages( + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[6 .. 7] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "pubsub topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeArchiveCursor(expected[6][0], expected[6][1]) + + ## When + let res = await driver.getMessages( + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + +suite "SQLite driver - query by time range": + asyncTest "start time only": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "end time only": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + endTime = some(ts(45, timeOrigin)), maxPageSize = 10, ascendingOrder = true + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[0 .. 4] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "start time and end time": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + ## When + let res = await driver.getMessages( + startTime = some(ts(15, timeOrigin)), + endTime = some(ts(45, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[2 .. 4] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "invalid time range - no results": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(45, timeOrigin)), + endTime = some(ts(15, timeOrigin)), + maxPageSize = 2, + ascendingOrder = true, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range start and content topic": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range start and content topic - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[2 .. 6].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range start, single content topic and cursor": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[3]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[4 .. 9] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range start, single content topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", sequence = messages.mapIt(it.payload) + + for msg in messages: + require ( + await driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[6]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + cursor = some(cursor), + startTime = some(ts(15, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expected[3 .. 4].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range, content topic, pubsub topic and cursor": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[1][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(0, timeOrigin)), + endTime = some(ts(45, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[3 .. 4] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range, content topic, pubsub topic and cursor - descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeArchiveCursor(expected[7][0], expected[7][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5].reversed() + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeArchiveCursor(expected[1][0], expected[1][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = true, + ) + + ## Then + check: + res.isOk() + + let expectedMessages = expected.mapIt(it[1]) + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages == expectedMessages[4 .. 5] + + ## Cleanup + (await driver.close()).expect("driver to close") + + asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range, descending order": + ## Given + const contentTopic = "test-content-topic" + const pubsubTopic = "test-pubsub-topic" + + let driver = newSqliteArchiveDriver() + + let timeOrigin = now() + let expected = + @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) + ), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) + ), + ), + ( + pubsubTopic, + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) + ), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) + ), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage( + @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) + ), + ), + ] + var messages = expected + + shuffle(messages) + debug "randomized message insertion sequence", + sequence = messages.mapIt(it[1].payload) + + for row in messages: + let (topic, msg) = row + require ( + await driver.put( + topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp + ) + ).isOk() + + let cursor = computeArchiveCursor(expected[1][0], expected[1][1]) + + ## When + let res = await driver.getMessages( + contentTopic = @[contentTopic], + pubsubTopic = some(pubsubTopic), + cursor = some(cursor), + startTime = some(ts(35, timeOrigin)), + endTime = some(ts(85, timeOrigin)), + maxPageSize = 10, + ascendingOrder = false, + ) + + ## Then + check: + res.isOk() + + let filteredMessages = res.tryGet().mapIt(it[1]) + check: + filteredMessages.len == 0 + + ## Cleanup + (await driver.close()).expect("driver to close") diff --git a/tests/waku_archive_legacy/test_retention_policy.nim b/tests/waku_archive_legacy/test_retention_policy.nim new file mode 100644 index 000000000..b1c3832de --- /dev/null +++ b/tests/waku_archive_legacy/test_retention_policy.nim @@ -0,0 +1,169 @@ +{.used.} + +import std/[sequtils, times], stew/results, testutils/unittests, chronos +import + waku/common/databases/db_sqlite, + waku/waku_core, + waku/waku_core/message/digest, + waku/waku_archive_legacy, + waku/waku_archive_legacy/driver/sqlite_driver, + waku/waku_archive_legacy/retention_policy, + waku/waku_archive_legacy/retention_policy/retention_policy_capacity, + waku/waku_archive_legacy/retention_policy/retention_policy_size, + ../waku_archive_legacy/archive_utils, + ../testlib/common, + ../testlib/wakucore + +suite "Waku Archive - Retention policy": + test "capacity retention policy - windowed message deletion": + ## Given + let + capacity = 100 + excess = 60 + + let driver = newSqliteArchiveDriver() + + let retentionPolicy: RetentionPolicy = + CapacityRetentionPolicy.new(capacity = capacity) + var putFutures = newSeq[Future[ArchiveDriverResult[void]]]() + + ## When + for i in 1 .. capacity + excess: + let msg = fakeWakuMessage( + payload = @[byte i], contentTopic = DefaultContentTopic, ts = Timestamp(i) + ) + putFutures.add( + driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ) + + discard waitFor allFinished(putFutures) + + require (waitFor retentionPolicy.execute(driver)).isOk() + + ## Then + let numMessages = (waitFor driver.getMessagesCount()).tryGet() + check: + # Expected number of messages is 120 because + # (capacity = 100) + (half of the overflow window = 15) + (5 messages added after after the last delete) + # the window size changes when changing `const maxStoreOverflow = 1.3 in sqlite_store + numMessages == 115 + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "size retention policy - windowed message deletion": + ## Given + let + # in bytes + sizeLimit: int64 = 52428 + excess = 325 + + let driver = newSqliteArchiveDriver() + + let retentionPolicy: RetentionPolicy = SizeRetentionPolicy.new(size = sizeLimit) + var putFutures = newSeq[Future[ArchiveDriverResult[void]]]() + + # make sure that the db is empty to before test begins + let storedMsg = (waitFor driver.getAllMessages()).tryGet() + # if there are messages in db, empty them + if storedMsg.len > 0: + let now = getNanosecondTime(getTime().toUnixFloat()) + require (waitFor driver.deleteMessagesOlderThanTimestamp(ts = now)).isOk() + require (waitFor driver.performVacuum()).isOk() + + ## When + ## + + # create a number of messages so that the size of the DB overshoots + for i in 1 .. excess: + let msg = fakeWakuMessage( + payload = @[byte i], contentTopic = DefaultContentTopic, ts = Timestamp(i) + ) + putFutures.add( + driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ) + + # waitFor is used to synchronously wait for the futures to complete. + discard waitFor allFinished(putFutures) + + ## Then + # calculate the current database size + let sizeDB = int64((waitFor driver.getDatabaseSize()).tryGet()) + + # NOTE: since vacuumin is done manually, this needs to be revisited if vacuuming done automatically + + # get the rows count pre-deletion + let rowsCountBeforeDeletion = (waitFor driver.getMessagesCount()).tryGet() + + # execute policy provided the current db size oveflows, results in rows deletion + require (sizeDB >= sizeLimit) + require (waitFor retentionPolicy.execute(driver)).isOk() + + # get the number or rows from database + let rowCountAfterDeletion = (waitFor driver.getMessagesCount()).tryGet() + + check: + # size of the database is used to check if the storage limit has been preserved + # check the current database size with the limitSize provided by the user + # it should be lower + rowCountAfterDeletion <= rowsCountBeforeDeletion + + ## Cleanup + (waitFor driver.close()).expect("driver to close") + + test "store capacity should be limited": + ## Given + const capacity = 5 + const contentTopic = "test-content-topic" + + let + driver = newSqliteArchiveDriver() + retentionPolicy: RetentionPolicy = + CapacityRetentionPolicy.new(capacity = capacity) + + let messages = + @[ + fakeWakuMessage(contentTopic = DefaultContentTopic, ts = ts(0)), + fakeWakuMessage(contentTopic = DefaultContentTopic, ts = ts(1)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(2)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(3)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(4)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(5)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(6)), + ] + + ## When + for msg in messages: + require ( + waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + require (waitFor retentionPolicy.execute(driver)).isOk() + + ## Then + let storedMsg = (waitFor driver.getAllMessages()).tryGet() + check: + storedMsg.len == capacity + storedMsg.all do(item: auto) -> bool: + let (pubsubTopic, msg, _, _, _) = item + msg.contentTopic == contentTopic and pubsubTopic == DefaultPubsubTopic + + ## Cleanup + (waitFor driver.close()).expect("driver to close") diff --git a/tests/waku_archive_legacy/test_waku_archive.nim b/tests/waku_archive_legacy/test_waku_archive.nim new file mode 100644 index 000000000..181560a28 --- /dev/null +++ b/tests/waku_archive_legacy/test_waku_archive.nim @@ -0,0 +1,543 @@ +{.used.} + +import + std/[options, sequtils], + testutils/unittests, + chronicles, + chronos, + libp2p/crypto/crypto + +import + waku/common/databases/db_sqlite, + waku/common/paging, + waku/waku_core, + waku/waku_core/message/digest, + waku/waku_archive_legacy/driver/sqlite_driver, + waku/waku_archive_legacy, + ../waku_archive_legacy/archive_utils, + ../testlib/common, + ../testlib/wakucore + +suite "Waku Archive - message handling": + test "it should archive a valid and non-ephemeral message": + ## Setup + let driver = newSqliteArchiveDriver() + let archive = newWakuArchive(driver) + + ## Given + let validSenderTime = now() + let message = fakeWakuMessage(ephemeral = false, ts = validSenderTime) + + ## When + waitFor archive.handleMessage(DefaultPubSubTopic, message) + + ## Then + check: + (waitFor driver.getMessagesCount()).tryGet() == 1 + + test "it should not archive ephemeral messages": + ## Setup + let driver = newSqliteArchiveDriver() + let archive = newWakuArchive(driver) + + ## Given + let msgList = + @[ + fakeWakuMessage(ephemeral = false, payload = "1"), + fakeWakuMessage(ephemeral = true, payload = "2"), + fakeWakuMessage(ephemeral = true, payload = "3"), + fakeWakuMessage(ephemeral = true, payload = "4"), + fakeWakuMessage(ephemeral = false, payload = "5"), + ] + + ## When + for msg in msgList: + waitFor archive.handleMessage(DefaultPubsubTopic, msg) + + ## Then + check: + (waitFor driver.getMessagesCount()).tryGet() == 2 + + test "it should archive a message with no sender timestamp": + ## Setup + let driver = newSqliteArchiveDriver() + let archive = newWakuArchive(driver) + + ## Given + let invalidSenderTime = 0 + let message = fakeWakuMessage(ts = invalidSenderTime) + + ## When + waitFor archive.handleMessage(DefaultPubSubTopic, message) + + ## Then + check: + (waitFor driver.getMessagesCount()).tryGet() == 1 + + test "it should not archive a message with a sender time variance greater than max time variance (future)": + ## Setup + let driver = newSqliteArchiveDriver() + let archive = newWakuArchive(driver) + + ## Given + let + now = now() + invalidSenderTime = now + MaxMessageTimestampVariance + 1_000_000_000 + # 1 second over the max variance + + let message = fakeWakuMessage(ts = invalidSenderTime) + + ## When + waitFor archive.handleMessage(DefaultPubSubTopic, message) + + ## Then + check: + (waitFor driver.getMessagesCount()).tryGet() == 0 + + test "it should not archive a message with a sender time variance greater than max time variance (past)": + ## Setup + let driver = newSqliteArchiveDriver() + let archive = newWakuArchive(driver) + + ## Given + let + now = now() + invalidSenderTime = now - MaxMessageTimestampVariance - 1 + + let message = fakeWakuMessage(ts = invalidSenderTime) + + ## When + waitFor archive.handleMessage(DefaultPubSubTopic, message) + + ## Then + check: + (waitFor driver.getMessagesCount()).tryGet() == 0 + +procSuite "Waku Archive - find messages": + ## Fixtures + let timeOrigin = now() + let msgListA = + @[ + fakeWakuMessage( + @[byte 00], contentTopic = ContentTopic("2"), ts = ts(00, timeOrigin) + ), + fakeWakuMessage( + @[byte 01], contentTopic = ContentTopic("1"), ts = ts(10, timeOrigin) + ), + fakeWakuMessage( + @[byte 02], contentTopic = ContentTopic("2"), ts = ts(20, timeOrigin) + ), + fakeWakuMessage( + @[byte 03], contentTopic = ContentTopic("1"), ts = ts(30, timeOrigin) + ), + fakeWakuMessage( + @[byte 04], contentTopic = ContentTopic("2"), ts = ts(40, timeOrigin) + ), + fakeWakuMessage( + @[byte 05], contentTopic = ContentTopic("1"), ts = ts(50, timeOrigin) + ), + fakeWakuMessage( + @[byte 06], contentTopic = ContentTopic("2"), ts = ts(60, timeOrigin) + ), + fakeWakuMessage( + @[byte 07], contentTopic = ContentTopic("1"), ts = ts(70, timeOrigin) + ), + fakeWakuMessage( + @[byte 08], contentTopic = ContentTopic("2"), ts = ts(80, timeOrigin) + ), + fakeWakuMessage( + @[byte 09], contentTopic = ContentTopic("1"), ts = ts(90, timeOrigin) + ), + ] + + let archiveA = block: + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + for msg in msgListA: + require ( + waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + archive + + test "handle query": + ## Setup + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + let topic = ContentTopic("1") + let + msg1 = fakeWakuMessage(contentTopic = topic) + msg2 = fakeWakuMessage() + + waitFor archive.handleMessage("foo", msg1) + waitFor archive.handleMessage("foo", msg2) + + ## Given + let req = ArchiveQuery(includeData: true, contentTopics: @[topic]) + + ## When + let queryRes = waitFor archive.findMessages(req) + + ## Then + check: + queryRes.isOk() + + let response = queryRes.tryGet() + check: + response.messages.len == 1 + response.messages == @[msg1] + + test "handle query with multiple content filters": + ## Setup + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + let + topic1 = ContentTopic("1") + topic2 = ContentTopic("2") + topic3 = ContentTopic("3") + + let + msg1 = fakeWakuMessage(contentTopic = topic1) + msg2 = fakeWakuMessage(contentTopic = topic2) + msg3 = fakeWakuMessage(contentTopic = topic3) + + waitFor archive.handleMessage("foo", msg1) + waitFor archive.handleMessage("foo", msg2) + waitFor archive.handleMessage("foo", msg3) + + ## Given + let req = ArchiveQuery(includeData: true, contentTopics: @[topic1, topic3]) + + ## When + let queryRes = waitFor archive.findMessages(req) + + ## Then + check: + queryRes.isOk() + + let response = queryRes.tryGet() + check: + response.messages.len() == 2 + response.messages.anyIt(it == msg1) + response.messages.anyIt(it == msg3) + + test "handle query with more than 10 content filters": + ## Setup + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + let queryTopics = toSeq(1 .. 15).mapIt(ContentTopic($it)) + + ## Given + let req = ArchiveQuery(contentTopics: queryTopics) + + ## When + let queryRes = waitFor archive.findMessages(req) + + ## Then + check: + queryRes.isErr() + + let error = queryRes.tryError() + check: + error.kind == ArchiveErrorKind.INVALID_QUERY + error.cause == "too many content topics" + + test "handle query with pubsub topic filter": + ## Setup + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + let + pubsubTopic1 = "queried-topic" + pubsubTopic2 = "non-queried-topic" + + let + contentTopic1 = ContentTopic("1") + contentTopic2 = ContentTopic("2") + contentTopic3 = ContentTopic("3") + + let + msg1 = fakeWakuMessage(contentTopic = contentTopic1) + msg2 = fakeWakuMessage(contentTopic = contentTopic2) + msg3 = fakeWakuMessage(contentTopic = contentTopic3) + + waitFor archive.handleMessage(pubsubtopic1, msg1) + waitFor archive.handleMessage(pubsubtopic2, msg2) + waitFor archive.handleMessage(pubsubtopic2, msg3) + + ## Given + # This query targets: pubsubtopic1 AND (contentTopic1 OR contentTopic3) + let req = ArchiveQuery( + includeData: true, + pubsubTopic: some(pubsubTopic1), + contentTopics: @[contentTopic1, contentTopic3], + ) + + ## When + let queryRes = waitFor archive.findMessages(req) + + ## Then + check: + queryRes.isOk() + + let response = queryRes.tryGet() + check: + response.messages.len() == 1 + response.messages.anyIt(it == msg1) + + test "handle query with pubsub topic filter - no match": + ## Setup + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + let + pubsubtopic1 = "queried-topic" + pubsubtopic2 = "non-queried-topic" + + let + msg1 = fakeWakuMessage() + msg2 = fakeWakuMessage() + msg3 = fakeWakuMessage() + + waitFor archive.handleMessage(pubsubtopic2, msg1) + waitFor archive.handleMessage(pubsubtopic2, msg2) + waitFor archive.handleMessage(pubsubtopic2, msg3) + + ## Given + let req = ArchiveQuery(pubsubTopic: some(pubsubTopic1)) + + ## When + let res = waitFor archive.findMessages(req) + + ## Then + check: + res.isOk() + + let response = res.tryGet() + check: + response.messages.len() == 0 + + test "handle query with pubsub topic filter - match the entire stored messages": + ## Setup + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + let pubsubTopic = "queried-topic" + + let + msg1 = fakeWakuMessage(payload = "TEST-1") + msg2 = fakeWakuMessage(payload = "TEST-2") + msg3 = fakeWakuMessage(payload = "TEST-3") + + waitFor archive.handleMessage(pubsubTopic, msg1) + waitFor archive.handleMessage(pubsubTopic, msg2) + waitFor archive.handleMessage(pubsubTopic, msg3) + + ## Given + let req = ArchiveQuery(includeData: true, pubsubTopic: some(pubsubTopic)) + + ## When + let res = waitFor archive.findMessages(req) + + ## Then + check: + res.isOk() + + let response = res.tryGet() + check: + response.messages.len() == 3 + response.messages.anyIt(it == msg1) + response.messages.anyIt(it == msg2) + response.messages.anyIt(it == msg3) + + test "handle query with forward pagination": + ## Given + let req = + ArchiveQuery(includeData: true, pageSize: 4, direction: PagingDirection.FORWARD) + + ## When + var nextReq = req # copy + + var pages = newSeq[seq[WakuMessage]](3) + var cursors = newSeq[Option[ArchiveCursor]](3) + + for i in 0 ..< 3: + let res = waitFor archiveA.findMessages(nextReq) + require res.isOk() + + # Keep query response content + let response = res.get() + pages[i] = response.messages + cursors[i] = response.cursor + + # Set/update the request cursor + nextReq.cursor = cursors[i] + + ## Then + check: + cursors[0] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[3])) + cursors[1] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[7])) + cursors[2] == none(ArchiveCursor) + + check: + pages[0] == msgListA[0 .. 3] + pages[1] == msgListA[4 .. 7] + pages[2] == msgListA[8 .. 9] + + test "handle query with backward pagination": + ## Given + let req = + ArchiveQuery(includeData: true, pageSize: 4, direction: PagingDirection.BACKWARD) + + ## When + var nextReq = req # copy + + var pages = newSeq[seq[WakuMessage]](3) + var cursors = newSeq[Option[ArchiveCursor]](3) + + for i in 0 ..< 3: + let res = waitFor archiveA.findMessages(nextReq) + require res.isOk() + + # Keep query response content + let response = res.get() + pages[i] = response.messages + cursors[i] = response.cursor + + # Set/update the request cursor + nextReq.cursor = cursors[i] + + ## Then + check: + cursors[0] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[6])) + cursors[1] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[2])) + cursors[2] == none(ArchiveCursor) + + check: + pages[0] == msgListA[6 .. 9] + pages[1] == msgListA[2 .. 5] + pages[2] == msgListA[0 .. 1] + + test "handle query with no paging info - auto-pagination": + ## Setup + let + driver = newSqliteArchiveDriver() + archive = newWakuArchive(driver) + + let msgList = + @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2")), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 5], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 6], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 7], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 8], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2")), + ] + + for msg in msgList: + require ( + waitFor driver.put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + ).isOk() + + ## Given + let req = ArchiveQuery(includeData: true, contentTopics: @[DefaultContentTopic]) + + ## When + let res = waitFor archive.findMessages(req) + + ## Then + check: + res.isOk() + + let response = res.tryGet() + check: + ## No pagination specified. Response will be auto-paginated with + ## up to MaxPageSize messages per page. + response.messages.len() == 8 + response.cursor.isNone() + + test "handle temporal history query with a valid time window": + ## Given + let req = ArchiveQuery( + includeData: true, + contentTopics: @[ContentTopic("1")], + startTime: some(ts(15, timeOrigin)), + endTime: some(ts(55, timeOrigin)), + direction: PagingDirection.FORWARD, + ) + + ## When + let res = waitFor archiveA.findMessages(req) + + ## Then + check res.isOk() + + let response = res.tryGet() + check: + response.messages.len() == 2 + response.messages.mapIt(it.timestamp) == @[ts(30, timeOrigin), ts(50, timeOrigin)] + + test "handle temporal history query with a zero-size time window": + ## A zero-size window results in an empty list of history messages + ## Given + let req = ArchiveQuery( + contentTopics: @[ContentTopic("1")], + startTime: some(Timestamp(2)), + endTime: some(Timestamp(2)), + ) + + ## When + let res = waitFor archiveA.findMessages(req) + + ## Then + check res.isOk() + + let response = res.tryGet() + check: + response.messages.len == 0 + + test "handle temporal history query with an invalid time window": + ## A history query with an invalid time range results in an empty list of history messages + ## Given + let req = ArchiveQuery( + contentTopics: @[ContentTopic("1")], + startTime: some(Timestamp(5)), + endTime: some(Timestamp(2)), + ) + + ## When + let res = waitFor archiveA.findMessages(req) + + ## Then + check res.isOk() + + let response = res.tryGet() + check: + response.messages.len == 0 diff --git a/tests/waku_store/test_wakunode_store.nim b/tests/waku_store/test_wakunode_store.nim index a3c943627..1f48d18f2 100644 --- a/tests/waku_store/test_wakunode_store.nim +++ b/tests/waku_store/test_wakunode_store.nim @@ -62,12 +62,7 @@ procSuite "WakuNode - Store": for kv in kvs: let message = kv.message.get() - let msg_digest = computeDigest(message) - require ( - waitFor driver.put( - DefaultPubsubTopic, message, msg_digest, kv.messageHash, message.timestamp - ) - ).isOk() + require (waitFor driver.put(kv.messageHash, DefaultPubsubTopic, message)).isOk() driver diff --git a/tests/waku_store_legacy/test_resume.nim b/tests/waku_store_legacy/test_resume.nim index a7eacd0bf..53e48834e 100644 --- a/tests/waku_store_legacy/test_resume.nim +++ b/tests/waku_store_legacy/test_resume.nim @@ -13,15 +13,15 @@ when defined(waku_exp_store_resume): import waku/[ common/databases/db_sqlite, - waku_archive/driver, - waku_archive/driver/sqlite_driver/sqlite_driver, + waku_archive_legacy/driver, + waku_archive_legacy/driver/sqlite_driver/sqlite_driver, node/peer_manager, waku_core, waku_core/message/digest, waku_store_legacy, ], ../waku_store_legacy/store_utils, - ../waku_archive/archive_utils, + ../waku_archive_legacy/archive_utils, ./testlib/common, ./testlib/switch diff --git a/tests/waku_store_legacy/test_wakunode_store.nim b/tests/waku_store_legacy/test_wakunode_store.nim index e2fe7d5e7..496ab753e 100644 --- a/tests/waku_store_legacy/test_wakunode_store.nim +++ b/tests/waku_store_legacy/test_wakunode_store.nim @@ -1,7 +1,7 @@ {.used.} import - stew/shims/net as stewNet, + std/net, testutils/unittests, chronicles, chronos, @@ -9,27 +9,22 @@ import libp2p/peerid, libp2p/multiaddress, libp2p/switch, - libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub, libp2p/protocols/pubsub/gossipsub import waku/[ - common/databases/db_sqlite, common/paging, waku_core, waku_core/message/digest, - waku_core/subscription, node/peer_manager, - waku_archive, - waku_archive/driver/sqlite_driver, + waku_archive_legacy, waku_filter_v2, waku_filter_v2/client, waku_store_legacy, waku_node, ], ../waku_store_legacy/store_utils, - ../waku_archive/archive_utils, - ../testlib/common, + ../waku_archive_legacy/archive_utils, ../testlib/wakucore, ../testlib/wakunode @@ -54,7 +49,7 @@ procSuite "WakuNode - Store Legacy": let driver = newSqliteArchiveDriver() for msg in msgListA: - let msg_digest = waku_archive.computeDigest(msg) + let msg_digest = waku_archive_legacy.computeDigest(msg) let msg_hash = computeMessageHash(DefaultPubsubTopic, msg) require ( waitFor driver.put(DefaultPubsubTopic, msg, msg_digest, msg_hash, msg.timestamp) @@ -72,7 +67,7 @@ procSuite "WakuNode - Store Legacy": waitFor allFutures(client.start(), server.start()) - let mountArchiveRes = server.mountArchive(archiveA) + let mountArchiveRes = server.mountLegacyArchive(archiveA) assert mountArchiveRes.isOk(), mountArchiveRes.error waitFor server.mountLegacyStore() @@ -106,7 +101,7 @@ procSuite "WakuNode - Store Legacy": waitFor allFutures(client.start(), server.start()) - let mountArchiveRes = server.mountArchive(archiveA) + let mountArchiveRes = server.mountLegacyArchive(archiveA) assert mountArchiveRes.isOk(), mountArchiveRes.error waitFor server.mountLegacyStore() @@ -161,7 +156,7 @@ procSuite "WakuNode - Store Legacy": waitFor allFutures(client.start(), server.start()) - let mountArchiveRes = server.mountArchive(archiveA) + let mountArchiveRes = server.mountLegacyArchive(archiveA) assert mountArchiveRes.isOk(), mountArchiveRes.error waitFor server.mountLegacyStore() @@ -223,7 +218,7 @@ procSuite "WakuNode - Store Legacy": waitFor filterSource.mountFilter() let driver = newSqliteArchiveDriver() - let mountArchiveRes = server.mountArchive(driver) + let mountArchiveRes = server.mountLegacyArchive(driver) assert mountArchiveRes.isOk(), mountArchiveRes.error waitFor server.mountLegacyStore() @@ -241,7 +236,7 @@ procSuite "WakuNode - Store Legacy": proc filterHandler( pubsubTopic: PubsubTopic, msg: WakuMessage ) {.async, gcsafe, closure.} = - await server.wakuArchive.handleMessage(pubsubTopic, msg) + await server.wakuLegacyArchive.handleMessage(pubsubTopic, msg) filterFut.complete((pubsubTopic, msg)) server.wakuFilterClient.registerPushHandler(filterHandler) @@ -286,7 +281,7 @@ procSuite "WakuNode - Store Legacy": waitFor allFutures(client.start(), server.start()) - let mountArchiveRes = server.mountArchive(archiveA) + let mountArchiveRes = server.mountLegacyArchive(archiveA) assert mountArchiveRes.isOk(), mountArchiveRes.error waitFor server.mountLegacyStore() @@ -302,7 +297,7 @@ procSuite "WakuNode - Store Legacy": pubsubTopic: "pubsubTopic", senderTime: now(), storeTime: now(), - digest: waku_archive.MessageDigest(data: data), + digest: waku_archive_legacy.MessageDigest(data: data), ) ## Given diff --git a/tests/wakunode_rest/test_rest_store.nim b/tests/wakunode_rest/test_rest_store.nim index 030fae2dc..32e8151db 100644 --- a/tests/wakunode_rest/test_rest_store.nim +++ b/tests/wakunode_rest/test_rest_store.nim @@ -40,16 +40,9 @@ logScope: proc put( store: ArchiveDriver, pubsubTopic: PubsubTopic, message: WakuMessage ): Future[Result[void, string]] = - let - digest = computeDigest(message) - msgHash = computeMessageHash(pubsubTopic, message) - receivedTime = - if message.timestamp > 0: - message.timestamp - else: - getNowInNanosecondTime() + let msgHash = computeMessageHash(pubsubTopic, message) - store.put(pubsubTopic, message, digest, msgHash, receivedTime) + store.put(msgHash, pubsubTopic, message) # Creates a new WakuNode proc testWakuNode(): WakuNode = diff --git a/waku/factory/external_config.nim b/waku/factory/external_config.nim index 8ce67bce4..71786ceb8 100644 --- a/waku/factory/external_config.nim +++ b/waku/factory/external_config.nim @@ -335,6 +335,12 @@ type WakuNodeConf* = object desc: "Enable/disable waku store protocol", defaultValue: false, name: "store" .}: bool + legacyStore* {. + desc: "Enable/disable waku store legacy mode", + defaultValue: true, + name: "legacy-store" + .}: bool + storenode* {. desc: "Peer multiaddress to query for storage", defaultValue: "", diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index ad9719ec9..53995a0ff 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -17,7 +17,14 @@ import ../waku_core, ../waku_rln_relay, ../discovery/waku_dnsdisc, - ../waku_archive, + ../waku_archive/retention_policy as policy, + ../waku_archive/retention_policy/builder as policy_builder, + ../waku_archive/driver as driver, + ../waku_archive/driver/builder as driver_builder, + ../waku_archive_legacy/retention_policy as legacy_policy, + ../waku_archive_legacy/retention_policy/builder as legacy_policy_builder, + ../waku_archive_legacy/driver as legacy_driver, + ../waku_archive_legacy/driver/builder as legacy_driver_builder, ../waku_store, ../waku_store/common as store_common, ../waku_store_legacy, @@ -28,8 +35,6 @@ import ../node/peer_manager/peer_store/waku_peer_storage, ../node/peer_manager/peer_store/migrations as peer_store_sqlite_migrations, ../waku_lightpush/common, - ../waku_archive/driver/builder, - ../waku_archive/retention_policy/builder, ../common/utils/parse_size_units, ../common/ratelimit @@ -219,15 +224,36 @@ proc setupProtocols( return err("failed to mount waku RLN relay protocol: " & getCurrentExceptionMsg()) if conf.store: - # Archive setup - let archiveDriverRes = waitFor ArchiveDriver.new( + if conf.legacyStore: + let archiveDriverRes = waitFor legacy_driver.ArchiveDriver.new( + conf.storeMessageDbUrl, conf.storeMessageDbVacuum, conf.storeMessageDbMigration, + conf.storeMaxNumDbConnections, onFatalErrorAction, + ) + if archiveDriverRes.isErr(): + return err("failed to setup legacy archive driver: " & archiveDriverRes.error) + + let retPolicyRes = + legacy_policy.RetentionPolicy.new(conf.storeMessageRetentionPolicy) + if retPolicyRes.isErr(): + return err("failed to create retention policy: " & retPolicyRes.error) + + let mountArcRes = + node.mountLegacyArchive(archiveDriverRes.get(), retPolicyRes.get()) + if mountArcRes.isErr(): + return err("failed to mount waku legacy archive protocol: " & mountArcRes.error) + + ## For now we always mount the future archive driver but if the legacy one is mounted, + ## then the legacy will be in charge of performing the archiving. + ## Regarding storage, the only diff between the current/future archive driver and the legacy + ## one, is that the legacy stores an extra field: the id (message digest.) + let archiveDriverRes = waitFor driver.ArchiveDriver.new( conf.storeMessageDbUrl, conf.storeMessageDbVacuum, conf.storeMessageDbMigration, conf.storeMaxNumDbConnections, onFatalErrorAction, ) if archiveDriverRes.isErr(): return err("failed to setup archive driver: " & archiveDriverRes.error) - let retPolicyRes = RetentionPolicy.new(conf.storeMessageRetentionPolicy) + let retPolicyRes = policy.RetentionPolicy.new(conf.storeMessageRetentionPolicy) if retPolicyRes.isErr(): return err("failed to create retention policy: " & retPolicyRes.error) @@ -235,20 +261,23 @@ proc setupProtocols( if mountArcRes.isErr(): return err("failed to mount waku archive protocol: " & mountArcRes.error) - # Store setup let rateLimitSetting: RateLimitSetting = (conf.requestRateLimit, chronos.seconds(conf.requestRatePeriod)) + + if conf.legacyStore: + # Store legacy setup + try: + await mountLegacyStore(node, rateLimitSetting) + except CatchableError: + return + err("failed to mount waku legacy store protocol: " & getCurrentExceptionMsg()) + + # Store setup try: await mountStore(node, rateLimitSetting) except CatchableError: return err("failed to mount waku store protocol: " & getCurrentExceptionMsg()) - try: - await mountLegacyStore(node, rateLimitSetting) - except CatchableError: - return - err("failed to mount waku legacy store protocol: " & getCurrentExceptionMsg()) - mountStoreClient(node) if conf.storenode != "": let storeNode = parsePeerInfo(conf.storenode) diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 4f91fde63..23d9799c3 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -27,6 +27,7 @@ import ../waku_core/topics/sharding, ../waku_relay, ../waku_archive, + ../waku_archive_legacy, ../waku_store_legacy/protocol as legacy_store, ../waku_store_legacy/client as legacy_store_client, ../waku_store_legacy/common as legacy_store_common, @@ -87,7 +88,8 @@ type peerManager*: PeerManager switch*: Switch wakuRelay*: WakuRelay - wakuArchive*: WakuArchive + wakuArchive*: waku_archive.WakuArchive + wakuLegacyArchive*: waku_archive_legacy.WakuArchive wakuLegacyStore*: legacy_store.WakuStore wakuLegacyStoreClient*: legacy_store_client.WakuStoreClient wakuStore*: store.WakuStore @@ -244,6 +246,11 @@ proc registerRelayDefaultHandler(node: WakuNode, topic: PubsubTopic) = await node.wakuFilter.handleMessage(topic, msg) proc archiveHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + if not node.wakuLegacyArchive.isNil(): + ## we try to store with legacy archive + await node.wakuLegacyArchive.handleMessage(topic, msg) + return + if node.wakuArchive.isNil(): return @@ -675,25 +682,45 @@ proc filterUnsubscribeAll*( ## Waku archive proc mountArchive*( - node: WakuNode, driver: ArchiveDriver, retentionPolicy = none(RetentionPolicy) + node: WakuNode, + driver: waku_archive.ArchiveDriver, + retentionPolicy = none(waku_archive.RetentionPolicy), ): Result[void, string] = - node.wakuArchive = WakuArchive.new(driver = driver, retentionPolicy = retentionPolicy).valueOr: + node.wakuArchive = waku_archive.WakuArchive.new( + driver = driver, retentionPolicy = retentionPolicy + ).valueOr: return err("error in mountArchive: " & error) node.wakuArchive.start() return ok() +proc mountLegacyArchive*( + node: WakuNode, + driver: waku_archive_legacy.ArchiveDriver, + retentionPolicy = none(waku_archive_legacy.RetentionPolicy), +): Result[void, string] = + node.wakuLegacyArchive = waku_archive_legacy.WakuArchive.new( + driver = driver, retentionPolicy = retentionPolicy + ).valueOr: + return err("error in mountLegacyArchive: " & error) + + node.wakuLegacyArchive.start() + + return ok() + ## Legacy Waku Store # TODO: Review this mapping logic. Maybe, move it to the appplication code -proc toArchiveQuery(request: legacy_store_common.HistoryQuery): ArchiveQuery = - ArchiveQuery( +proc toArchiveQuery( + request: legacy_store_common.HistoryQuery +): waku_archive_legacy.ArchiveQuery = + waku_archive_legacy.ArchiveQuery( pubsubTopic: request.pubsubTopic, contentTopics: request.contentTopics, cursor: request.cursor.map( - proc(cursor: HistoryCursor): ArchiveCursor = - ArchiveCursor( + proc(cursor: HistoryCursor): waku_archive_legacy.ArchiveCursor = + waku_archive_legacy.ArchiveCursor( pubsubTopic: cursor.pubsubTopic, senderTime: cursor.senderTime, storeTime: cursor.storeTime, @@ -707,11 +734,14 @@ proc toArchiveQuery(request: legacy_store_common.HistoryQuery): ArchiveQuery = ) # TODO: Review this mapping logic. Maybe, move it to the appplication code -proc toHistoryResult*(res: ArchiveResult): legacy_store_common.HistoryResult = +proc toHistoryResult*( + res: waku_archive_legacy.ArchiveResult +): legacy_store_common.HistoryResult = if res.isErr(): let error = res.error case res.error.kind - of ArchiveErrorKind.DRIVER_ERROR, ArchiveErrorKind.INVALID_QUERY: + of waku_archive_legacy.ArchiveErrorKind.DRIVER_ERROR, + waku_archive_legacy.ArchiveErrorKind.INVALID_QUERY: err(HistoryError(kind: HistoryErrorKind.BAD_REQUEST, cause: res.error.cause)) else: err(HistoryError(kind: HistoryErrorKind.UNKNOWN)) @@ -721,7 +751,7 @@ proc toHistoryResult*(res: ArchiveResult): legacy_store_common.HistoryResult = HistoryResponse( messages: response.messages, cursor: response.cursor.map( - proc(cursor: ArchiveCursor): HistoryCursor = + proc(cursor: waku_archive_legacy.ArchiveCursor): HistoryCursor = HistoryCursor( pubsubTopic: cursor.pubsubTopic, senderTime: cursor.senderTime, @@ -737,7 +767,7 @@ proc mountLegacyStore*( ) {.async.} = info "mounting waku legacy store protocol" - if node.wakuArchive.isNil(): + if node.wakuLegacyArchive.isNil(): error "failed to mount waku legacy store protocol", error = "waku archive not set" return @@ -750,7 +780,7 @@ proc mountLegacyStore*( return err(error) let request = request.toArchiveQuery() - let response = await node.wakuArchive.findMessagesV2(request) + let response = await node.wakuLegacyArchive.findMessagesV2(request) return response.toHistoryResult() node.wakuLegacyStore = legacy_store.WakuStore.new( @@ -831,8 +861,8 @@ when defined(waku_exp_store_resume): ## Waku Store -proc toArchiveQuery(request: StoreQueryRequest): ArchiveQuery = - var query = ArchiveQuery() +proc toArchiveQuery(request: StoreQueryRequest): waku_archive.ArchiveQuery = + var query = waku_archive.ArchiveQuery() query.includeData = request.includeData query.pubsubTopic = request.pubsubTopic @@ -840,12 +870,7 @@ proc toArchiveQuery(request: StoreQueryRequest): ArchiveQuery = query.startTime = request.startTime query.endTime = request.endTime query.hashes = request.messageHashes - - if request.paginationCursor.isSome(): - var cursor = ArchiveCursor() - cursor.hash = request.paginationCursor.get() - query.cursor = some(cursor) - + query.cursor = request.paginationCursor query.direction = request.paginationForward if request.paginationLimit.isSome(): @@ -853,7 +878,7 @@ proc toArchiveQuery(request: StoreQueryRequest): ArchiveQuery = return query -proc toStoreResult(res: ArchiveResult): StoreQueryResult = +proc toStoreResult(res: waku_archive.ArchiveResult): StoreQueryResult = let response = res.valueOr: return err(StoreError.new(300, "archive error: " & $error)) @@ -873,8 +898,7 @@ proc toStoreResult(res: ArchiveResult): StoreQueryResult = res.messages[i].message = some(response.messages[i]) res.messages[i].pubsubTopic = some(response.topics[i]) - if response.cursor.isSome(): - res.paginationCursor = some(response.cursor.get().hash) + res.paginationCursor = response.cursor return ok(res) diff --git a/waku/waku_archive/archive.nim b/waku/waku_archive/archive.nim index 70caf78cc..ed4bb5b0b 100644 --- a/waku/waku_archive/archive.nim +++ b/waku/waku_archive/archive.nim @@ -1,7 +1,7 @@ {.push raises: [].} import - std/[times, options, sequtils, strutils, algorithm], + std/[times, options, sequtils, algorithm], stew/[results, byteutils], chronicles, chronos, @@ -52,9 +52,6 @@ proc validate*(msg: WakuMessage): Result[void, string] = # Ephemeral message, do not store return - if msg.timestamp == 0: - return ok() - let now = getNanosecondTime(getTime().toUnixFloat()) lowerBound = now - MaxMessageTimestampVariance @@ -89,38 +86,24 @@ proc handleMessage*( waku_archive_errors.inc(labelValues = [error]) return - let - msgDigest = computeDigest(msg) - msgDigestHex = msgDigest.data.to0xHex() - msgHash = computeMessageHash(pubsubTopic, msg) - msgHashHex = msgHash.to0xHex() - msgTimestamp = - if msg.timestamp > 0: - msg.timestamp - else: - getNanosecondTime(getTime().toUnixFloat()) - - notice "archive handling message", - msg_hash = msgHashHex, - pubsubTopic = pubsubTopic, - contentTopic = msg.contentTopic, - msgTimestamp = msg.timestamp, - usedTimestamp = msgTimestamp, - digest = msgDigestHex + let msgHash = computeMessageHash(pubsubTopic, msg) let insertStartTime = getTime().toUnixFloat() - (await self.driver.put(pubsubTopic, msg, msgDigest, msgHash, msgTimestamp)).isOkOr: + (await self.driver.put(msgHash, pubsubTopic, msg)).isOkOr: waku_archive_errors.inc(labelValues = [insertFailure]) - error "failed to insert message", error = error + trace "failed to insert message", + hash_hash = msgHash.to0xHex(), + pubsubTopic = pubsubTopic, + contentTopic = msg.contentTopic, + timestamp = msg.timestamp, + error = error notice "message archived", - msg_hash = msgHashHex, + hash_hash = msgHash.to0xHex(), pubsubTopic = pubsubTopic, contentTopic = msg.contentTopic, - msgTimestamp = msg.timestamp, - usedTimestamp = msgTimestamp, - digest = msgDigestHex + timestamp = msg.timestamp let insertDuration = getTime().toUnixFloat() - insertStartTime waku_archive_insert_duration_seconds.observe(insertDuration) @@ -130,6 +113,16 @@ proc findMessages*( ): Future[ArchiveResult] {.async, gcsafe.} = ## Search the archive to return a single page of messages matching the query criteria + if query.cursor.isSome(): + let cursor = query.cursor.get() + + if cursor.len != 32: + return + err(ArchiveError.invalidQuery("invalid cursor hash length: " & $cursor.len)) + + if cursor == EmptyWakuMessageHash: + return err(ArchiveError.invalidQuery("all zeroes cursor hash")) + let maxPageSize = if query.pageSize <= 0: DefaultPageSize @@ -138,18 +131,12 @@ proc findMessages*( let isAscendingOrder = query.direction.into() - if query.contentTopics.len > 100: - return err(ArchiveError.invalidQuery("too many content topics")) - - if query.cursor.isSome() and query.cursor.get().hash.len != 32: - return err(ArchiveError.invalidQuery("invalid cursor hash length")) - let queryStartTime = getTime().toUnixFloat() let rows = ( await self.driver.getMessages( includeData = query.includeData, - contentTopic = query.contentTopics, + contentTopics = query.contentTopics, pubsubTopic = query.pubsubTopic, cursor = query.cursor, startTime = query.startTime, @@ -160,7 +147,6 @@ proc findMessages*( ) ).valueOr: return err(ArchiveError(kind: ArchiveErrorKind.DRIVER_ERROR, cause: error)) - let queryDuration = getTime().toUnixFloat() - queryStartTime waku_archive_query_duration_seconds.observe(queryDuration) @@ -172,115 +158,33 @@ proc findMessages*( if rows.len == 0: return ok(ArchiveResponse(hashes: hashes, messages: messages, cursor: cursor)) - ## Messages let pageSize = min(rows.len, int(maxPageSize)) - #TODO once store v2 is removed, unzip instead of 2x map - #TODO once store v2 is removed, update driver to not return messages when not needed + hashes = rows[0 ..< pageSize].mapIt(it[0]) + if query.includeData: - topics = rows[0 ..< pageSize].mapIt(it[0]) - messages = rows[0 ..< pageSize].mapIt(it[1]) + topics = rows[0 ..< pageSize].mapIt(it[1]) + messages = rows[0 ..< pageSize].mapIt(it[2]) - hashes = rows[0 ..< pageSize].mapIt(it[4]) - - ## Cursor if rows.len > int(maxPageSize): ## Build last message cursor ## The cursor is built from the last message INCLUDED in the response ## (i.e. the second last message in the rows list) - #TODO Once Store v2 is removed keep only message and hash - let (pubsubTopic, message, digest, storeTimestamp, hash) = rows[^2] + let (hash, _, _) = rows[^2] - #TODO Once Store v2 is removed, the cursor becomes the hash of the last message - cursor = some( - ArchiveCursor( - digest: MessageDigest.fromBytes(digest), - storeTime: storeTimestamp, - sendertime: message.timestamp, - pubsubTopic: pubsubTopic, - hash: hash, - ) - ) + cursor = some(hash) - # All messages MUST be returned in chronological order + # Messages MUST be returned in chronological order if not isAscendingOrder: reverse(hashes) - reverse(messages) reverse(topics) + reverse(messages) return ok( - ArchiveResponse(hashes: hashes, messages: messages, topics: topics, cursor: cursor) + ArchiveResponse(cursor: cursor, topics: topics, hashes: hashes, messages: messages) ) -proc findMessagesV2*( - self: WakuArchive, query: ArchiveQuery -): Future[ArchiveResult] {.async, deprecated, gcsafe.} = - ## Search the archive to return a single page of messages matching the query criteria - - let maxPageSize = - if query.pageSize <= 0: - DefaultPageSize - else: - min(query.pageSize, MaxPageSize) - - let isAscendingOrder = query.direction.into() - - if query.contentTopics.len > 100: - return err(ArchiveError.invalidQuery("too many content topics")) - - let queryStartTime = getTime().toUnixFloat() - - let rows = ( - await self.driver.getMessagesV2( - contentTopic = query.contentTopics, - pubsubTopic = query.pubsubTopic, - cursor = query.cursor, - startTime = query.startTime, - endTime = query.endTime, - maxPageSize = maxPageSize + 1, - ascendingOrder = isAscendingOrder, - ) - ).valueOr: - return err(ArchiveError(kind: ArchiveErrorKind.DRIVER_ERROR, cause: error)) - - let queryDuration = getTime().toUnixFloat() - queryStartTime - waku_archive_query_duration_seconds.observe(queryDuration) - - var messages = newSeq[WakuMessage]() - var cursor = none(ArchiveCursor) - - if rows.len == 0: - return ok(ArchiveResponse(messages: messages, cursor: cursor)) - - ## Messages - let pageSize = min(rows.len, int(maxPageSize)) - - messages = rows[0 ..< pageSize].mapIt(it[1]) - - ## Cursor - if rows.len > int(maxPageSize): - ## Build last message cursor - ## The cursor is built from the last message INCLUDED in the response - ## (i.e. the second last message in the rows list) - - let (pubsubTopic, message, digest, storeTimestamp, _) = rows[^2] - - cursor = some( - ArchiveCursor( - digest: MessageDigest.fromBytes(digest), - storeTime: storeTimestamp, - sendertime: message.timestamp, - pubsubTopic: pubsubTopic, - ) - ) - - # All messages MUST be returned in chronological order - if not isAscendingOrder: - reverse(messages) - - return ok(ArchiveResponse(messages: messages, cursor: cursor)) - proc periodicRetentionPolicy(self: WakuArchive) {.async.} = debug "executing message retention policy" diff --git a/waku/waku_archive/common.nim b/waku/waku_archive/common.nim index b0c018ab0..b88b70f05 100644 --- a/waku/waku_archive/common.nim +++ b/waku/waku_archive/common.nim @@ -3,44 +3,13 @@ import std/options, results, stew/byteutils, stew/arrayops, nimcrypto/sha2 import ../waku_core, ../common/paging -## Waku message digest - -type MessageDigest* = MDigest[256] - -proc fromBytes*(T: type MessageDigest, src: seq[byte]): T = - var data: array[32, byte] - - let byteCount = copyFrom[byte](data, src) - - assert byteCount == 32 - - return MessageDigest(data: data) - -proc computeDigest*(msg: WakuMessage): MessageDigest = - var ctx: sha256 - ctx.init() - defer: - ctx.clear() - - ctx.update(msg.contentTopic.toBytes()) - ctx.update(msg.payload) - - # Computes the hash - return ctx.finish() - ## Public API types type - #TODO Once Store v2 is removed, the cursor becomes the hash of the last message - ArchiveCursor* = object - digest*: MessageDigest - storeTime*: Timestamp - senderTime*: Timestamp - pubsubTopic*: PubsubTopic - hash*: WakuMessageHash + ArchiveCursor* = WakuMessageHash ArchiveQuery* = object - includeData*: bool # indicate if messages should be returned in addition to hashes. + includeData*: bool pubsubTopic*: Option[PubsubTopic] contentTopics*: seq[ContentTopic] cursor*: Option[ArchiveCursor] diff --git a/waku/waku_archive/driver.nim b/waku/waku_archive/driver.nim index a70b688bc..49174b571 100644 --- a/waku/waku_archive/driver.nim +++ b/waku/waku_archive/driver.nim @@ -9,18 +9,15 @@ type ArchiveDriverResult*[T] = Result[T, string] ArchiveDriver* = ref object of RootObj -#TODO Once Store v2 is removed keep only messages and hashes -type ArchiveRow* = (PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash) +type ArchiveRow* = (WakuMessageHash, PubsubTopic, WakuMessage) # ArchiveDriver interface method put*( driver: ArchiveDriver, + messageHash: WakuMessageHash, pubsubTopic: PubsubTopic, message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, ): Future[ArchiveDriverResult[void]] {.base, async.} = discard @@ -29,22 +26,10 @@ method getAllMessages*( ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, async.} = discard -method getMessagesV2*( - driver: ArchiveDriver, - contentTopic = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - maxPageSize = DefaultPageSize, - ascendingOrder = true, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, deprecated, async.} = - discard - method getMessages*( driver: ArchiveDriver, - includeData = false, - contentTopic = newSeq[ContentTopic](0), + includeData = true, + contentTopics = newSeq[ContentTopic](0), pubsubTopic = none(PubsubTopic), cursor = none(ArchiveCursor), startTime = none(Timestamp), diff --git a/waku/waku_archive/driver/postgres_driver/migrations.nim b/waku/waku_archive/driver/postgres_driver/migrations.nim index 976c5af9e..f99146bbf 100644 --- a/waku/waku_archive/driver/postgres_driver/migrations.nim +++ b/waku/waku_archive/driver/postgres_driver/migrations.nim @@ -9,7 +9,7 @@ import logScope: topics = "waku archive migration" -const SchemaVersion* = 5 # increase this when there is an update in the database schema +const SchemaVersion* = 6 # increase this when there is an update in the database schema proc breakIntoStatements*(script: string): seq[string] = ## Given a full migration script, that can potentially contain a list diff --git a/waku/waku_archive/driver/postgres_driver/partitions_manager.nim b/waku/waku_archive/driver/postgres_driver/partitions_manager.nim index 3ecf88fa1..0591209ce 100644 --- a/waku/waku_archive/driver/postgres_driver/partitions_manager.nim +++ b/waku/waku_archive/driver/postgres_driver/partitions_manager.nim @@ -1,7 +1,7 @@ ## This module is aimed to handle the creation and truncation of partition tables ## in order to limit the space occupied in disk by the database. ## -## The created partitions are referenced by the 'storedAt' field. +## The created partitions are referenced by the 'timestamp' field. ## import std/[deques, times] diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index 48e06409a..975780e81 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -26,102 +26,101 @@ type PostgresDriver* = ref object of ArchiveDriver futLoopPartitionFactory: Future[void] const InsertRowStmtName = "InsertRow" -const InsertRowStmtDefinition = # TODO: get the sql queries from a file - """INSERT INTO messages (id, messageHash, storedAt, contentTopic, payload, pubsubTopic, - version, timestamp, meta) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, CASE WHEN $9 = '' THEN NULL ELSE $9 END) ON CONFLICT DO NOTHING;""" +const InsertRowStmtDefinition = + """INSERT INTO messages (id, messageHash, pubsubTopic, contentTopic, payload, + version, timestamp, meta) VALUES ($1, $2, $3, $4, $5, $6, $7, CASE WHEN $8 = '' THEN NULL ELSE $8 END) ON CONFLICT DO NOTHING;""" + +const SelectClause = + """SELECT messageHash, pubsubTopic, contentTopic, payload, version, timestamp, meta FROM messages """ const SelectNoCursorAscStmtName = "SelectWithoutCursorAsc" -const SelectClause = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages """ const SelectNoCursorAscStmtDef = SelectClause & - """ - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - storedAt >= $4 AND - storedAt <= $5 - ORDER BY storedAt ASC, messageHash ASC LIMIT $6;""" + """WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + timestamp >= $4 AND + timestamp <= $5 + ORDER BY timestamp ASC, messageHash ASC LIMIT $6;""" + +const SelectNoCursorNoDataAscStmtName = "SelectWithoutCursorAndDataAsc" +const SelectNoCursorNoDataAscStmtDef = + """SELECT messageHash FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + timestamp >= $4 AND + timestamp <= $5 + ORDER BY timestamp ASC, messageHash ASC LIMIT $6;""" const SelectNoCursorDescStmtName = "SelectWithoutCursorDesc" const SelectNoCursorDescStmtDef = SelectClause & - """ - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - storedAt >= $4 AND - storedAt <= $5 - ORDER BY storedAt DESC, messageHash DESC LIMIT $6;""" + """WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + timestamp >= $4 AND + timestamp <= $5 + ORDER BY timestamp DESC, messageHash DESC LIMIT $6;""" + +const SelectNoCursorNoDataDescStmtName = "SelectWithoutCursorAndDataDesc" +const SelectNoCursorNoDataDescStmtDef = + """SELECT messageHash FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + timestamp >= $4 AND + timestamp <= $5 + ORDER BY timestamp DESC, messageHash DESC LIMIT $6;""" const SelectWithCursorDescStmtName = "SelectWithCursorDesc" const SelectWithCursorDescStmtDef = SelectClause & - """ - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - (storedAt, messageHash) < ($4,$5) AND - storedAt >= $6 AND - storedAt <= $7 - ORDER BY storedAt DESC, messageHash DESC LIMIT $8;""" + """WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + (timestamp, messageHash) < ($4,$5) AND + timestamp >= $6 AND + timestamp <= $7 + ORDER BY timestamp DESC, messageHash DESC LIMIT $8;""" + +const SelectWithCursorNoDataDescStmtName = "SelectWithCursorNoDataDesc" +const SelectWithCursorNoDataDescStmtDef = + """SELECT messageHash FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + (timestamp, messageHash) < ($4,$5) AND + timestamp >= $6 AND + timestamp <= $7 + ORDER BY timestamp DESC, messageHash DESC LIMIT $8;""" const SelectWithCursorAscStmtName = "SelectWithCursorAsc" const SelectWithCursorAscStmtDef = SelectClause & - """ - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - (storedAt, messageHash) > ($4,$5) AND - storedAt >= $6 AND - storedAt <= $7 - ORDER BY storedAt ASC, messageHash ASC LIMIT $8;""" + """WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + (timestamp, messageHash) > ($4,$5) AND + timestamp >= $6 AND + timestamp <= $7 + ORDER BY timestamp ASC, messageHash ASC LIMIT $8;""" -const SelectMessageByHashName = "SelectMessageByHash" -const SelectMessageByHashDef = SelectClause & """WHERE messageHash = $1""" +const SelectWithCursorNoDataAscStmtName = "SelectWithCursorNoDataAsc" +const SelectWithCursorNoDataAscStmtDef = + """SELECT messageHash FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + (timestamp, messageHash) > ($4,$5) AND + timestamp >= $6 AND + timestamp <= $7 + ORDER BY timestamp ASC, messageHash ASC LIMIT $8;""" -const SelectNoCursorV2AscStmtName = "SelectWithoutCursorV2Asc" -const SelectNoCursorV2AscStmtDef = - SelectClause & - """ - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - storedAt >= $3 AND - storedAt <= $4 - ORDER BY storedAt ASC LIMIT $5;""" - -const SelectNoCursorV2DescStmtName = "SelectWithoutCursorV2Desc" -const SelectNoCursorV2DescStmtDef = - SelectClause & - """ - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - storedAt >= $3 AND - storedAt <= $4 - ORDER BY storedAt DESC LIMIT $5;""" - -const SelectWithCursorV2DescStmtName = "SelectWithCursorV2Desc" -const SelectWithCursorV2DescStmtDef = - SelectClause & - """ - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - (storedAt, id) < ($3,$4) AND - storedAt >= $5 AND - storedAt <= $6 - ORDER BY storedAt DESC LIMIT $7;""" - -const SelectWithCursorV2AscStmtName = "SelectWithCursorV2Asc" -const SelectWithCursorV2AscStmtDef = - SelectClause & - """ - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - (storedAt, id) > ($3,$4) AND - storedAt >= $5 AND - storedAt <= $6 - ORDER BY storedAt ASC LIMIT $7;""" +const SelectCursorByHashName = "SelectMessageByHash" +const SelectCursorByHashDef = + """SELECT timestamp FROM messages + WHERE messageHash = $1""" const DefaultMaxNumConns = 50 @@ -160,9 +159,65 @@ proc reset*(s: PostgresDriver): Future[ArchiveDriverResult[void]] {.async.} = let ret = await s.decreaseDatabaseSize(targetSize, forceRemoval) return ret +proc timeCursorCallbackImpl(pqResult: ptr PGresult, timeCursor: var Option[Timestamp]) = + ## Callback to get a timestamp out of the DB. + ## Used to get the cursor timestamp. + + let numFields = pqResult.pqnfields() + if numFields != 1: + error "Wrong number of fields" + return + + let rawTimestamp = $(pqgetvalue(pqResult, 0, 0)) + + trace "db output", rawTimestamp + + if rawTimestamp.len < 1: + return + + let catchable = catch: + parseBiggestInt(rawTimestamp) + + if catchable.isErr(): + error "could not parse correctly", error = catchable.error.msg + return + + timeCursor = some(catchable.get()) + +proc hashCallbackImpl( + pqResult: ptr PGresult, rows: var seq[(WakuMessageHash, PubsubTopic, WakuMessage)] +) = + ## Callback to get a hash out of the DB. + ## Used when queries only ask for hashes + + let numFields = pqResult.pqnfields() + if numFields != 1: + error "Wrong number of fields" + return + + for iRow in 0 ..< pqResult.pqNtuples(): + let rawHash = $(pqgetvalue(pqResult, iRow, 0)) + + trace "db output", rawHash + + if rawHash.len < 1: + return + + let catchable = catch: + parseHexStr(rawHash) + + if catchable.isErr(): + error "could not parse correctly", error = catchable.error.msg + return + + let hashHex = catchable.get() + let msgHash = fromBytes(hashHex.toOpenArrayByte(0, 31)) + + rows.add((msgHash, "", WakuMessage())) + proc rowCallbackImpl( pqResult: ptr PGresult, - outRows: var seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)], + outRows: var seq[(WakuMessageHash, PubsubTopic, WakuMessage)], ) = ## Proc aimed to contain the logic of the callback passed to the `psasyncpool`. ## That callback is used in "SELECT" queries. @@ -171,101 +226,96 @@ proc rowCallbackImpl( ## outRows - seq of Store-rows. This is populated from the info contained in pqResult let numFields = pqResult.pqnfields() - if numFields != 9: + if numFields != 7: error "Wrong number of fields" return for iRow in 0 ..< pqResult.pqNtuples(): - var wakuMessage: WakuMessage - var timestamp: Timestamp - var version: uint - var pubSubTopic: string - var contentTopic: string - var storedAt: int64 - var digest: string - var payload: string - var hashHex: string - var msgHash: WakuMessageHash - var meta: string + var + rawHash: string + rawPayload: string + rawVersion: string + rawTimestamp: string + rawMeta: string + + hashHex: string + msgHash: WakuMessageHash + + pubSubTopic: string + + contentTopic: string + payload: string + version: uint + timestamp: Timestamp + meta: string + wakuMessage: WakuMessage + + rawHash = $(pqgetvalue(pqResult, iRow, 0)) + pubSubTopic = $(pqgetvalue(pqResult, iRow, 1)) + contentTopic = $(pqgetvalue(pqResult, iRow, 2)) + rawPayload = $(pqgetvalue(pqResult, iRow, 3)) + rawVersion = $(pqgetvalue(pqResult, iRow, 4)) + rawTimestamp = $(pqgetvalue(pqResult, iRow, 5)) + rawMeta = $(pqgetvalue(pqResult, iRow, 6)) + + trace "db output", + rawHash, pubSubTopic, contentTopic, rawPayload, rawVersion, rawTimestamp, rawMeta try: - storedAt = parseInt($(pqgetvalue(pqResult, iRow, 0))) - contentTopic = $(pqgetvalue(pqResult, iRow, 1)) - payload = parseHexStr($(pqgetvalue(pqResult, iRow, 2))) - pubSubTopic = $(pqgetvalue(pqResult, iRow, 3)) - version = parseUInt($(pqgetvalue(pqResult, iRow, 4))) - timestamp = parseInt($(pqgetvalue(pqResult, iRow, 5))) - digest = parseHexStr($(pqgetvalue(pqResult, iRow, 6))) - hashHex = parseHexStr($(pqgetvalue(pqResult, iRow, 7))) - meta = parseHexStr($(pqgetvalue(pqResult, iRow, 8))) - msgHash = fromBytes(hashHex.toOpenArrayByte(0, 31)) + hashHex = parseHexStr(rawHash) + payload = parseHexStr(rawPayload) + version = parseUInt(rawVersion) + timestamp = parseInt(rawTimestamp) + meta = parseHexStr(rawMeta) except ValueError: error "could not parse correctly", error = getCurrentExceptionMsg() - wakuMessage.timestamp = timestamp - wakuMessage.version = uint32(version) + msgHash = fromBytes(hashHex.toOpenArrayByte(0, 31)) + wakuMessage.contentTopic = contentTopic wakuMessage.payload = @(payload.toOpenArrayByte(0, payload.high)) + wakuMessage.version = uint32(version) + wakuMessage.timestamp = timestamp wakuMessage.meta = @(meta.toOpenArrayByte(0, meta.high)) - outRows.add( - ( - pubSubTopic, - wakuMessage, - @(digest.toOpenArrayByte(0, digest.high)), - storedAt, - msgHash, - ) - ) + outRows.add((msgHash, pubSubTopic, wakuMessage)) method put*( s: PostgresDriver, + messageHash: WakuMessageHash, pubsubTopic: PubsubTopic, message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, ): Future[ArchiveDriverResult[void]] {.async.} = - let digest = toHex(digest.data) let messageHash = toHex(messageHash) - let rxTime = $receivedTime + let contentTopic = message.contentTopic let payload = toHex(message.payload) let version = $message.version let timestamp = $message.timestamp let meta = toHex(message.meta) - trace "put PostgresDriver", timestamp = timestamp + trace "put PostgresDriver", + messageHash, contentTopic, payload, version, timestamp, meta + + ## this is not needed for store-v3. Nevertheless, we will keep that temporarily + ## until we completely remove the store/archive-v2 logic + let fakeId = "0" return await s.writeConnPool.runStmt( InsertRowStmtName, InsertRowStmtDefinition, + @[fakeId, messageHash, pubsubTopic, contentTopic, payload, version, timestamp, meta], @[ - digest, messageHash, rxTime, contentTopic, payload, pubsubTopic, version, - timestamp, meta, - ], - @[ - int32(digest.len), + int32(fakeId.len), int32(messageHash.len), - int32(rxTime.len), + int32(pubsubTopic.len), int32(contentTopic.len), int32(payload.len), - int32(pubsubTopic.len), int32(version.len), int32(timestamp.len), int32(meta.len), ], - @[ - int32(0), - int32(0), - int32(0), - int32(0), - int32(0), - int32(0), - int32(0), - int32(0), - int32(0), - ], + @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], ) method getAllMessages*( @@ -273,15 +323,15 @@ method getAllMessages*( ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = ## Retrieve all messages from the store. - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] proc rowCallback(pqResult: ptr PGresult) = rowCallbackImpl(pqResult, rows) ( await s.readConnPool.pgQuery( - """SELECT storedAt, contentTopic, - payload, pubsubTopic, version, timestamp, - id, messageHash, meta FROM messages ORDER BY storedAt ASC""", + """SELECT messageHash, pubsubTopic, contentTopic, payload, version, timestamp, meta + FROM messages + ORDER BY timestamp ASC, messageHash ASC""", newSeq[string](0), rowCallback, ) @@ -322,9 +372,28 @@ proc getPartitionsList( return ok(partitions) +proc getTimeCursor( + s: PostgresDriver, hashHex: string +): Future[ArchiveDriverResult[Option[Timestamp]]] {.async.} = + var timeCursor: Option[Timestamp] + + proc cursorCallback(pqResult: ptr PGresult) = + timeCursorCallbackImpl(pqResult, timeCursor) + + ?await s.readConnPool.runStmt( + SelectCursorByHashName, + SelectCursorByHashDef, + @[hashHex], + @[int32(hashHex.len)], + @[int32(0)], + cursorCallback, + ) + + return ok(timeCursor) + proc getMessagesArbitraryQuery( s: PostgresDriver, - contentTopic: seq[ContentTopic] = @[], + contentTopics: seq[ContentTopic] = @[], pubsubTopic = none(PubsubTopic), cursor = none(ArchiveCursor), startTime = none(Timestamp), @@ -335,15 +404,28 @@ proc getMessagesArbitraryQuery( ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = ## This proc allows to handle atypical queries. We don't use prepared statements for those. - var query = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages""" + var query = SelectClause var statements: seq[string] var args: seq[string] - if contentTopic.len > 0: - let cstmt = "contentTopic IN (" & "?".repeat(contentTopic.len).join(",") & ")" + if cursor.isSome(): + let hashHex = toHex(cursor.get()) + + let timeCursor = ?await s.getTimeCursor(hashHex) + + if timeCursor.isNone(): + return err("cursor not found") + + let comp = if ascendingOrder: ">" else: "<" + statements.add("(timestamp, messageHash) " & comp & " (?,?)") + + args.add($timeCursor.get()) + args.add(hashHex) + + if contentTopics.len > 0: + let cstmt = "contentTopic IN (" & "?".repeat(contentTopics.len).join(",") & ")" statements.add(cstmt) - for t in contentTopic: + for t in contentTopics: args.add(t) if hexHashes.len > 0: @@ -356,41 +438,12 @@ proc getMessagesArbitraryQuery( statements.add("pubsubTopic = ?") args.add(pubsubTopic.get()) - if cursor.isSome(): - let hashHex = toHex(cursor.get().hash) - - var entree: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc entreeCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, entree) - - ( - await s.readConnPool.runStmt( - SelectMessageByHashName, - SelectMessageByHashDef, - @[hashHex], - @[int32(hashHex.len)], - @[int32(0)], - entreeCallback, - ) - ).isOkOr: - return err("failed to run query with cursor: " & $error) - - if entree.len == 0: - return ok(entree) - - let storetime = entree[0][3] - - let comp = if ascendingOrder: ">" else: "<" - statements.add("(storedAt, messageHash) " & comp & " (?,?)") - args.add($storetime) - args.add(hashHex) - if startTime.isSome(): - statements.add("storedAt >= ?") + statements.add("timestamp >= ?") args.add($startTime.get()) if endTime.isSome(): - statements.add("storedAt <= ?") + statements.add("timestamp <= ?") args.add($endTime.get()) if statements.len > 0: @@ -402,12 +455,12 @@ proc getMessagesArbitraryQuery( else: direction = "DESC" - query &= " ORDER BY storedAt " & direction & ", messageHash " & direction + query &= " ORDER BY timestamp " & direction & ", messageHash " & direction query &= " LIMIT ?" args.add($maxPageSize) - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] proc rowCallback(pqResult: ptr PGresult) = rowCallbackImpl(pqResult, rows) @@ -416,45 +469,61 @@ proc getMessagesArbitraryQuery( return ok(rows) -proc getMessagesV2ArbitraryQuery( +proc getMessageHashesArbitraryQuery( s: PostgresDriver, - contentTopic: seq[ContentTopic] = @[], + contentTopics: seq[ContentTopic] = @[], pubsubTopic = none(PubsubTopic), cursor = none(ArchiveCursor), startTime = none(Timestamp), endTime = none(Timestamp), + hexHashes: seq[string] = @[], maxPageSize = DefaultPageSize, ascendingOrder = true, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = +): Future[ArchiveDriverResult[seq[(WakuMessageHash, PubsubTopic, WakuMessage)]]] {. + async +.} = ## This proc allows to handle atypical queries. We don't use prepared statements for those. - var query = - """SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages""" + var query = """SELECT messageHash FROM messages""" var statements: seq[string] var args: seq[string] - if contentTopic.len > 0: - let cstmt = "contentTopic IN (" & "?".repeat(contentTopic.len).join(",") & ")" + if cursor.isSome(): + let hashHex = toHex(cursor.get()) + + let timeCursor = ?await s.getTimeCursor(hashHex) + + if timeCursor.isNone(): + return err("cursor not found") + + let comp = if ascendingOrder: ">" else: "<" + statements.add("(timestamp, messageHash) " & comp & " (?,?)") + + args.add($timeCursor.get()) + args.add(hashHex) + + if contentTopics.len > 0: + let cstmt = "contentTopic IN (" & "?".repeat(contentTopics.len).join(",") & ")" statements.add(cstmt) - for t in contentTopic: + for t in contentTopics: + args.add(t) + + if hexHashes.len > 0: + let cstmt = "messageHash IN (" & "?".repeat(hexHashes.len).join(",") & ")" + statements.add(cstmt) + for t in hexHashes: args.add(t) if pubsubTopic.isSome(): statements.add("pubsubTopic = ?") args.add(pubsubTopic.get()) - if cursor.isSome(): - let comp = if ascendingOrder: ">" else: "<" - statements.add("(storedAt, id) " & comp & " (?,?)") - args.add($cursor.get().storeTime) - args.add(toHex(cursor.get().digest.data)) - if startTime.isSome(): - statements.add("storedAt >= ?") + statements.add("timestamp >= ?") args.add($startTime.get()) if endTime.isSome(): - statements.add("storedAt <= ?") + statements.add("timestamp <= ?") args.add($endTime.get()) if statements.len > 0: @@ -466,14 +535,14 @@ proc getMessagesV2ArbitraryQuery( else: direction = "DESC" - query &= " ORDER BY storedAt " & direction & ", id " & direction + query &= " ORDER BY timestamp " & direction & ", messageHash " & direction query &= " LIMIT ?" args.add($maxPageSize) - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) + hashCallbackImpl(pqResult, rows) (await s.readConnPool.pgQuery(query, args, rowCallback)).isOkOr: return err("failed to run query: " & $error) @@ -491,12 +560,11 @@ proc getMessagesPreparedStmt( maxPageSize = DefaultPageSize, ascOrder = true, ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - ## This proc aims to run the most typical queries in a more performant way, i.e. by means of - ## prepared statements. - ## - ## contentTopic - string with list of conten topics. e.g: "'ctopic1','ctopic2','ctopic3'" + ## This proc aims to run the most typical queries in a more performant way, + ## i.e. by means of prepared statements. + + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] proc rowCallback(pqResult: ptr PGresult) = rowCallbackImpl(pqResult, rows) @@ -504,59 +572,7 @@ proc getMessagesPreparedStmt( let endTimeStr = $endTime let limit = $maxPageSize - if cursor.isSome(): - let hash = toHex(cursor.get().hash) - - var entree: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - - proc entreeCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, entree) - - ( - await s.readConnPool.runStmt( - SelectMessageByHashName, - SelectMessageByHashDef, - @[hash], - @[int32(hash.len)], - @[int32(0)], - entreeCallback, - ) - ).isOkOr: - return err("failed to run query with cursor: " & $error) - - if entree.len == 0: - return ok(entree) - - let storeTime = $entree[0][3] - - var stmtName = - if ascOrder: SelectWithCursorAscStmtName else: SelectWithCursorDescStmtName - var stmtDef = - if ascOrder: SelectWithCursorAscStmtDef else: SelectWithCursorDescStmtDef - - ( - await s.readConnPool.runStmt( - stmtName, - stmtDef, - @[ - contentTopic, hashes, pubsubTopic, storeTime, hash, startTimeStr, endTimeStr, - limit, - ], - @[ - int32(contentTopic.len), - int32(pubsubTopic.len), - int32(storeTime.len), - int32(hash.len), - int32(startTimeStr.len), - int32(endTimeStr.len), - int32(limit.len), - ], - @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], - rowCallback, - ) - ).isOkOr: - return err("failed to run query with cursor: " & $error) - else: + if cursor.isNone(): var stmtName = if ascOrder: SelectNoCursorAscStmtName else: SelectNoCursorDescStmtName var stmtDef = if ascOrder: SelectNoCursorAscStmtDef else: SelectNoCursorDescStmtDef @@ -577,91 +593,147 @@ proc getMessagesPreparedStmt( rowCallback, ) ).isOkOr: - return err("failed to run query without cursor: " & $error) + return err(stmtName & ": " & $error) + + return ok(rows) + + let hashHex = toHex(cursor.get()) + + let timeCursor = ?await s.getTimeCursor(hashHex) + + if timeCursor.isNone(): + return err("cursor not found") + + let timeString = $timeCursor.get() + + var stmtName = + if ascOrder: SelectWithCursorAscStmtName else: SelectWithCursorDescStmtName + var stmtDef = + if ascOrder: SelectWithCursorAscStmtDef else: SelectWithCursorDescStmtDef + + ( + await s.readConnPool.runStmt( + stmtName, + stmtDef, + @[ + contentTopic, hashes, pubsubTopic, hashHex, timeString, startTimeStr, + endTimeStr, limit, + ], + @[ + int32(contentTopic.len), + int32(hashes.len), + int32(pubsubTopic.len), + int32(timeString.len), + int32(hashHex.len), + int32(startTimeStr.len), + int32(endTimeStr.len), + int32(limit.len), + ], + @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], + rowCallback, + ) + ).isOkOr: + return err(stmtName & ": " & $error) return ok(rows) -proc getMessagesV2PreparedStmt( +proc getMessageHashesPreparedStmt( s: PostgresDriver, contentTopic: string, pubsubTopic: PubsubTopic, cursor = none(ArchiveCursor), startTime: Timestamp, endTime: Timestamp, + hashes: string, maxPageSize = DefaultPageSize, ascOrder = true, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = - ## This proc aims to run the most typical queries in a more performant way, i.e. by means of - ## prepared statements. - ## - ## contentTopic - string with list of conten topics. e.g: "'ctopic1','ctopic2','ctopic3'" +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + ## This proc aims to run the most typical queries in a more performant way, + ## i.e. by means of prepared statements. + + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) + hashCallbackImpl(pqResult, rows) let startTimeStr = $startTime let endTimeStr = $endTime let limit = $maxPageSize - if cursor.isSome(): + if cursor.isNone(): var stmtName = - if ascOrder: SelectWithCursorV2AscStmtName else: SelectWithCursorV2DescStmtName + if ascOrder: SelectNoCursorNoDataAscStmtName else: SelectNoCursorNoDataDescStmtName var stmtDef = - if ascOrder: SelectWithCursorV2AscStmtDef else: SelectWithCursorV2DescStmtDef - - let digest = toHex(cursor.get().digest.data) - let storeTime = $cursor.get().storeTime + if ascOrder: SelectNoCursorNoDataAscStmtDef else: SelectNoCursorNoDataDescStmtDef ( await s.readConnPool.runStmt( stmtName, stmtDef, - @[contentTopic, pubsubTopic, storeTime, digest, startTimeStr, endTimeStr, limit], + @[contentTopic, hashes, pubsubTopic, startTimeStr, endTimeStr, limit], @[ int32(contentTopic.len), + int32(hashes.len), int32(pubsubTopic.len), - int32(storeTime.len), - int32(digest.len), int32(startTimeStr.len), int32(endTimeStr.len), int32(limit.len), ], - @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], + @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], rowCallback, ) ).isOkOr: - return err("failed to run query with cursor: " & $error) - else: - var stmtName = - if ascOrder: SelectNoCursorV2AscStmtName else: SelectNoCursorV2DescStmtName - var stmtDef = - if ascOrder: SelectNoCursorV2AscStmtDef else: SelectNoCursorV2DescStmtDef + return err(stmtName & ": " & $error) - ( - await s.readConnPool.runStmt( - stmtName, - stmtDef, - @[contentTopic, pubsubTopic, startTimeStr, endTimeStr, limit], - @[ - int32(contentTopic.len), - int32(pubsubTopic.len), - int32(startTimeStr.len), - int32(endTimeStr.len), - int32(limit.len), - ], - @[int32(0), int32(0), int32(0), int32(0), int32(0)], - rowCallback, - ) - ).isOkOr: - return err("failed to run query without cursor: " & $error) + return ok(rows) + + let hashHex = toHex(cursor.get()) + + let timeCursor = ?await s.getTimeCursor(hashHex) + + if timeCursor.isNone(): + return err("cursor not found") + + let timeString = $timeCursor.get() + + var stmtName = + if ascOrder: + SelectWithCursorNoDataAscStmtName + else: + SelectWithCursorNoDataDescStmtName + var stmtDef = + if ascOrder: SelectWithCursorNoDataAscStmtDef else: SelectWithCursorNoDataDescStmtDef + + ( + await s.readConnPool.runStmt( + stmtName, + stmtDef, + @[ + contentTopic, hashes, pubsubTopic, hashHex, timeString, startTimeStr, + endTimeStr, limit, + ], + @[ + int32(contentTopic.len), + int32(hashes.len), + int32(pubsubTopic.len), + int32(timeString.len), + int32(hashHex.len), + int32(startTimeStr.len), + int32(endTimeStr.len), + int32(limit.len), + ], + @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], + rowCallback, + ) + ).isOkOr: + return err(stmtName & ": " & $error) return ok(rows) method getMessages*( s: PostgresDriver, - includeData = false, - contentTopicSeq = newSeq[ContentTopic](0), + includeData = true, + contentTopics = newSeq[ContentTopic](0), pubsubTopic = none(PubsubTopic), cursor = none(ArchiveCursor), startTime = none(Timestamp), @@ -672,54 +744,43 @@ method getMessages*( ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = let hexHashes = hashes.mapIt(toHex(it)) - if contentTopicSeq.len == 1 and hexHashes.len == 1 and pubsubTopic.isSome() and + if contentTopics.len > 0 and hexHashes.len > 0 and pubsubTopic.isSome() and startTime.isSome() and endTime.isSome(): ## Considered the most common query. Therefore, we use prepared statements to optimize it. - return await s.getMessagesPreparedStmt( - contentTopicSeq.join(","), - PubsubTopic(pubsubTopic.get()), - cursor, - startTime.get(), - endTime.get(), - hexHashes.join(","), - maxPageSize, - ascendingOrder, - ) + if includeData: + return await s.getMessagesPreparedStmt( + contentTopics.join(","), + PubsubTopic(pubsubTopic.get()), + cursor, + startTime.get(), + endTime.get(), + hexHashes.join(","), + maxPageSize, + ascendingOrder, + ) + else: + return await s.getMessageHashesPreparedStmt( + contentTopics.join(","), + PubsubTopic(pubsubTopic.get()), + cursor, + startTime.get(), + endTime.get(), + hexHashes.join(","), + maxPageSize, + ascendingOrder, + ) else: - ## We will run atypical query. In this case we don't use prepared statemets - return await s.getMessagesArbitraryQuery( - contentTopicSeq, pubsubTopic, cursor, startTime, endTime, hexHashes, maxPageSize, - ascendingOrder, - ) - -method getMessagesV2*( - s: PostgresDriver, - contentTopicSeq = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - maxPageSize = DefaultPageSize, - ascendingOrder = true, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = - if contentTopicSeq.len == 1 and pubsubTopic.isSome() and startTime.isSome() and - endTime.isSome(): - ## Considered the most common query. Therefore, we use prepared statements to optimize it. - return await s.getMessagesV2PreparedStmt( - contentTopicSeq.join(","), - PubsubTopic(pubsubTopic.get()), - cursor, - startTime.get(), - endTime.get(), - maxPageSize, - ascendingOrder, - ) - else: - ## We will run atypical query. In this case we don't use prepared statemets - return await s.getMessagesV2ArbitraryQuery( - contentTopicSeq, pubsubTopic, cursor, startTime, endTime, maxPageSize, - ascendingOrder, - ) + if includeData: + ## We will run atypical query. In this case we don't use prepared statemets + return await s.getMessagesArbitraryQuery( + contentTopics, pubsubTopic, cursor, startTime, endTime, hexHashes, maxPageSize, + ascendingOrder, + ) + else: + return await s.getMessageHashesArbitraryQuery( + contentTopics, pubsubTopic, cursor, startTime, endTime, hexHashes, maxPageSize, + ascendingOrder, + ) proc getStr( s: PostgresDriver, query: string @@ -791,7 +852,7 @@ method getOldestMessageTimestamp*( let oldestPartitionTimeNanoSec = oldestPartition.getPartitionStartTimeInNanosec() - let intRes = await s.getInt("SELECT MIN(storedAt) FROM messages") + let intRes = await s.getInt("SELECT MIN(timestamp) FROM messages") if intRes.isErr(): ## Just return the oldest partition time considering the partitions set return ok(Timestamp(oldestPartitionTimeNanoSec)) @@ -801,7 +862,7 @@ method getOldestMessageTimestamp*( method getNewestMessageTimestamp*( s: PostgresDriver ): Future[ArchiveDriverResult[Timestamp]] {.async.} = - let intRes = await s.getInt("SELECT MAX(storedAt) FROM messages") + let intRes = await s.getInt("SELECT MAX(timestamp) FROM messages") if intRes.isErr(): return err("error in getNewestMessageTimestamp: " & intRes.error) @@ -811,9 +872,9 @@ method deleteOldestMessagesNotWithinLimit*( s: PostgresDriver, limit: int ): Future[ArchiveDriverResult[void]] {.async.} = let execRes = await s.writeConnPool.pgQuery( - """DELETE FROM messages WHERE id NOT IN + """DELETE FROM messages WHERE messageHash NOT IN ( - SELECT id FROM messages ORDER BY storedAt DESC LIMIT ? + SELECT messageHash FROM messages ORDER BY timestamp DESC LIMIT ? );""", @[$limit], ) @@ -824,7 +885,7 @@ method deleteOldestMessagesNotWithinLimit*( method close*(s: PostgresDriver): Future[ArchiveDriverResult[void]] {.async.} = ## Cancel the partition factory loop - s.futLoopPartitionFactory.cancel() + s.futLoopPartitionFactory.cancelSoon() ## Close the database connection let writeCloseRes = await s.writeConnPool.close() @@ -961,7 +1022,7 @@ proc addPartition( self: PostgresDriver, startTime: Timestamp ): Future[ArchiveDriverResult[void]] {.async.} = ## Creates a partition table that will store the messages that fall in the range - ## `startTime` <= storedAt < `startTime + duration`. + ## `startTime` <= timestamp < `startTime + duration`. ## `startTime` is measured in seconds since epoch let beginning = startTime @@ -992,7 +1053,8 @@ proc addPartition( let constraintName = partitionName & "_by_range_check" let addTimeConstraintQuery = "ALTER TABLE " & partitionName & " ADD CONSTRAINT " & constraintName & - " CHECK ( storedAt >= " & fromInNanoSec & " AND storedAt < " & untilInNanoSec & " );" + " CHECK ( timestamp >= " & fromInNanoSec & " AND timestamp < " & untilInNanoSec & + " );" (await self.performWriteQueryWithLock(addTimeConstraintQuery)).isOkOr: return err(fmt"error creating constraint [{partitionName}]: " & $error) @@ -1280,7 +1342,11 @@ method deleteMessagesOlderThanTimestamp*( (await s.removePartitionsOlderThan(tsNanoSec)).isOkOr: return err("error while removing older partitions: " & $error) - (await s.writeConnPool.pgQuery("DELETE FROM messages WHERE storedAt < " & $tsNanoSec)).isOkOr: + ( + await s.writeConnPool.pgQuery( + "DELETE FROM messages WHERE timestamp < " & $tsNanoSec + ) + ).isOkOr: return err("error in deleteMessagesOlderThanTimestamp: " & $error) return ok() diff --git a/waku/waku_archive/driver/queue_driver/index.nim b/waku/waku_archive/driver/queue_driver/index.nim index 22e612aab..113d426d4 100644 --- a/waku/waku_archive/driver/queue_driver/index.nim +++ b/waku/waku_archive/driver/queue_driver/index.nim @@ -1,58 +1,16 @@ {.push raises: [].} -import stew/byteutils, nimcrypto/sha2 -import ../../../waku_core, ../../common +import stew/byteutils +import ../../../waku_core type Index* = object ## This type contains the description of an Index used in the pagination of WakuMessages - pubsubTopic*: string - senderTime*: Timestamp # the time at which the message is generated - receiverTime*: Timestamp - digest*: MessageDigest # calculated over payload and content topic + time*: Timestamp # the time at which the message is generated hash*: WakuMessageHash - -proc compute*( - T: type Index, msg: WakuMessage, receivedTime: Timestamp, pubsubTopic: PubsubTopic -): T = - ## Takes a WakuMessage with received timestamp and returns its Index. - let - digest = computeDigest(msg) - senderTime = msg.timestamp - hash = computeMessageHash(pubsubTopic, msg) - - return Index( - pubsubTopic: pubsubTopic, - senderTime: senderTime, - receiverTime: receivedTime, - digest: digest, - hash: hash, - ) - -proc tohistoryCursor*(index: Index): ArchiveCursor = - return ArchiveCursor( - pubsubTopic: index.pubsubTopic, - senderTime: index.senderTime, - storeTime: index.receiverTime, - digest: index.digest, - hash: index.hash, - ) - -proc toIndex*(index: ArchiveCursor): Index = - return Index( - pubsubTopic: index.pubsubTopic, - senderTime: index.senderTime, - receiverTime: index.storeTime, - digest: index.digest, - hash: index.hash, - ) + pubsubTopic*: PubsubTopic proc `==`*(x, y: Index): bool = - ## receiverTime plays no role in index equality - return - ( - (x.senderTime == y.senderTime) and (x.digest == y.digest) and - (x.pubsubTopic == y.pubsubTopic) - ) or (x.hash == y.hash) # this applies to store v3 queries only + return x.hash == y.hash proc cmp*(x, y: Index): int = ## compares x and y @@ -61,28 +19,11 @@ proc cmp*(x, y: Index): int = ## returns 1 if x > y ## ## Default sorting order priority is: - ## 1. senderTimestamp - ## 2. receiverTimestamp (a fallback only if senderTimestamp unset on either side, and all other fields unequal) - ## 3. message digest - ## 4. pubsubTopic + ## 1. time + ## 2. hash - if x == y: - # Quick exit ensures receiver time does not affect index equality - return 0 + let timeCMP = cmp(x.time, y.time) + if timeCMP != 0: + return timeCMP - # Timestamp has a higher priority for comparison - let - # Use receiverTime where senderTime is unset - xTimestamp = if x.senderTime == 0: x.receiverTime else: x.senderTime - yTimestamp = if y.senderTime == 0: y.receiverTime else: y.senderTime - - let timecmp = cmp(xTimestamp, yTimestamp) - if timecmp != 0: - return timecmp - - # Continue only when timestamps are equal - let digestcmp = cmp(x.digest.data, y.digest.data) - if digestcmp != 0: - return digestcmp - - return cmp(x.pubsubTopic, y.pubsubTopic) + return cmp(x.hash, y.hash) diff --git a/waku/waku_archive/driver/queue_driver/queue_driver.nim b/waku/waku_archive/driver/queue_driver/queue_driver.nim index 23051e9cd..df21cf8f4 100644 --- a/waku/waku_archive/driver/queue_driver/queue_driver.nim +++ b/waku/waku_archive/driver/queue_driver/queue_driver.nim @@ -133,9 +133,7 @@ proc getPage( if predicate.isNil() or predicate(key, data): numberOfItems += 1 - outSeq.add( - (key.pubsubTopic, data, @(key.digest.data), key.receiverTime, key.hash) - ) + outSeq.add((key.hash, key.pubsubTopic, data)) currentEntry = if forward: @@ -227,19 +225,12 @@ proc add*( method put*( driver: QueueDriver, + messageHash: WakuMessageHash, pubsubTopic: PubsubTopic, message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, ): Future[ArchiveDriverResult[void]] {.async.} = - let index = Index( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - receiverTime: receivedTime, - digest: digest, - hash: messageHash, - ) + let index = + Index(time: message.timestamp, hash: messageHash, pubsubTopic: pubsubTopic) return driver.add(index, message) @@ -256,8 +247,8 @@ method existsTable*( method getMessages*( driver: QueueDriver, - includeData = false, - contentTopic: seq[ContentTopic] = @[], + includeData = true, + contentTopics: seq[ContentTopic] = @[], pubsubTopic = none(PubsubTopic), cursor = none(ArchiveCursor), startTime = none(Timestamp), @@ -266,14 +257,17 @@ method getMessages*( maxPageSize = DefaultPageSize, ascendingOrder = true, ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - let cursor = cursor.map(toIndex) + var index = none(Index) + + if cursor.isSome(): + index = some(Index(hash: cursor.get())) let matchesQuery: QueryFilterMatcher = func (index: Index, msg: WakuMessage): bool = if pubsubTopic.isSome() and index.pubsubTopic != pubsubTopic.get(): return false - if contentTopic.len > 0 and msg.contentTopic notin contentTopic: + if contentTopics.len > 0 and msg.contentTopic notin contentTopics: return false if startTime.isSome() and msg.timestamp < startTime.get(): @@ -287,11 +281,14 @@ method getMessages*( return true - var pageRes: QueueDriverGetPageResult - try: - pageRes = driver.getPage(maxPageSize, ascendingOrder, cursor, matchesQuery) - except CatchableError, Exception: - return err(getCurrentExceptionMsg()) + let catchable = catch: + driver.getPage(maxPageSize, ascendingOrder, index, matchesQuery) + + let pageRes: QueueDriverGetPageResult = + if catchable.isErr(): + return err(catchable.error.msg) + else: + catchable.get() if pageRes.isErr(): return err($pageRes.error) @@ -328,7 +325,7 @@ method getOldestMessageTimestamp*( ): Future[ArchiveDriverResult[Timestamp]] {.async.} = return driver.first().map( proc(index: Index): Timestamp = - index.receiverTime + index.time ) method getNewestMessageTimestamp*( @@ -336,7 +333,7 @@ method getNewestMessageTimestamp*( ): Future[ArchiveDriverResult[Timestamp]] {.async.} = return driver.last().map( proc(index: Index): Timestamp = - index.receiverTime + index.time ) method deleteMessagesOlderThanTimestamp*( diff --git a/waku/waku_archive/driver/sqlite_driver/queries.nim b/waku/waku_archive/driver/sqlite_driver/queries.nim index ff0bd904a..0a167937e 100644 --- a/waku/waku_archive/driver/sqlite_driver/queries.nim +++ b/waku/waku_archive/driver/sqlite_driver/queries.nim @@ -5,8 +5,7 @@ import chronicles import ../../../common/databases/db_sqlite, ../../../common/databases/common, - ../../../waku_core, - ./cursor + ../../../waku_core const DbTable = "Message" @@ -16,7 +15,7 @@ type SqlQueryStr = string proc queryRowWakuMessageCallback( s: ptr sqlite3_stmt, - contentTopicCol, payloadCol, versionCol, senderTimestampCol, metaCol: cint, + contentTopicCol, payloadCol, versionCol, timestampCol, metaCol: cint, ): WakuMessage = let topic = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, contentTopicCol)) @@ -30,22 +29,20 @@ proc queryRowWakuMessageCallback( metaLength = sqlite3_column_bytes(s, metaCol) payload = @(toOpenArray(p, 0, payloadLength - 1)) version = sqlite3_column_int64(s, versionCol) - senderTimestamp = sqlite3_column_int64(s, senderTimestampCol) + timestamp = sqlite3_column_int64(s, timestampCol) meta = @(toOpenArray(m, 0, metaLength - 1)) return WakuMessage( contentTopic: ContentTopic(contentTopic), payload: payload, version: uint32(version), - timestamp: Timestamp(senderTimestamp), + timestamp: Timestamp(timestamp), meta: meta, ) -proc queryRowReceiverTimestampCallback( - s: ptr sqlite3_stmt, storedAtCol: cint -): Timestamp = - let storedAt = sqlite3_column_int64(s, storedAtCol) - return Timestamp(storedAt) +proc queryRowTimestampCallback(s: ptr sqlite3_stmt, timestampCol: cint): Timestamp = + let timestamp = sqlite3_column_int64(s, timestampCol) + return Timestamp(timestamp) proc queryRowPubsubTopicCallback( s: ptr sqlite3_stmt, pubsubTopicCol: cint @@ -59,14 +56,6 @@ proc queryRowPubsubTopicCallback( return pubsubTopic -proc queryRowDigestCallback(s: ptr sqlite3_stmt, digestCol: cint): seq[byte] = - let - digestPointer = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, digestCol)) - digestLength = sqlite3_column_bytes(s, digestCol) - digest = @(toOpenArray(digestPointer, 0, digestLength - 1)) - - return digest - proc queryRowWakuMessageHashCallback( s: ptr sqlite3_stmt, hashCol: cint ): WakuMessageHash = @@ -82,11 +71,10 @@ proc queryRowWakuMessageHashCallback( ## Create table proc createTableQuery(table: string): SqlQueryStr = - "CREATE TABLE IF NOT EXISTS " & table & " (" & " pubsubTopic BLOB NOT NULL," & + "CREATE TABLE IF NOT EXISTS " & table & " (" & + " messageHash BLOB NOT NULL PRIMARY KEY," & " pubsubTopic BLOB NOT NULL," & " contentTopic BLOB NOT NULL," & " payload BLOB," & " version INTEGER NOT NULL," & - " timestamp INTEGER NOT NULL," & " id BLOB," & " messageHash BLOB," & - " storedAt INTEGER NOT NULL," & " meta BLOB," & - " CONSTRAINT messageIndex PRIMARY KEY (messageHash)" & ") WITHOUT ROWID;" + " timestamp INTEGER NOT NULL," & " meta BLOB" & ") WITHOUT ROWID;" proc createTable*(db: SqliteDatabase): DatabaseResult[void] = let query = createTableQuery(DbTable) @@ -102,7 +90,7 @@ proc createTable*(db: SqliteDatabase): DatabaseResult[void] = ## Create indices proc createOldestMessageTimestampIndexQuery(table: string): SqlQueryStr = - "CREATE INDEX IF NOT EXISTS i_ts ON " & table & " (storedAt);" + "CREATE INDEX IF NOT EXISTS i_ts ON " & table & " (timestamp);" proc createOldestMessageTimestampIndex*(db: SqliteDatabase): DatabaseResult[void] = let query = createOldestMessageTimestampIndexQuery(DbTable) @@ -115,39 +103,15 @@ proc createOldestMessageTimestampIndex*(db: SqliteDatabase): DatabaseResult[void ) return ok() -proc createHistoryQueryIndexQuery(table: string): SqlQueryStr = - "CREATE INDEX IF NOT EXISTS i_query ON " & table & - " (contentTopic, pubsubTopic, storedAt, id);" - -proc createHistoryQueryIndex*(db: SqliteDatabase): DatabaseResult[void] = - let query = createHistoryQueryIndexQuery(DbTable) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard - , - ) - return ok() - ## Insert message -type InsertMessageParams* = ( - seq[byte], - seq[byte], - Timestamp, - seq[byte], - seq[byte], - seq[byte], - int64, - Timestamp, - seq[byte], -) +type InsertMessageParams* = + (seq[byte], seq[byte], seq[byte], seq[byte], int64, Timestamp, seq[byte]) proc insertMessageQuery(table: string): SqlQueryStr = return "INSERT INTO " & table & - "(id, messageHash, storedAt, contentTopic, payload, pubsubTopic, version, timestamp, meta)" & - " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);" + "(messageHash, pubsubTopic, contentTopic, payload, version, timestamp, meta)" & + " VALUES (?, ?, ?, ?, ?, ?, ?);" proc prepareInsertMessageStmt*( db: SqliteDatabase @@ -176,14 +140,12 @@ proc getMessageCount*(db: SqliteDatabase): DatabaseResult[int64] = ## Get oldest message receiver timestamp proc selectOldestMessageTimestampQuery(table: string): SqlQueryStr = - return "SELECT MIN(storedAt) FROM " & table + return "SELECT MIN(timestamp) FROM " & table -proc selectOldestReceiverTimestamp*( - db: SqliteDatabase -): DatabaseResult[Timestamp] {.inline.} = +proc selectOldestTimestamp*(db: SqliteDatabase): DatabaseResult[Timestamp] {.inline.} = var timestamp: Timestamp proc queryRowCallback(s: ptr sqlite3_stmt) = - timestamp = queryRowReceiverTimestampCallback(s, 0) + timestamp = queryRowTimestampCallback(s, 0) let query = selectOldestMessageTimestampQuery(DbTable) let res = db.query(query, queryRowCallback) @@ -195,14 +157,12 @@ proc selectOldestReceiverTimestamp*( ## Get newest message receiver timestamp proc selectNewestMessageTimestampQuery(table: string): SqlQueryStr = - return "SELECT MAX(storedAt) FROM " & table + return "SELECT MAX(timestamp) FROM " & table -proc selectNewestReceiverTimestamp*( - db: SqliteDatabase -): DatabaseResult[Timestamp] {.inline.} = +proc selectNewestTimestamp*(db: SqliteDatabase): DatabaseResult[Timestamp] {.inline.} = var timestamp: Timestamp proc queryRowCallback(s: ptr sqlite3_stmt) = - timestamp = queryRowReceiverTimestampCallback(s, 0) + timestamp = queryRowTimestampCallback(s, 0) let query = selectNewestMessageTimestampQuery(DbTable) let res = db.query(query, queryRowCallback) @@ -214,7 +174,7 @@ proc selectNewestReceiverTimestamp*( ## Delete messages older than timestamp proc deleteMessagesOlderThanTimestampQuery(table: string, ts: Timestamp): SqlQueryStr = - return "DELETE FROM " & table & " WHERE storedAt < " & $ts + return "DELETE FROM " & table & " WHERE timestamp < " & $ts proc deleteMessagesOlderThanTimestamp*( db: SqliteDatabase, ts: int64 @@ -233,9 +193,9 @@ proc deleteMessagesOlderThanTimestamp*( proc deleteOldestMessagesNotWithinLimitQuery(table: string, limit: int): SqlQueryStr = return - "DELETE FROM " & table & " WHERE (storedAt, id, pubsubTopic) NOT IN (" & - " SELECT storedAt, id, pubsubTopic FROM " & table & - " ORDER BY storedAt DESC, id DESC" & " LIMIT " & $limit & ");" + "DELETE FROM " & table & " WHERE (timestamp, messageHash) NOT IN (" & + " SELECT timestamp, messageHash FROM " & table & + " ORDER BY timestamp DESC, messageHash DESC" & " LIMIT " & $limit & ");" proc deleteOldestMessagesNotWithinLimit*( db: SqliteDatabase, limit: int @@ -255,37 +215,50 @@ proc deleteOldestMessagesNotWithinLimit*( proc selectAllMessagesQuery(table: string): SqlQueryStr = return - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" & - " FROM " & table & " ORDER BY storedAt ASC" + "SELECT messageHash, pubsubTopic, contentTopic, payload, version, timestamp, meta" & + " FROM " & table & " ORDER BY timestamp ASC" proc selectAllMessages*( db: SqliteDatabase -): DatabaseResult[ - seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] -] {.gcsafe.} = +): DatabaseResult[seq[(WakuMessageHash, PubsubTopic, WakuMessage)]] = ## Retrieve all messages from the store. - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] proc queryRowCallback(s: ptr sqlite3_stmt) = let - pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) + hash = queryRowWakuMessageHashCallback(s, hashCol = 0) + pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 1) wakuMessage = queryRowWakuMessageCallback( s, - contentTopicCol = 1, - payloadCol = 2, + contentTopicCol = 2, + payloadCol = 3, versionCol = 4, - senderTimestampCol = 5, - metaCol = 8, + timestampCol = 5, + metaCol = 6, ) - digest = queryRowDigestCallback(s, digestCol = 6) - storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) - hash = queryRowWakuMessageHashCallback(s, hashCol = 7) - rows.add((pubsubTopic, wakuMessage, digest, storedAt, hash)) + rows.add((hash, pubsubTopic, wakuMessage)) let query = selectAllMessagesQuery(DbTable) - let res = db.query(query, queryRowCallback) - if res.isErr(): - return err(res.error()) + db.query(query, queryRowCallback).isOkOr: + return err("select all messages failed: " & $error) + + return ok(rows) + +## Select all messages without data + +proc selectAllMessageHashesQuery(table: string): SqlQueryStr = + return "SELECT messageHash" & " FROM " & table & " ORDER BY timestamp ASC" + +proc selectAllMessageHashes*(db: SqliteDatabase): DatabaseResult[seq[WakuMessageHash]] = + ## Retrieve all messages from the store. + var rows: seq[WakuMessageHash] + proc queryRowCallback(s: ptr sqlite3_stmt) = + let hash = queryRowWakuMessageHashCallback(s, hashCol = 0) + rows.add(hash) + + let query = selectAllMessageHashesQuery(DbTable) + db.query(query, queryRowCallback).isOkOr: + return err("select all message hashes failed: " & $error) return ok(rows) @@ -301,75 +274,6 @@ proc combineClauses(clauses: varargs[Option[string]]): Option[string] = where &= " AND " & clause return some(where) -proc whereClausev2( - cursor: bool, - pubsubTopic: Option[PubsubTopic], - contentTopic: seq[ContentTopic], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - ascending: bool, -): Option[string] {.deprecated.} = - let cursorClause = - if cursor: - let comp = if ascending: ">" else: "<" - - some("(storedAt, id) " & comp & " (?, ?)") - else: - none(string) - - let pubsubTopicClause = - if pubsubTopic.isNone(): - none(string) - else: - some("pubsubTopic = (?)") - - let contentTopicClause = - if contentTopic.len <= 0: - none(string) - else: - var where = "contentTopic IN (" - where &= "?" - for _ in 1 ..< contentTopic.len: - where &= ", ?" - where &= ")" - some(where) - - let startTimeClause = - if startTime.isNone(): - none(string) - else: - some("storedAt >= (?)") - - let endTimeClause = - if endTime.isNone(): - none(string) - else: - some("storedAt <= (?)") - - return combineClauses( - cursorClause, pubsubTopicClause, contentTopicClause, startTimeClause, endTimeClause - ) - -proc selectMessagesWithLimitQueryv2( - table: string, where: Option[string], limit: uint, ascending = true, v3 = false -): SqlQueryStr {.deprecated.} = - let order = if ascending: "ASC" else: "DESC" - - var query: string - - query = - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" - query &= " FROM " & table - - if where.isSome(): - query &= " WHERE " & where.get() - - query &= " ORDER BY storedAt " & order & ", id " & order - - query &= " LIMIT " & $limit & ";" - - return query - proc prepareStmt( db: SqliteDatabase, stmt: string ): DatabaseResult[SqliteStmt[void, void]] = @@ -377,113 +281,6 @@ proc prepareStmt( checkErr sqlite3_prepare_v2(db.env, stmt, stmt.len.cint, addr s, nil) return ok(SqliteStmt[void, void](s)) -proc execSelectMessagesV2WithLimitStmt( - s: SqliteStmt, - cursor: Option[DbCursor], - pubsubTopic: Option[PubsubTopic], - contentTopic: seq[ContentTopic], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - onRowCallback: DataProc, -): DatabaseResult[void] {.deprecated.} = - let s = RawStmtPtr(s) - - # Bind params - var paramIndex = 1 - - if cursor.isSome(): - let (storedAt, id, _) = cursor.get() - checkErr bindParam(s, paramIndex, storedAt) - paramIndex += 1 - checkErr bindParam(s, paramIndex, id) - paramIndex += 1 - - if pubsubTopic.isSome(): - let pubsubTopic = toBytes(pubsubTopic.get()) - checkErr bindParam(s, paramIndex, pubsubTopic) - paramIndex += 1 - - for topic in contentTopic: - checkErr bindParam(s, paramIndex, topic.toBytes()) - paramIndex += 1 - - if startTime.isSome(): - let time = startTime.get() - checkErr bindParam(s, paramIndex, time) - paramIndex += 1 - - if endTime.isSome(): - let time = endTime.get() - checkErr bindParam(s, paramIndex, time) - paramIndex += 1 - - try: - while true: - let v = sqlite3_step(s) - case v - of SQLITE_ROW: - onRowCallback(s) - of SQLITE_DONE: - return ok() - else: - return err($sqlite3_errstr(v)) - except Exception, CatchableError: - error "exception in execSelectMessagesV2WithLimitStmt", - error = getCurrentExceptionMsg() - - # release implicit transaction - discard sqlite3_reset(s) # same return information as step - discard sqlite3_clear_bindings(s) # no errors possible - -proc selectMessagesByHistoryQueryWithLimit*( - db: SqliteDatabase, - contentTopic: seq[ContentTopic], - pubsubTopic: Option[PubsubTopic], - cursor: Option[DbCursor], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - limit: uint, - ascending: bool, -): DatabaseResult[ - seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] -] {.deprecated.} = - var messages: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] = - @[] - - proc queryRowCallback(s: ptr sqlite3_stmt) = - let - pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) - message = queryRowWakuMessageCallback( - s, - contentTopicCol = 1, - payloadCol = 2, - versionCol = 4, - senderTimestampCol = 5, - metaCol = 8, - ) - digest = queryRowDigestCallback(s, digestCol = 6) - storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) - hash = queryRowWakuMessageHashCallback(s, hashCol = 7) - - messages.add((pubsubTopic, message, digest, storedAt, hash)) - - let query = block: - let where = whereClausev2( - cursor.isSome(), pubsubTopic, contentTopic, startTime, endTime, ascending - ) - - selectMessagesWithLimitQueryv2(DbTable, where, limit, ascending) - - let dbStmt = ?db.prepareStmt(query) - ?dbStmt.execSelectMessagesV2WithLimitStmt( - cursor, pubsubTopic, contentTopic, startTime, endTime, queryRowCallback - ) - dbStmt.dispose() - - return ok(messages) - -### Store v3 ### - proc execSelectMessageByHash( s: SqliteStmt, hash: WakuMessageHash, onRowCallback: DataProc ): DatabaseResult[void] = @@ -508,14 +305,23 @@ proc execSelectMessageByHash( discard sqlite3_reset(s) # same return information as step discard sqlite3_clear_bindings(s) # no errors possible -proc selectMessageByHashQuery(): SqlQueryStr = - var query: string +proc selectTimestampByHashQuery(table: string): SqlQueryStr = + return "SELECT timestamp FROM " & table & " WHERE messageHash = (?)" - query = "SELECT contentTopic, payload, version, timestamp, meta, messageHash" - query &= " FROM " & DbTable - query &= " WHERE messageHash = (?)" +proc getCursorTimestamp( + db: SqliteDatabase, hash: WakuMessageHash +): DatabaseResult[Option[Timestamp]] = + var timestamp = none(Timestamp) - return query + proc queryRowCallback(s: ptr sqlite3_stmt) = + timestamp = some(queryRowTimestampCallback(s, 0)) + + let query = selectTimestampByHashQuery(DbTable) + let dbStmt = ?db.prepareStmt(query) + ?dbStmt.execSelectMessageByHash(hash, queryRowCallback) + dbStmt.dispose() + + return ok(timestamp) proc whereClause( cursor: bool, @@ -555,13 +361,13 @@ proc whereClause( if startTime.isNone(): none(string) else: - some("storedAt >= (?)") + some("timestamp >= (?)") let endTimeClause = if endTime.isNone(): none(string) else: - some("storedAt <= (?)") + some("timestamp <= (?)") let hashesClause = if hashes.len <= 0: @@ -643,20 +449,36 @@ proc execSelectMessagesWithLimitStmt( discard sqlite3_clear_bindings(s) # no errors possible proc selectMessagesWithLimitQuery( - table: string, where: Option[string], limit: uint, ascending = true, v3 = false + table: string, where: Option[string], limit: uint, ascending = true ): SqlQueryStr = let order = if ascending: "ASC" else: "DESC" var query: string query = - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" + "SELECT messageHash, pubsubTopic, contentTopic, payload, version, timestamp, meta" query &= " FROM " & table if where.isSome(): query &= " WHERE " & where.get() - query &= " ORDER BY storedAt " & order & ", messageHash " & order + query &= " ORDER BY timestamp " & order & ", messageHash " & order + + query &= " LIMIT " & $limit & ";" + + return query + +proc selectMessageHashesWithLimitQuery( + table: string, where: Option[string], limit: uint, ascending = true +): SqlQueryStr = + let order = if ascending: "ASC" else: "DESC" + + var query = "SELECT messageHash FROM " & table + + if where.isSome(): + query &= " WHERE " & where.get() + + query &= " ORDER BY timestamp " & order & ", messageHash " & order query &= " LIMIT " & $limit & ";" @@ -672,79 +494,101 @@ proc selectMessagesByStoreQueryWithLimit*( hashes: seq[WakuMessageHash], limit: uint, ascending: bool, -): DatabaseResult[ - seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] -] = - # Must first get the message timestamp before paginating by time - let newCursor = - if cursor.isSome() and cursor.get() != EmptyWakuMessageHash: - let hash: WakuMessageHash = cursor.get() +): DatabaseResult[seq[(WakuMessageHash, PubsubTopic, WakuMessage)]] = + var timeCursor = none((Timestamp, WakuMessageHash)) - var wakuMessage: Option[WakuMessage] + if cursor.isSome(): + let hash: WakuMessageHash = cursor.get() - proc queryRowCallback(s: ptr sqlite3_stmt) = - wakuMessage = some( - queryRowWakuMessageCallback( - s, - contentTopicCol = 0, - payloadCol = 1, - versionCol = 2, - senderTimestampCol = 3, - metaCol = 4, - ) - ) + let timeOpt = ?getCursorTimestamp(db, hash) - let query = selectMessageByHashQuery() - let dbStmt = ?db.prepareStmt(query) - ?dbStmt.execSelectMessageByHash(hash, queryRowCallback) - dbStmt.dispose() + if timeOpt.isNone(): + return err("cursor not found") - if wakuMessage.isSome(): - let time = wakuMessage.get().timestamp + timeCursor = some((timeOpt.get(), hash)) - some((time, hash)) - else: - return err("cursor not found") - else: - none((Timestamp, WakuMessageHash)) - - var messages: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] = - @[] + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] = @[] proc queryRowCallback(s: ptr sqlite3_stmt) = let - pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) + hash = queryRowWakuMessageHashCallback(s, hashCol = 0) + pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 1) message = queryRowWakuMessageCallback( s, - contentTopicCol = 1, - payloadCol = 2, + contentTopicCol = 2, + payloadCol = 3, versionCol = 4, - senderTimestampCol = 5, - metaCol = 8, + timestampCol = 5, + metaCol = 6, ) - digest = queryRowDigestCallback(s, digestCol = 6) - storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) - hash = queryRowWakuMessageHashCallback(s, hashCol = 7) - messages.add((pubsubTopic, message, digest, storedAt, hash)) + rows.add((hash, pubsubTopic, message)) - let query = block: - let where = whereClause( - newCursor.isSome(), - pubsubTopic, - contentTopic, - startTime, - endTime, - hashes, - ascending, - ) + let where = whereClause( + timeCursor.isSome(), + pubsubTopic, + contentTopic, + startTime, + endTime, + hashes, + ascending, + ) - selectMessagesWithLimitQuery(DbTable, where, limit, ascending, true) + let query = selectMessagesWithLimitQuery(DbTable, where, limit, ascending) let dbStmt = ?db.prepareStmt(query) ?dbStmt.execSelectMessagesWithLimitStmt( - newCursor, pubsubTopic, contentTopic, startTime, endTime, hashes, queryRowCallback + timeCursor, pubsubTopic, contentTopic, startTime, endTime, hashes, queryRowCallback ) dbStmt.dispose() - return ok(messages) + return ok(rows) + +proc selectMessageHashesByStoreQueryWithLimit*( + db: SqliteDatabase, + contentTopic: seq[ContentTopic], + pubsubTopic: Option[PubsubTopic], + cursor: Option[WakuMessageHash], + startTime: Option[Timestamp], + endTime: Option[Timestamp], + hashes: seq[WakuMessageHash], + limit: uint, + ascending: bool, +): DatabaseResult[seq[(WakuMessageHash, PubsubTopic, WakuMessage)]] = + var timeCursor = none((Timestamp, WakuMessageHash)) + + if cursor.isSome(): + let hash: WakuMessageHash = cursor.get() + + let timeOpt = ?getCursorTimestamp(db, hash) + + if timeOpt.isNone(): + return err("cursor not found") + + timeCursor = some((timeOpt.get(), hash)) + + var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] = @[] + + proc queryRowCallback(s: ptr sqlite3_stmt) = + let hash = queryRowWakuMessageHashCallback(s, hashCol = 0) + rows.add((hash, "", WakuMessage())) + + let where = whereClause( + timeCursor.isSome(), + pubsubTopic, + contentTopic, + startTime, + endTime, + hashes, + ascending, + ) + + let query = selectMessageHashesWithLimitQuery(DbTable, where, limit, ascending) + + let dbStmt = ?db.prepareStmt(query) + ?dbStmt.execSelectMessagesWithLimitStmt( + timeCursor, pubsubTopic, contentTopic, startTime, endTime, hashes, queryRowCallback + ) + dbStmt.dispose() + + return ok(rows) diff --git a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim index 91af943d1..d872b9f15 100644 --- a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim +++ b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim @@ -9,7 +9,6 @@ import ../../../waku_core/message/digest, ../../common, ../../driver, - ./cursor, ./queries logScope: @@ -28,11 +27,7 @@ proc init(db: SqliteDatabase): ArchiveDriverResult[void] = # Create indices, if don't exist let resRtIndex = createOldestMessageTimestampIndex(db) if resRtIndex.isErr(): - return err("failed to create i_rt index: " & resRtIndex.error()) - - let resMsgIndex = createHistoryQueryIndex(db) - if resMsgIndex.isErr(): - return err("failed to create i_query index: " & resMsgIndex.error()) + return err("failed to create i_ts index: " & resRtIndex.error()) return ok() @@ -52,24 +47,20 @@ proc new*(T: type SqliteDriver, db: SqliteDatabase): ArchiveDriverResult[T] = method put*( s: SqliteDriver, + messageHash: WakuMessageHash, pubsubTopic: PubsubTopic, message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, ): Future[ArchiveDriverResult[void]] {.async.} = ## Inserts a message into the store let res = s.insertStmt.exec( ( - @(digest.data), # id - @(messageHash), # messageHash - receivedTime, # storedAt - toBytes(message.contentTopic), # contentTopic - message.payload, # payload - toBytes(pubsubTopic), # pubsubTopic - int64(message.version), # version - message.timestamp, # senderTimestamp - message.meta, # meta + @(messageHash), + toBytes(pubsubTopic), + toBytes(message.contentTopic), + message.payload, + int64(message.version), + message.timestamp, + message.meta, ) ) @@ -81,34 +72,10 @@ method getAllMessages*( ## Retrieve all messages from the store. return s.db.selectAllMessages() -method getMessagesV2*( - s: SqliteDriver, - contentTopic = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - maxPageSize = DefaultPageSize, - ascendingOrder = true, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = - let cursor = cursor.map(toDbCursor) - - let rowsRes = s.db.selectMessagesByHistoryQueryWithLimit( - contentTopic, - pubsubTopic, - cursor, - startTime, - endTime, - limit = maxPageSize, - ascending = ascendingOrder, - ) - - return rowsRes - method getMessages*( s: SqliteDriver, - includeData = false, - contentTopic = newSeq[ContentTopic](0), + includeData = true, + contentTopics = newSeq[ContentTopic](0), pubsubTopic = none(PubsubTopic), cursor = none(ArchiveCursor), startTime = none(Timestamp), @@ -117,14 +84,20 @@ method getMessages*( maxPageSize = DefaultPageSize, ascendingOrder = true, ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - let cursor = - if cursor.isSome(): - some(cursor.get().hash) - else: - none(WakuMessageHash) + if not includeData: + return s.db.selectMessageHashesByStoreQueryWithLimit( + contentTopics, + pubsubTopic, + cursor, + startTime, + endTime, + hashes, + limit = maxPageSize, + ascending = ascendingOrder, + ) - let rowsRes = s.db.selectMessagesByStoreQueryWithLimit( - contentTopic, + return s.db.selectMessagesByStoreQueryWithLimit( + contentTopics, pubsubTopic, cursor, startTime, @@ -134,8 +107,6 @@ method getMessages*( ascending = ascendingOrder, ) - return rowsRes - method getMessagesCount*( s: SqliteDriver ): Future[ArchiveDriverResult[int64]] {.async.} = @@ -156,12 +127,12 @@ method performVacuum*(s: SqliteDriver): Future[ArchiveDriverResult[void]] {.asyn method getOldestMessageTimestamp*( s: SqliteDriver ): Future[ArchiveDriverResult[Timestamp]] {.async.} = - return s.db.selectOldestReceiverTimestamp() + return s.db.selectOldestTimestamp() method getNewestMessageTimestamp*( s: SqliteDriver ): Future[ArchiveDriverResult[Timestamp]] {.async.} = - return s.db.selectnewestReceiverTimestamp() + return s.db.selectnewestTimestamp() method deleteMessagesOlderThanTimestamp*( s: SqliteDriver, ts: Timestamp diff --git a/waku/waku_archive_legacy.nim b/waku/waku_archive_legacy.nim new file mode 100644 index 000000000..e1d7df776 --- /dev/null +++ b/waku/waku_archive_legacy.nim @@ -0,0 +1,7 @@ +import + ./waku_archive_legacy/common, + ./waku_archive_legacy/archive, + ./waku_archive_legacy/driver, + ./waku_archive_legacy/retention_policy + +export common, archive, driver, retention_policy diff --git a/waku/waku_archive_legacy/archive.nim b/waku/waku_archive_legacy/archive.nim new file mode 100644 index 000000000..753a2e64b --- /dev/null +++ b/waku/waku_archive_legacy/archive.nim @@ -0,0 +1,323 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import + std/[times, options, sequtils, strutils, algorithm], + stew/[results, byteutils], + chronicles, + chronos, + metrics +import + ../common/paging, + ./driver, + ./retention_policy, + ../waku_core, + ../waku_core/message/digest, + ./common, + ./archive_metrics + +logScope: + topics = "waku archive" + +const + DefaultPageSize*: uint = 20 + MaxPageSize*: uint = 100 + + # Retention policy + WakuArchiveDefaultRetentionPolicyInterval* = chronos.minutes(30) + + # Metrics reporting + WakuArchiveDefaultMetricsReportInterval* = chronos.minutes(1) + + # Message validation + # 20 seconds maximum allowable sender timestamp "drift" + MaxMessageTimestampVariance* = getNanoSecondTime(20) + +type MessageValidator* = + proc(msg: WakuMessage): Result[void, string] {.closure, gcsafe, raises: [].} + +## Archive + +type WakuArchive* = ref object + driver: ArchiveDriver + + validator: MessageValidator + + retentionPolicy: Option[RetentionPolicy] + + retentionPolicyHandle: Future[void] + metricsHandle: Future[void] + +proc validate*(msg: WakuMessage): Result[void, string] = + if msg.ephemeral: + # Ephemeral message, do not store + return + + if msg.timestamp == 0: + return ok() + + let + now = getNanosecondTime(getTime().toUnixFloat()) + lowerBound = now - MaxMessageTimestampVariance + upperBound = now + MaxMessageTimestampVariance + + if msg.timestamp < lowerBound: + return err(invalidMessageOld) + + if upperBound < msg.timestamp: + return err(invalidMessageFuture) + + return ok() + +proc new*( + T: type WakuArchive, + driver: ArchiveDriver, + validator: MessageValidator = validate, + retentionPolicy = none(RetentionPolicy), +): Result[T, string] = + if driver.isNil(): + return err("archive driver is Nil") + + let archive = + WakuArchive(driver: driver, validator: validator, retentionPolicy: retentionPolicy) + + return ok(archive) + +proc handleMessage*( + self: WakuArchive, pubsubTopic: PubsubTopic, msg: WakuMessage +) {.async.} = + self.validator(msg).isOkOr: + waku_legacy_archive_errors.inc(labelValues = [error]) + return + + let + msgDigest = computeDigest(msg) + msgDigestHex = msgDigest.data.to0xHex() + msgHash = computeMessageHash(pubsubTopic, msg) + msgHashHex = msgHash.to0xHex() + msgTimestamp = + if msg.timestamp > 0: + msg.timestamp + else: + getNanosecondTime(getTime().toUnixFloat()) + + trace "handling message", + msg_hash = msgHashHex, + pubsubTopic = pubsubTopic, + contentTopic = msg.contentTopic, + msgTimestamp = msg.timestamp, + usedTimestamp = msgTimestamp, + digest = msgDigestHex + + let insertStartTime = getTime().toUnixFloat() + + (await self.driver.put(pubsubTopic, msg, msgDigest, msgHash, msgTimestamp)).isOkOr: + waku_legacy_archive_errors.inc(labelValues = [insertFailure]) + error "failed to insert message", error = error + return + + debug "message archived", + msg_hash = msgHashHex, + pubsubTopic = pubsubTopic, + contentTopic = msg.contentTopic, + msgTimestamp = msg.timestamp, + usedTimestamp = msgTimestamp, + digest = msgDigestHex + + let insertDuration = getTime().toUnixFloat() - insertStartTime + waku_legacy_archive_insert_duration_seconds.observe(insertDuration) + +proc findMessages*( + self: WakuArchive, query: ArchiveQuery +): Future[ArchiveResult] {.async, gcsafe.} = + ## Search the archive to return a single page of messages matching the query criteria + + let maxPageSize = + if query.pageSize <= 0: + DefaultPageSize + else: + min(query.pageSize, MaxPageSize) + + let isAscendingOrder = query.direction.into() + + if query.contentTopics.len > 10: + return err(ArchiveError.invalidQuery("too many content topics")) + + if query.cursor.isSome() and query.cursor.get().hash.len != 32: + return err(ArchiveError.invalidQuery("invalid cursor hash length")) + + let queryStartTime = getTime().toUnixFloat() + + let rows = ( + await self.driver.getMessages( + includeData = query.includeData, + contentTopic = query.contentTopics, + pubsubTopic = query.pubsubTopic, + cursor = query.cursor, + startTime = query.startTime, + endTime = query.endTime, + hashes = query.hashes, + maxPageSize = maxPageSize + 1, + ascendingOrder = isAscendingOrder, + ) + ).valueOr: + return err(ArchiveError(kind: ArchiveErrorKind.DRIVER_ERROR, cause: error)) + + let queryDuration = getTime().toUnixFloat() - queryStartTime + waku_legacy_archive_query_duration_seconds.observe(queryDuration) + + var hashes = newSeq[WakuMessageHash]() + var messages = newSeq[WakuMessage]() + var topics = newSeq[PubsubTopic]() + var cursor = none(ArchiveCursor) + + if rows.len == 0: + return ok(ArchiveResponse(hashes: hashes, messages: messages, cursor: cursor)) + + ## Messages + let pageSize = min(rows.len, int(maxPageSize)) + + if query.includeData: + topics = rows[0 ..< pageSize].mapIt(it[0]) + messages = rows[0 ..< pageSize].mapIt(it[1]) + + hashes = rows[0 ..< pageSize].mapIt(it[4]) + + ## Cursor + if rows.len > int(maxPageSize): + ## Build last message cursor + ## The cursor is built from the last message INCLUDED in the response + ## (i.e. the second last message in the rows list) + + let (pubsubTopic, message, digest, storeTimestamp, hash) = rows[^2] + + cursor = some( + ArchiveCursor( + digest: MessageDigest.fromBytes(digest), + storeTime: storeTimestamp, + sendertime: message.timestamp, + pubsubTopic: pubsubTopic, + hash: hash, + ) + ) + + # All messages MUST be returned in chronological order + if not isAscendingOrder: + reverse(hashes) + reverse(messages) + reverse(topics) + + return ok( + ArchiveResponse(hashes: hashes, messages: messages, topics: topics, cursor: cursor) + ) + +proc findMessagesV2*( + self: WakuArchive, query: ArchiveQuery +): Future[ArchiveResult] {.async, deprecated, gcsafe.} = + ## Search the archive to return a single page of messages matching the query criteria + + let maxPageSize = + if query.pageSize <= 0: + DefaultPageSize + else: + min(query.pageSize, MaxPageSize) + + let isAscendingOrder = query.direction.into() + + if query.contentTopics.len > 10: + return err(ArchiveError.invalidQuery("too many content topics")) + + let queryStartTime = getTime().toUnixFloat() + + let rows = ( + await self.driver.getMessagesV2( + contentTopic = query.contentTopics, + pubsubTopic = query.pubsubTopic, + cursor = query.cursor, + startTime = query.startTime, + endTime = query.endTime, + maxPageSize = maxPageSize + 1, + ascendingOrder = isAscendingOrder, + ) + ).valueOr: + return err(ArchiveError(kind: ArchiveErrorKind.DRIVER_ERROR, cause: error)) + + let queryDuration = getTime().toUnixFloat() - queryStartTime + waku_legacy_archive_query_duration_seconds.observe(queryDuration) + + var messages = newSeq[WakuMessage]() + var cursor = none(ArchiveCursor) + + if rows.len == 0: + return ok(ArchiveResponse(messages: messages, cursor: cursor)) + + ## Messages + let pageSize = min(rows.len, int(maxPageSize)) + + messages = rows[0 ..< pageSize].mapIt(it[1]) + + ## Cursor + if rows.len > int(maxPageSize): + ## Build last message cursor + ## The cursor is built from the last message INCLUDED in the response + ## (i.e. the second last message in the rows list) + + let (pubsubTopic, message, digest, storeTimestamp, _) = rows[^2] + + cursor = some( + ArchiveCursor( + digest: MessageDigest.fromBytes(digest), + storeTime: storeTimestamp, + sendertime: message.timestamp, + pubsubTopic: pubsubTopic, + ) + ) + + # All messages MUST be returned in chronological order + if not isAscendingOrder: + reverse(messages) + + return ok(ArchiveResponse(messages: messages, cursor: cursor)) + +proc periodicRetentionPolicy(self: WakuArchive) {.async.} = + debug "executing message retention policy" + + let policy = self.retentionPolicy.get() + + while true: + (await policy.execute(self.driver)).isOkOr: + waku_legacy_archive_errors.inc(labelValues = [retPolicyFailure]) + error "failed execution of retention policy", error = error + + await sleepAsync(WakuArchiveDefaultRetentionPolicyInterval) + +proc periodicMetricReport(self: WakuArchive) {.async.} = + while true: + let countRes = (await self.driver.getMessagesCount()) + if countRes.isErr(): + error "loopReportStoredMessagesMetric failed to get messages count", + error = countRes.error + else: + let count = countRes.get() + waku_legacy_archive_messages.set(count, labelValues = ["stored"]) + + await sleepAsync(WakuArchiveDefaultMetricsReportInterval) + +proc start*(self: WakuArchive) = + if self.retentionPolicy.isSome(): + self.retentionPolicyHandle = self.periodicRetentionPolicy() + + self.metricsHandle = self.periodicMetricReport() + +proc stopWait*(self: WakuArchive) {.async.} = + var futures: seq[Future[void]] + + if self.retentionPolicy.isSome() and not self.retentionPolicyHandle.isNil(): + futures.add(self.retentionPolicyHandle.cancelAndWait()) + + if not self.metricsHandle.isNil: + futures.add(self.metricsHandle.cancelAndWait()) + + await noCancel(allFutures(futures)) diff --git a/waku/waku_archive_legacy/archive_metrics.nim b/waku/waku_archive_legacy/archive_metrics.nim new file mode 100644 index 000000000..e99a6196f --- /dev/null +++ b/waku/waku_archive_legacy/archive_metrics.nim @@ -0,0 +1,23 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import metrics + +declarePublicGauge waku_legacy_archive_messages, + "number of historical messages", ["type"] +declarePublicGauge waku_legacy_archive_errors, + "number of store protocol errors", ["type"] +declarePublicGauge waku_legacy_archive_queries, "number of store queries received" +declarePublicHistogram waku_legacy_archive_insert_duration_seconds, + "message insertion duration" +declarePublicHistogram waku_legacy_archive_query_duration_seconds, + "history query duration" + +# Error types (metric label values) +const + invalidMessageOld* = "invalid_message_too_old" + invalidMessageFuture* = "invalid_message_future_timestamp" + insertFailure* = "insert_failure" + retPolicyFailure* = "retpolicy_failure" diff --git a/waku/waku_archive_legacy/common.nim b/waku/waku_archive_legacy/common.nim new file mode 100644 index 000000000..9ef67178f --- /dev/null +++ b/waku/waku_archive_legacy/common.nim @@ -0,0 +1,87 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/options, results, stew/byteutils, stew/arrayops, nimcrypto/sha2 +import ../waku_core, ../common/paging + +## Waku message digest + +type MessageDigest* = MDigest[256] + +proc fromBytes*(T: type MessageDigest, src: seq[byte]): T = + var data: array[32, byte] + + let byteCount = copyFrom[byte](data, src) + + assert byteCount == 32 + + return MessageDigest(data: data) + +proc computeDigest*(msg: WakuMessage): MessageDigest = + var ctx: sha256 + ctx.init() + defer: + ctx.clear() + + ctx.update(msg.contentTopic.toBytes()) + ctx.update(msg.payload) + + # Computes the hash + return ctx.finish() + +## Public API types + +type + #TODO Once Store v2 is removed, the cursor becomes the hash of the last message + ArchiveCursor* = object + digest*: MessageDigest + storeTime*: Timestamp + senderTime*: Timestamp + pubsubTopic*: PubsubTopic + hash*: WakuMessageHash + + ArchiveQuery* = object + includeData*: bool # indicate if messages should be returned in addition to hashes. + pubsubTopic*: Option[PubsubTopic] + contentTopics*: seq[ContentTopic] + cursor*: Option[ArchiveCursor] + startTime*: Option[Timestamp] + endTime*: Option[Timestamp] + hashes*: seq[WakuMessageHash] + pageSize*: uint + direction*: PagingDirection + + ArchiveResponse* = object + hashes*: seq[WakuMessageHash] + messages*: seq[WakuMessage] + topics*: seq[PubsubTopic] + cursor*: Option[ArchiveCursor] + + ArchiveErrorKind* {.pure.} = enum + UNKNOWN = uint32(0) + DRIVER_ERROR = uint32(1) + INVALID_QUERY = uint32(2) + + ArchiveError* = object + case kind*: ArchiveErrorKind + of DRIVER_ERROR, INVALID_QUERY: + # TODO: Add an enum to be able to distinguish between error causes + cause*: string + else: + discard + + ArchiveResult* = Result[ArchiveResponse, ArchiveError] + +proc `$`*(err: ArchiveError): string = + case err.kind + of ArchiveErrorKind.DRIVER_ERROR: + "DIRVER_ERROR: " & err.cause + of ArchiveErrorKind.INVALID_QUERY: + "INVALID_QUERY: " & err.cause + of ArchiveErrorKind.UNKNOWN: + "UNKNOWN" + +proc invalidQuery*(T: type ArchiveError, cause: string): T = + ArchiveError(kind: ArchiveErrorKind.INVALID_QUERY, cause: cause) diff --git a/waku/waku_archive_legacy/driver.nim b/waku/waku_archive_legacy/driver.nim new file mode 100644 index 000000000..98dccdf0a --- /dev/null +++ b/waku/waku_archive_legacy/driver.nim @@ -0,0 +1,119 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/options, results, chronos +import ../waku_core, ./common + +const DefaultPageSize*: uint = 25 + +type + ArchiveDriverResult*[T] = Result[T, string] + ArchiveDriver* = ref object of RootObj + +#TODO Once Store v2 is removed keep only messages and hashes +type ArchiveRow* = (PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash) + +# ArchiveDriver interface + +method put*( + driver: ArchiveDriver, + pubsubTopic: PubsubTopic, + message: WakuMessage, + digest: MessageDigest, + messageHash: WakuMessageHash, + receivedTime: Timestamp, +): Future[ArchiveDriverResult[void]] {.base, async.} = + discard + +method getAllMessages*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, async.} = + discard + +method getMessagesV2*( + driver: ArchiveDriver, + contentTopic = newSeq[ContentTopic](0), + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, deprecated, async.} = + discard + +method getMessages*( + driver: ArchiveDriver, + includeData = true, + contentTopic = newSeq[ContentTopic](0), + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hashes = newSeq[WakuMessageHash](0), + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, async.} = + discard + +method getMessagesCount*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[int64]] {.base, async.} = + discard + +method getPagesCount*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[int64]] {.base, async.} = + discard + +method getPagesSize*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[int64]] {.base, async.} = + discard + +method getDatabaseSize*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[int64]] {.base, async.} = + discard + +method performVacuum*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[void]] {.base, async.} = + discard + +method getOldestMessageTimestamp*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[Timestamp]] {.base, async.} = + discard + +method getNewestMessageTimestamp*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[Timestamp]] {.base, async.} = + discard + +method deleteMessagesOlderThanTimestamp*( + driver: ArchiveDriver, ts: Timestamp +): Future[ArchiveDriverResult[void]] {.base, async.} = + discard + +method deleteOldestMessagesNotWithinLimit*( + driver: ArchiveDriver, limit: int +): Future[ArchiveDriverResult[void]] {.base, async.} = + discard + +method decreaseDatabaseSize*( + driver: ArchiveDriver, targetSizeInBytes: int64, forceRemoval: bool = false +): Future[ArchiveDriverResult[void]] {.base, async.} = + discard + +method close*( + driver: ArchiveDriver +): Future[ArchiveDriverResult[void]] {.base, async.} = + discard + +method existsTable*( + driver: ArchiveDriver, tableName: string +): Future[ArchiveDriverResult[bool]] {.base, async.} = + discard diff --git a/waku/waku_archive_legacy/driver/builder.nim b/waku/waku_archive_legacy/driver/builder.nim new file mode 100644 index 000000000..c05e25eec --- /dev/null +++ b/waku/waku_archive_legacy/driver/builder.nim @@ -0,0 +1,125 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import results, chronicles, chronos +import + ../driver, + ../../common/databases/dburl, + ../../common/databases/db_sqlite, + ../../common/error_handling, + ./sqlite_driver, + ./sqlite_driver/migrations as archive_driver_sqlite_migrations, + ./queue_driver + +export sqlite_driver, queue_driver + +when defined(postgres): + import ## These imports add dependency with an external libpq library + ./postgres_driver/migrations as archive_postgres_driver_migrations, + ./postgres_driver + export postgres_driver + +proc new*( + T: type ArchiveDriver, + url: string, + vacuum: bool, + migrate: bool, + maxNumConn: int, + onFatalErrorAction: OnFatalErrorHandler, +): Future[Result[T, string]] {.async.} = + ## url - string that defines the database + ## vacuum - if true, a cleanup operation will be applied to the database + ## migrate - if true, the database schema will be updated + ## maxNumConn - defines the maximum number of connections to handle simultaneously (Postgres) + ## onFatalErrorAction - called if, e.g., the connection with db got lost + + let dbUrlValidationRes = dburl.validateDbUrl(url) + if dbUrlValidationRes.isErr(): + return err("DbUrl failure in ArchiveDriver.new: " & dbUrlValidationRes.error) + + let engineRes = dburl.getDbEngine(url) + if engineRes.isErr(): + return err("error getting db engine in setupWakuArchiveDriver: " & engineRes.error) + + let engine = engineRes.get() + + case engine + of "sqlite": + let pathRes = dburl.getDbPath(url) + if pathRes.isErr(): + return err("error get path in setupWakuArchiveDriver: " & pathRes.error) + + let dbRes = SqliteDatabase.new(pathRes.get()) + if dbRes.isErr(): + return err("error in setupWakuArchiveDriver: " & dbRes.error) + + let db = dbRes.get() + + # SQLite vacuum + let sqliteStatsRes = db.gatherSqlitePageStats() + if sqliteStatsRes.isErr(): + return err("error while gathering sqlite stats: " & $sqliteStatsRes.error) + + let (pageSize, pageCount, freelistCount) = sqliteStatsRes.get() + debug "sqlite database page stats", + pageSize = pageSize, pages = pageCount, freePages = freelistCount + + if vacuum and (pageCount > 0 and freelistCount > 0): + let vacuumRes = db.performSqliteVacuum() + if vacuumRes.isErr(): + return err("error in vacuum sqlite: " & $vacuumRes.error) + + # Database migration + if migrate: + let migrateRes = archive_driver_sqlite_migrations.migrate(db) + if migrateRes.isErr(): + return err("error in migrate sqlite: " & $migrateRes.error) + + debug "setting up sqlite waku archive driver" + let res = SqliteDriver.new(db) + if res.isErr(): + return err("failed to init sqlite archive driver: " & res.error) + + return ok(res.get()) + of "postgres": + when defined(postgres): + let res = PostgresDriver.new( + dbUrl = url, + maxConnections = maxNumConn, + onFatalErrorAction = onFatalErrorAction, + ) + if res.isErr(): + return err("failed to init postgres archive driver: " & res.error) + + let driver = res.get() + + # Database migration + if migrate: + let migrateRes = await archive_postgres_driver_migrations.migrate(driver) + if migrateRes.isErr(): + return err("ArchiveDriver build failed in migration: " & $migrateRes.error) + + ## This should be started once we make sure the 'messages' table exists + ## Hence, this should be run after the migration is completed. + asyncSpawn driver.startPartitionFactory(onFatalErrorAction) + + info "waiting for a partition to be created" + for i in 0 ..< 100: + if driver.containsAnyPartition(): + break + await sleepAsync(chronos.milliseconds(100)) + + if not driver.containsAnyPartition(): + onFatalErrorAction("a partition could not be created") + + return ok(driver) + else: + return err( + "Postgres has been configured but not been compiled. Check compiler definitions." + ) + else: + debug "setting up in-memory waku archive driver" + let driver = QueueDriver.new() # Defaults to a capacity of 25.000 messages + return ok(driver) diff --git a/waku/waku_archive_legacy/driver/postgres_driver.nim b/waku/waku_archive_legacy/driver/postgres_driver.nim new file mode 100644 index 000000000..a106eb2c4 --- /dev/null +++ b/waku/waku_archive_legacy/driver/postgres_driver.nim @@ -0,0 +1,11 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import + ./postgres_driver/postgres_driver, + ./postgres_driver/partitions_manager, + ./postgres_driver/postgres_healthcheck + +export postgres_driver, partitions_manager, postgres_healthcheck diff --git a/waku/waku_archive_legacy/driver/postgres_driver/migrations.nim b/waku/waku_archive_legacy/driver/postgres_driver/migrations.nim new file mode 100644 index 000000000..e404d4648 --- /dev/null +++ b/waku/waku_archive_legacy/driver/postgres_driver/migrations.nim @@ -0,0 +1,89 @@ +{.push raises: [].} + +import std/strutils, results, chronicles, chronos +import + ../../../common/databases/common, + ../../../../migrations/message_store_postgres/pg_migration_manager, + ../postgres_driver + +logScope: + topics = "waku archive migration" + +const SchemaVersion* = 6 # increase this when there is an update in the database schema + +proc breakIntoStatements*(script: string): seq[string] = + ## Given a full migration script, that can potentially contain a list + ## of SQL statements, this proc splits it into the contained isolated statements + ## that should be executed one after the other. + var statements = newSeq[string]() + + let lines = script.split('\n') + + var simpleStmt: string + var plSqlStatement: string + var insidePlSqlScript = false + for line in lines: + if line.strip().len == 0: + continue + + if insidePlSqlScript: + if line.contains("END $$"): + ## End of the Pl/SQL script + plSqlStatement &= line + statements.add(plSqlStatement) + plSqlStatement = "" + insidePlSqlScript = false + continue + else: + plSqlStatement &= line & "\n" + + if line.contains("DO $$"): + ## Beginning of the Pl/SQL script + insidePlSqlScript = true + plSqlStatement &= line & "\n" + + if not insidePlSqlScript: + if line.contains(';'): + ## End of simple statement + simpleStmt &= line + statements.add(simpleStmt) + simpleStmt = "" + else: + simpleStmt &= line & "\n" + + return statements + +proc migrate*( + driver: PostgresDriver, targetVersion = SchemaVersion +): Future[DatabaseResult[void]] {.async.} = + debug "starting message store's postgres database migration" + + let currentVersion = (await driver.getCurrentVersion()).valueOr: + return err("migrate error could not retrieve current version: " & $error) + + if currentVersion == targetVersion: + debug "database schema is up to date", + currentVersion = currentVersion, targetVersion = targetVersion + return ok() + + info "database schema is outdated", + currentVersion = currentVersion, targetVersion = targetVersion + + # Load migration scripts + let scripts = pg_migration_manager.getMigrationScripts(currentVersion, targetVersion) + + # Run the migration scripts + for script in scripts: + for statement in script.breakIntoStatements(): + debug "executing migration statement", statement = statement + + (await driver.performWriteQuery(statement)).isOkOr: + error "failed to execute migration statement", + statement = statement, error = error + return err("failed to execute migration statement") + + debug "migration statement executed succesfully", statement = statement + + debug "finished message store's postgres database migration" + + return ok() diff --git a/waku/waku_archive_legacy/driver/postgres_driver/partitions_manager.nim b/waku/waku_archive_legacy/driver/postgres_driver/partitions_manager.nim new file mode 100644 index 000000000..52a01cef8 --- /dev/null +++ b/waku/waku_archive_legacy/driver/postgres_driver/partitions_manager.nim @@ -0,0 +1,102 @@ +## This module is aimed to handle the creation and truncation of partition tables +## in order to limit the space occupied in disk by the database. +## +## The created partitions are referenced by the 'storedAt' field. +## + +import std/deques +import chronos, chronicles + +logScope: + topics = "waku archive partitions_manager" + +## The time range has seconds resolution +type TimeRange* = tuple[beginning: int64, `end`: int64] + +type + Partition = object + name: string + timeRange: TimeRange + + PartitionManager* = ref object + partitions: Deque[Partition] + # FIFO of partition table names. The first is the oldest partition + +proc new*(T: type PartitionManager): T = + return PartitionManager() + +proc getPartitionFromDateTime*( + self: PartitionManager, targetMoment: int64 +): Result[Partition, string] = + ## Returns the partition name that might store a message containing the passed timestamp. + ## In order words, it simply returns the partition name which contains the given timestamp. + ## targetMoment - represents the time of interest, measured in seconds since epoch. + + if self.partitions.len == 0: + return err("There are no partitions") + + for partition in self.partitions: + let timeRange = partition.timeRange + + let beginning = timeRange.beginning + let `end` = timeRange.`end` + + if beginning <= targetMoment and targetMoment < `end`: + return ok(partition) + + return err("Couldn't find a partition table for given time: " & $targetMoment) + +proc getNewestPartition*(self: PartitionManager): Result[Partition, string] = + if self.partitions.len == 0: + return err("there are no partitions allocated") + + let newestPartition = self.partitions.peekLast + return ok(newestPartition) + +proc getOldestPartition*(self: PartitionManager): Result[Partition, string] = + if self.partitions.len == 0: + return err("there are no partitions allocated") + + let oldestPartition = self.partitions.peekFirst + return ok(oldestPartition) + +proc addPartitionInfo*( + self: PartitionManager, partitionName: string, beginning: int64, `end`: int64 +) = + ## The given partition range has seconds resolution. + ## We just store information of the new added partition merely to keep track of it. + let partitionInfo = Partition(name: partitionName, timeRange: (beginning, `end`)) + trace "Adding partition info" + self.partitions.addLast(partitionInfo) + +proc removeOldestPartitionName*(self: PartitionManager) = + ## Simply removed the partition from the tracked/known partitions queue. + ## Just remove it and ignore it. + discard self.partitions.popFirst() + +proc isEmpty*(self: PartitionManager): bool = + return self.partitions.len == 0 + +proc getLastMoment*(partition: Partition): int64 = + ## Considering the time range covered by the partition, this + ## returns the `end` time (number of seconds since epoch) of such range. + let lastTimeInSec = partition.timeRange.`end` + return lastTimeInSec + +proc getPartitionStartTimeInNanosec*(partition: Partition): int64 = + return partition.timeRange.beginning * 1_000_000_000 + +proc containsMoment*(partition: Partition, time: int64): bool = + ## Returns true if the given moment is contained within the partition window, + ## 'false' otherwise. + ## time - number of seconds since epoch + if partition.timeRange.beginning <= time and time < partition.timeRange.`end`: + return true + + return false + +proc getName*(partition: Partition): string = + return partition.name + +func `==`*(a, b: Partition): bool {.inline.} = + return a.name == b.name diff --git a/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim new file mode 100644 index 000000000..6895813f3 --- /dev/null +++ b/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim @@ -0,0 +1,1159 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import + std/[nre, options, sequtils, strutils, strformat, times], + stew/[byteutils, arrayops], + results, + chronos, + db_connector/[postgres, db_common], + chronicles +import + ../../../common/error_handling, + ../../../waku_core, + ../../common, + ../../driver, + ../../../common/databases/db_postgres as waku_postgres, + ./postgres_healthcheck, + ./partitions_manager + +type PostgresDriver* = ref object of ArchiveDriver + ## Establish a separate pools for read/write operations + writeConnPool: PgAsyncPool + readConnPool: PgAsyncPool + + ## Partition container + partitionMngr: PartitionManager + futLoopPartitionFactory: Future[void] + +const InsertRowStmtName = "InsertRow" +const InsertRowStmtDefinition = # TODO: get the sql queries from a file + """INSERT INTO messages (id, messageHash, contentTopic, payload, pubsubTopic, + version, timestamp, meta) VALUES ($1, $2, $3, $4, $5, $6, $7, CASE WHEN $8 = '' THEN NULL ELSE $8 END) ON CONFLICT DO NOTHING;""" + +const SelectNoCursorAscStmtName = "SelectWithoutCursorAsc" +const SelectNoCursorAscStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + timestamp >= $4 AND + timestamp <= $5 + ORDER BY timestamp ASC, messageHash ASC LIMIT $6;""" + +const SelectNoCursorDescStmtName = "SelectWithoutCursorDesc" +const SelectNoCursorDescStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + timestamp >= $4 AND + timestamp <= $5 + ORDER BY timestamp DESC, messageHash DESC LIMIT $6;""" + +const SelectWithCursorDescStmtName = "SelectWithCursorDesc" +const SelectWithCursorDescStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + (timestamp, messageHash) < ($4,$5) AND + timestamp >= $6 AND + timestamp <= $7 + ORDER BY timestamp DESC, messageHash DESC LIMIT $8;""" + +const SelectWithCursorAscStmtName = "SelectWithCursorAsc" +const SelectWithCursorAscStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + messageHash IN ($2) AND + pubsubTopic = $3 AND + (timestamp, messageHash) > ($4,$5) AND + timestamp >= $6 AND + timestamp <= $7 + ORDER BY timestamp ASC, messageHash ASC LIMIT $8;""" + +const SelectMessageByHashName = "SelectMessageByHash" +const SelectMessageByHashDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages WHERE messageHash = $1""" + +const SelectNoCursorV2AscStmtName = "SelectWithoutCursorV2Asc" +const SelectNoCursorV2AscStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + pubsubTopic = $2 AND + timestamp >= $3 AND + timestamp <= $4 + ORDER BY timestamp ASC LIMIT $5;""" + +const SelectNoCursorV2DescStmtName = "SelectWithoutCursorV2Desc" +const SelectNoCursorV2DescStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + pubsubTopic = $2 AND + timestamp >= $3 AND + timestamp <= $4 + ORDER BY timestamp DESC LIMIT $5;""" + +const SelectWithCursorV2DescStmtName = "SelectWithCursorV2Desc" +const SelectWithCursorV2DescStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + pubsubTopic = $2 AND + (timestamp, id) < ($3,$4) AND + timestamp >= $5 AND + timestamp <= $6 + ORDER BY timestamp DESC LIMIT $7;""" + +const SelectWithCursorV2AscStmtName = "SelectWithCursorV2Asc" +const SelectWithCursorV2AscStmtDef = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages + WHERE contentTopic IN ($1) AND + pubsubTopic = $2 AND + (timestamp, id) > ($3,$4) AND + timestamp >= $5 AND + timestamp <= $6 + ORDER BY timestamp ASC LIMIT $7;""" + +const DefaultMaxNumConns = 50 + +proc new*( + T: type PostgresDriver, + dbUrl: string, + maxConnections = DefaultMaxNumConns, + onFatalErrorAction: OnFatalErrorHandler = nil, +): ArchiveDriverResult[T] = + ## Very simplistic split of max connections + let maxNumConnOnEachPool = int(maxConnections / 2) + + let readConnPool = PgAsyncPool.new(dbUrl, maxNumConnOnEachPool).valueOr: + return err("error creating read conn pool PgAsyncPool") + + let writeConnPool = PgAsyncPool.new(dbUrl, maxNumConnOnEachPool).valueOr: + return err("error creating write conn pool PgAsyncPool") + + if not isNil(onFatalErrorAction): + asyncSpawn checkConnectivity(readConnPool, onFatalErrorAction) + + if not isNil(onFatalErrorAction): + asyncSpawn checkConnectivity(writeConnPool, onFatalErrorAction) + + let driver = PostgresDriver( + writeConnPool: writeConnPool, + readConnPool: readConnPool, + partitionMngr: PartitionManager.new(), + ) + return ok(driver) + +proc reset*(s: PostgresDriver): Future[ArchiveDriverResult[void]] {.async.} = + ## Clear the database partitions + let targetSize = 0 + let forceRemoval = true + let ret = await s.decreaseDatabaseSize(targetSize, forceRemoval) + return ret + +proc rowCallbackImpl( + pqResult: ptr PGresult, + outRows: var seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)], +) = + ## Proc aimed to contain the logic of the callback passed to the `psasyncpool`. + ## That callback is used in "SELECT" queries. + ## + ## pqResult - contains the query results + ## outRows - seq of Store-rows. This is populated from the info contained in pqResult + + let numFields = pqResult.pqnfields() + if numFields != 8: + error "Wrong number of fields, expected 8", numFields + return + + for iRow in 0 ..< pqResult.pqNtuples(): + var wakuMessage: WakuMessage + var timestamp: Timestamp + var version: uint + var pubSubTopic: string + var contentTopic: string + var digest: string + var payload: string + var hashHex: string + var msgHash: WakuMessageHash + var meta: string + + try: + contentTopic = $(pqgetvalue(pqResult, iRow, 0)) + payload = parseHexStr($(pqgetvalue(pqResult, iRow, 1))) + pubSubTopic = $(pqgetvalue(pqResult, iRow, 2)) + version = parseUInt($(pqgetvalue(pqResult, iRow, 3))) + timestamp = parseInt($(pqgetvalue(pqResult, iRow, 4))) + digest = parseHexStr($(pqgetvalue(pqResult, iRow, 5))) + hashHex = parseHexStr($(pqgetvalue(pqResult, iRow, 6))) + meta = parseHexStr($(pqgetvalue(pqResult, iRow, 7))) + msgHash = fromBytes(hashHex.toOpenArrayByte(0, 31)) + except ValueError: + error "could not parse correctly", error = getCurrentExceptionMsg() + + wakuMessage.timestamp = timestamp + wakuMessage.version = uint32(version) + wakuMessage.contentTopic = contentTopic + wakuMessage.payload = @(payload.toOpenArrayByte(0, payload.high)) + wakuMessage.meta = @(meta.toOpenArrayByte(0, meta.high)) + + outRows.add( + ( + pubSubTopic, + wakuMessage, + @(digest.toOpenArrayByte(0, digest.high)), + timestamp, + msgHash, + ) + ) + +method put*( + s: PostgresDriver, + pubsubTopic: PubsubTopic, + message: WakuMessage, + digest: MessageDigest, + messageHash: WakuMessageHash, + receivedTime: Timestamp, +): Future[ArchiveDriverResult[void]] {.async.} = + let digest = toHex(digest.data) + let messageHash = toHex(messageHash) + let contentTopic = message.contentTopic + let payload = toHex(message.payload) + let version = $message.version + let timestamp = $message.timestamp + let meta = toHex(message.meta) + + trace "put PostgresDriver", timestamp = timestamp + + return await s.writeConnPool.runStmt( + InsertRowStmtName, + InsertRowStmtDefinition, + @[digest, messageHash, contentTopic, payload, pubsubTopic, version, timestamp, meta], + @[ + int32(digest.len), + int32(messageHash.len), + int32(contentTopic.len), + int32(payload.len), + int32(pubsubTopic.len), + int32(version.len), + int32(timestamp.len), + int32(meta.len), + ], + @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], + ) + +method getAllMessages*( + s: PostgresDriver +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + ## Retrieve all messages from the store. + + var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + proc rowCallback(pqResult: ptr PGresult) = + rowCallbackImpl(pqResult, rows) + + ( + await s.readConnPool.pgQuery( + """SELECT contentTopic, + payload, pubsubTopic, version, timestamp, + id, messageHash, meta FROM messages ORDER BY timestamp ASC""", + newSeq[string](0), + rowCallback, + ) + ).isOkOr: + return err("failed in query: " & $error) + + return ok(rows) + +proc getPartitionsList( + s: PostgresDriver +): Future[ArchiveDriverResult[seq[string]]] {.async.} = + ## Retrieves the seq of partition table names. + ## e.g: @["messages_1708534333_1708534393", "messages_1708534273_1708534333"] + + var partitions: seq[string] + proc rowCallback(pqResult: ptr PGresult) = + for iRow in 0 ..< pqResult.pqNtuples(): + let partitionName = $(pqgetvalue(pqResult, iRow, 0)) + partitions.add(partitionName) + + ( + await s.readConnPool.pgQuery( + """ + SELECT child.relname AS partition_name + FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace + JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace + WHERE parent.relname='messages' + ORDER BY partition_name ASC + """, + newSeq[string](0), + rowCallback, + ) + ).isOkOr: + return err("getPartitionsList failed in query: " & $error) + + return ok(partitions) + +proc getMessagesArbitraryQuery( + s: PostgresDriver, + contentTopic: seq[ContentTopic] = @[], + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hexHashes: seq[string] = @[], + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + ## This proc allows to handle atypical queries. We don't use prepared statements for those. + + var query = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages""" + var statements: seq[string] + var args: seq[string] + + if contentTopic.len > 0: + let cstmt = "contentTopic IN (" & "?".repeat(contentTopic.len).join(",") & ")" + statements.add(cstmt) + for t in contentTopic: + args.add(t) + + if hexHashes.len > 0: + let cstmt = "messageHash IN (" & "?".repeat(hexHashes.len).join(",") & ")" + statements.add(cstmt) + for t in hexHashes: + args.add(t) + + if pubsubTopic.isSome(): + statements.add("pubsubTopic = ?") + args.add(pubsubTopic.get()) + + if cursor.isSome(): + let hashHex = toHex(cursor.get().hash) + + var entree: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + proc entreeCallback(pqResult: ptr PGresult) = + rowCallbackImpl(pqResult, entree) + + ( + await s.readConnPool.runStmt( + SelectMessageByHashName, + SelectMessageByHashDef, + @[hashHex], + @[int32(hashHex.len)], + @[int32(0)], + entreeCallback, + ) + ).isOkOr: + return err("failed to run query with cursor: " & $error) + + if entree.len == 0: + return ok(entree) + + let storetime = entree[0][3] + + let comp = if ascendingOrder: ">" else: "<" + statements.add("(timestamp, messageHash) " & comp & " (?,?)") + args.add($storetime) + args.add(hashHex) + + if startTime.isSome(): + statements.add("timestamp >= ?") + args.add($startTime.get()) + + if endTime.isSome(): + statements.add("timestamp <= ?") + args.add($endTime.get()) + + if statements.len > 0: + query &= " WHERE " & statements.join(" AND ") + + var direction: string + if ascendingOrder: + direction = "ASC" + else: + direction = "DESC" + + query &= " ORDER BY timestamp " & direction & ", messageHash " & direction + + query &= " LIMIT ?" + args.add($maxPageSize) + + var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + proc rowCallback(pqResult: ptr PGresult) = + rowCallbackImpl(pqResult, rows) + + (await s.readConnPool.pgQuery(query, args, rowCallback)).isOkOr: + return err("failed to run query: " & $error) + + return ok(rows) + +proc getMessagesV2ArbitraryQuery( + s: PostgresDriver, + contentTopic: seq[ContentTopic] = @[], + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = + ## This proc allows to handle atypical queries. We don't use prepared statements for those. + + var query = + """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages""" + var statements: seq[string] + var args: seq[string] + + if contentTopic.len > 0: + let cstmt = "contentTopic IN (" & "?".repeat(contentTopic.len).join(",") & ")" + statements.add(cstmt) + for t in contentTopic: + args.add(t) + + if pubsubTopic.isSome(): + statements.add("pubsubTopic = ?") + args.add(pubsubTopic.get()) + + if cursor.isSome(): + let comp = if ascendingOrder: ">" else: "<" + statements.add("(timestamp, id) " & comp & " (?,?)") + args.add($cursor.get().storeTime) + args.add(toHex(cursor.get().digest.data)) + + if startTime.isSome(): + statements.add("timestamp >= ?") + args.add($startTime.get()) + + if endTime.isSome(): + statements.add("timestamp <= ?") + args.add($endTime.get()) + + if statements.len > 0: + query &= " WHERE " & statements.join(" AND ") + + var direction: string + if ascendingOrder: + direction = "ASC" + else: + direction = "DESC" + + query &= " ORDER BY timestamp " & direction & ", id " & direction + + query &= " LIMIT ?" + args.add($maxPageSize) + + var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + proc rowCallback(pqResult: ptr PGresult) = + rowCallbackImpl(pqResult, rows) + + (await s.readConnPool.pgQuery(query, args, rowCallback)).isOkOr: + return err("failed to run query: " & $error) + + return ok(rows) + +proc getMessagesPreparedStmt( + s: PostgresDriver, + contentTopic: string, + pubsubTopic: PubsubTopic, + cursor = none(ArchiveCursor), + startTime: Timestamp, + endTime: Timestamp, + hashes: string, + maxPageSize = DefaultPageSize, + ascOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + ## This proc aims to run the most typical queries in a more performant way, i.e. by means of + ## prepared statements. + ## + ## contentTopic - string with list of conten topics. e.g: "'ctopic1','ctopic2','ctopic3'" + + var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + proc rowCallback(pqResult: ptr PGresult) = + rowCallbackImpl(pqResult, rows) + + let startTimeStr = $startTime + let endTimeStr = $endTime + let limit = $maxPageSize + + if cursor.isSome(): + let hash = toHex(cursor.get().hash) + + var entree: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + + proc entreeCallback(pqResult: ptr PGresult) = + rowCallbackImpl(pqResult, entree) + + ( + await s.readConnPool.runStmt( + SelectMessageByHashName, + SelectMessageByHashDef, + @[hash], + @[int32(hash.len)], + @[int32(0)], + entreeCallback, + ) + ).isOkOr: + return err("failed to run query with cursor: " & $error) + + if entree.len == 0: + return ok(entree) + + let timestamp = $entree[0][3] + + var stmtName = + if ascOrder: SelectWithCursorAscStmtName else: SelectWithCursorDescStmtName + var stmtDef = + if ascOrder: SelectWithCursorAscStmtDef else: SelectWithCursorDescStmtDef + + ( + await s.readConnPool.runStmt( + stmtName, + stmtDef, + @[ + contentTopic, hashes, pubsubTopic, timestamp, hash, startTimeStr, endTimeStr, + limit, + ], + @[ + int32(contentTopic.len), + int32(hashes.len), + int32(pubsubTopic.len), + int32(timestamp.len), + int32(hash.len), + int32(startTimeStr.len), + int32(endTimeStr.len), + int32(limit.len), + ], + @[ + int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0) + ], + rowCallback, + ) + ).isOkOr: + return err("failed to run query with cursor: " & $error) + else: + var stmtName = + if ascOrder: SelectNoCursorAscStmtName else: SelectNoCursorDescStmtName + var stmtDef = if ascOrder: SelectNoCursorAscStmtDef else: SelectNoCursorDescStmtDef + + ( + await s.readConnPool.runStmt( + stmtName, + stmtDef, + @[contentTopic, hashes, pubsubTopic, startTimeStr, endTimeStr, limit], + @[ + int32(contentTopic.len), + int32(hashes.len), + int32(pubsubTopic.len), + int32(startTimeStr.len), + int32(endTimeStr.len), + int32(limit.len), + ], + @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], + rowCallback, + ) + ).isOkOr: + return err("failed to run query without cursor: " & $error) + + return ok(rows) + +proc getMessagesV2PreparedStmt( + s: PostgresDriver, + contentTopic: string, + pubsubTopic: PubsubTopic, + cursor = none(ArchiveCursor), + startTime: Timestamp, + endTime: Timestamp, + maxPageSize = DefaultPageSize, + ascOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = + ## This proc aims to run the most typical queries in a more performant way, i.e. by means of + ## prepared statements. + ## + ## contentTopic - string with list of conten topics. e.g: "'ctopic1','ctopic2','ctopic3'" + + var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + proc rowCallback(pqResult: ptr PGresult) = + rowCallbackImpl(pqResult, rows) + + let startTimeStr = $startTime + let endTimeStr = $endTime + let limit = $maxPageSize + + if cursor.isSome(): + var stmtName = + if ascOrder: SelectWithCursorV2AscStmtName else: SelectWithCursorV2DescStmtName + var stmtDef = + if ascOrder: SelectWithCursorV2AscStmtDef else: SelectWithCursorV2DescStmtDef + + let digest = toHex(cursor.get().digest.data) + let timestamp = $cursor.get().storeTime + + ( + await s.readConnPool.runStmt( + stmtName, + stmtDef, + @[contentTopic, pubsubTopic, timestamp, digest, startTimeStr, endTimeStr, limit], + @[ + int32(contentTopic.len), + int32(pubsubTopic.len), + int32(timestamp.len), + int32(digest.len), + int32(startTimeStr.len), + int32(endTimeStr.len), + int32(limit.len), + ], + @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], + rowCallback, + ) + ).isOkOr: + return err("failed to run query with cursor: " & $error) + else: + var stmtName = + if ascOrder: SelectNoCursorV2AscStmtName else: SelectNoCursorV2DescStmtName + var stmtDef = + if ascOrder: SelectNoCursorV2AscStmtDef else: SelectNoCursorV2DescStmtDef + + ( + await s.readConnPool.runStmt( + stmtName, + stmtDef, + @[contentTopic, pubsubTopic, startTimeStr, endTimeStr, limit], + @[ + int32(contentTopic.len), + int32(pubsubTopic.len), + int32(startTimeStr.len), + int32(endTimeStr.len), + int32(limit.len), + ], + @[int32(0), int32(0), int32(0), int32(0), int32(0)], + rowCallback, + ) + ).isOkOr: + return err("failed to run query without cursor: " & $error) + + return ok(rows) + +method getMessages*( + s: PostgresDriver, + includeData = true, + contentTopicSeq = newSeq[ContentTopic](0), + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hashes = newSeq[WakuMessageHash](0), + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + let hexHashes = hashes.mapIt(toHex(it)) + + if contentTopicSeq.len == 1 and hexHashes.len == 1 and pubsubTopic.isSome() and + startTime.isSome() and endTime.isSome(): + ## Considered the most common query. Therefore, we use prepared statements to optimize it. + return await s.getMessagesPreparedStmt( + contentTopicSeq.join(","), + PubsubTopic(pubsubTopic.get()), + cursor, + startTime.get(), + endTime.get(), + hexHashes.join(","), + maxPageSize, + ascendingOrder, + ) + else: + ## We will run atypical query. In this case we don't use prepared statemets + return await s.getMessagesArbitraryQuery( + contentTopicSeq, pubsubTopic, cursor, startTime, endTime, hexHashes, maxPageSize, + ascendingOrder, + ) + +method getMessagesV2*( + s: PostgresDriver, + contentTopicSeq = newSeq[ContentTopic](0), + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = + if contentTopicSeq.len == 1 and pubsubTopic.isSome() and startTime.isSome() and + endTime.isSome(): + ## Considered the most common query. Therefore, we use prepared statements to optimize it. + return await s.getMessagesV2PreparedStmt( + contentTopicSeq.join(","), + PubsubTopic(pubsubTopic.get()), + cursor, + startTime.get(), + endTime.get(), + maxPageSize, + ascendingOrder, + ) + else: + ## We will run atypical query. In this case we don't use prepared statemets + return await s.getMessagesV2ArbitraryQuery( + contentTopicSeq, pubsubTopic, cursor, startTime, endTime, maxPageSize, + ascendingOrder, + ) + +proc getStr( + s: PostgresDriver, query: string +): Future[ArchiveDriverResult[string]] {.async.} = + # Performs a query that is expected to return a single string + + var ret: string + proc rowCallback(pqResult: ptr PGresult) = + if pqResult.pqnfields() != 1: + error "Wrong number of fields in getStr" + return + + if pqResult.pqNtuples() != 1: + error "Wrong number of rows in getStr" + return + + ret = $(pqgetvalue(pqResult, 0, 0)) + + (await s.readConnPool.pgQuery(query, newSeq[string](0), rowCallback)).isOkOr: + return err("failed in getRow: " & $error) + + return ok(ret) + +proc getInt( + s: PostgresDriver, query: string +): Future[ArchiveDriverResult[int64]] {.async.} = + # Performs a query that is expected to return a single numeric value (int64) + + var retInt = 0'i64 + let str = (await s.getStr(query)).valueOr: + return err("could not get str in getInt: " & $error) + + try: + retInt = parseInt(str) + except ValueError: + return err( + "exception in getInt, parseInt, str: " & str & " query: " & query & " exception: " & + getCurrentExceptionMsg() + ) + + return ok(retInt) + +method getDatabaseSize*( + s: PostgresDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + let intRes = (await s.getInt("SELECT pg_database_size(current_database())")).valueOr: + return err("error in getDatabaseSize: " & error) + + let databaseSize: int64 = int64(intRes) + return ok(databaseSize) + +method getMessagesCount*( + s: PostgresDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + let intRes = await s.getInt("SELECT COUNT(1) FROM messages") + if intRes.isErr(): + return err("error in getMessagesCount: " & intRes.error) + + return ok(intRes.get()) + +method getOldestMessageTimestamp*( + s: PostgresDriver +): Future[ArchiveDriverResult[Timestamp]] {.async.} = + ## In some cases it could happen that we have + ## empty partitions which are older than the current stored rows. + ## In those cases we want to consider those older partitions as the oldest considered timestamp. + let oldestPartition = s.partitionMngr.getOldestPartition().valueOr: + return err("could not get oldest partition: " & $error) + + let oldestPartitionTimeNanoSec = oldestPartition.getPartitionStartTimeInNanosec() + + let intRes = await s.getInt("SELECT MIN(timestamp) FROM messages") + if intRes.isErr(): + ## Just return the oldest partition time considering the partitions set + return ok(Timestamp(oldestPartitionTimeNanoSec)) + + return ok(Timestamp(min(intRes.get(), oldestPartitionTimeNanoSec))) + +method getNewestMessageTimestamp*( + s: PostgresDriver +): Future[ArchiveDriverResult[Timestamp]] {.async.} = + let intRes = await s.getInt("SELECT MAX(timestamp) FROM messages") + if intRes.isErr(): + return err("error in getNewestMessageTimestamp: " & intRes.error) + + return ok(Timestamp(intRes.get())) + +method deleteOldestMessagesNotWithinLimit*( + s: PostgresDriver, limit: int +): Future[ArchiveDriverResult[void]] {.async.} = + let execRes = await s.writeConnPool.pgQuery( + """DELETE FROM messages WHERE id NOT IN + ( + SELECT id FROM messages ORDER BY timestamp DESC LIMIT ? + );""", + @[$limit], + ) + if execRes.isErr(): + return err("error in deleteOldestMessagesNotWithinLimit: " & execRes.error) + + return ok() + +method close*(s: PostgresDriver): Future[ArchiveDriverResult[void]] {.async.} = + ## Cancel the partition factory loop + s.futLoopPartitionFactory.cancel() + + ## Close the database connection + let writeCloseRes = await s.writeConnPool.close() + let readCloseRes = await s.readConnPool.close() + + writeCloseRes.isOkOr: + return err("error closing write pool: " & $error) + + readCloseRes.isOkOr: + return err("error closing read pool: " & $error) + + return ok() + +proc sleep*( + s: PostgresDriver, seconds: int +): Future[ArchiveDriverResult[void]] {.async.} = + # This is for testing purposes only. It is aimed to test the proper + # implementation of asynchronous requests. It merely triggers a sleep in the + # database for the amount of seconds given as a parameter. + + proc rowCallback(result: ptr PGresult) = + ## We are not interested in any value in this case + discard + + try: + let params = @[$seconds] + (await s.writeConnPool.pgQuery("SELECT pg_sleep(?)", params, rowCallback)).isOkOr: + return err("error in postgres_driver sleep: " & $error) + except DbError: + # This always raises an exception although the sleep works + return err("exception sleeping: " & getCurrentExceptionMsg()) + + return ok() + +proc performWriteQuery*( + s: PostgresDriver, query: string +): Future[ArchiveDriverResult[void]] {.async.} = + ## Performs a query that somehow changes the state of the database + + (await s.writeConnPool.pgQuery(query)).isOkOr: + return err("error in performWriteQuery: " & $error) + + return ok() + +proc addPartition( + self: PostgresDriver, startTime: Timestamp, duration: timer.Duration +): Future[ArchiveDriverResult[void]] {.async.} = + ## Creates a partition table that will store the messages that fall in the range + ## `startTime` <= timestamp < `startTime + duration`. + ## `startTime` is measured in seconds since epoch + + let beginning = startTime + let `end` = (startTime + duration.seconds) + + let fromInSec: string = $beginning + let untilInSec: string = $`end` + + let fromInNanoSec: string = fromInSec & "000000000" + let untilInNanoSec: string = untilInSec & "000000000" + + let partitionName = "messages_" & fromInSec & "_" & untilInSec + + let createPartitionQuery = + "CREATE TABLE IF NOT EXISTS " & partitionName & " PARTITION OF " & + "messages FOR VALUES FROM ('" & fromInNanoSec & "') TO ('" & untilInNanoSec & "');" + + (await self.performWriteQuery(createPartitionQuery)).isOkOr: + return err(fmt"error adding partition [{partitionName}]: " & $error) + + debug "new partition added", query = createPartitionQuery + + self.partitionMngr.addPartitionInfo(partitionName, beginning, `end`) + return ok() + +proc initializePartitionsInfo( + self: PostgresDriver +): Future[ArchiveDriverResult[void]] {.async.} = + let partitionNamesRes = await self.getPartitionsList() + if not partitionNamesRes.isOk(): + return err("Could not retrieve partitions list: " & $partitionNamesRes.error) + else: + let partitionNames = partitionNamesRes.get() + for partitionName in partitionNames: + ## partitionName contains something like 'messages_1708449815_1708449875' + let bothTimes = partitionName.replace("messages_", "") + let times = bothTimes.split("_") + if times.len != 2: + return err(fmt"loopPartitionFactory wrong partition name {partitionName}") + + var beginning: int64 + try: + beginning = parseInt(times[0]) + except ValueError: + return err("Could not parse beginning time: " & getCurrentExceptionMsg()) + + var `end`: int64 + try: + `end` = parseInt(times[1]) + except ValueError: + return err("Could not parse end time: " & getCurrentExceptionMsg()) + + self.partitionMngr.addPartitionInfo(partitionName, beginning, `end`) + + return ok() + +const DefaultDatabasePartitionCheckTimeInterval = timer.minutes(10) +const PartitionsRangeInterval = timer.hours(1) ## Time range covered by each parition + +proc loopPartitionFactory( + self: PostgresDriver, onFatalError: OnFatalErrorHandler +) {.async.} = + ## Loop proc that continuously checks whether we need to create a new partition. + ## Notice that the deletion of partitions is handled by the retention policy modules. + + debug "starting loopPartitionFactory" + + if PartitionsRangeInterval < DefaultDatabasePartitionCheckTimeInterval: + onFatalError( + "partition factory partition range interval should be bigger than check interval" + ) + + ## First of all, let's make the 'partition_manager' aware of the current partitions + (await self.initializePartitionsInfo()).isOkOr: + onFatalError("issue in loopPartitionFactory: " & $error) + + while true: + trace "Check if we need to create a new partition" + + let now = times.now().toTime().toUnix() + + if self.partitionMngr.isEmpty(): + debug "adding partition because now there aren't more partitions" + (await self.addPartition(now, PartitionsRangeInterval)).isOkOr: + onFatalError("error when creating a new partition from empty state: " & $error) + else: + let newestPartitionRes = self.partitionMngr.getNewestPartition() + if newestPartitionRes.isErr(): + onFatalError("could not get newest partition: " & $newestPartitionRes.error) + + let newestPartition = newestPartitionRes.get() + if newestPartition.containsMoment(now): + debug "creating a new partition for the future" + ## The current used partition is the last one that was created. + ## Thus, let's create another partition for the future. + + ( + await self.addPartition( + newestPartition.getLastMoment(), PartitionsRangeInterval + ) + ).isOkOr: + onFatalError("could not add the next partition for 'now': " & $error) + elif now >= newestPartition.getLastMoment(): + debug "creating a new partition to contain current messages" + ## There is no partition to contain the current time. + ## This happens if the node has been stopped for quite a long time. + ## Then, let's create the needed partition to contain 'now'. + (await self.addPartition(now, PartitionsRangeInterval)).isOkOr: + onFatalError("could not add the next partition: " & $error) + + await sleepAsync(DefaultDatabasePartitionCheckTimeInterval) + +proc startPartitionFactory*( + self: PostgresDriver, onFatalError: OnFatalErrorHandler +) {.async.} = + self.futLoopPartitionFactory = self.loopPartitionFactory(onFatalError) + +proc getTableSize*( + self: PostgresDriver, tableName: string +): Future[ArchiveDriverResult[string]] {.async.} = + ## Returns a human-readable representation of the size for the requested table. + ## tableName - table of interest. + + let tableSize = ( + await self.getStr( + fmt""" + SELECT pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size" + FROM pg_class C + where relname = '{tableName}'""" + ) + ).valueOr: + return err("error in getDatabaseSize: " & error) + + return ok(tableSize) + +proc removePartition( + self: PostgresDriver, partitionName: string +): Future[ArchiveDriverResult[void]] {.async.} = + var partSize = "" + let partSizeRes = await self.getTableSize(partitionName) + if partSizeRes.isOk(): + partSize = partSizeRes.get() + + ## Detach and remove the partition concurrently to not block the parent table (messages) + let detachPartitionQuery = + "ALTER TABLE messages DETACH PARTITION " & partitionName & " CONCURRENTLY;" + debug "removeOldestPartition", query = detachPartitionQuery + (await self.performWriteQuery(detachPartitionQuery)).isOkOr: + return err(fmt"error in {detachPartitionQuery}: " & $error) + + ## Drop the partition + let dropPartitionQuery = "DROP TABLE " & partitionName + debug "removeOldestPartition drop partition", query = dropPartitionQuery + (await self.performWriteQuery(dropPartitionQuery)).isOkOr: + return err(fmt"error in {dropPartitionQuery}: " & $error) + + debug "removed partition", partition_name = partitionName, partition_size = partSize + self.partitionMngr.removeOldestPartitionName() + + return ok() + +proc removePartitionsOlderThan( + self: PostgresDriver, tsInNanoSec: Timestamp +): Future[ArchiveDriverResult[void]] {.async.} = + ## Removes old partitions that don't contain the specified timestamp + + let tsInSec = Timestamp(float(tsInNanoSec) / 1_000_000_000) + + var oldestPartition = self.partitionMngr.getOldestPartition().valueOr: + return err("could not get oldest partition in removePartitionOlderThan: " & $error) + + while not oldestPartition.containsMoment(tsInSec): + (await self.removePartition(oldestPartition.getName())).isOkOr: + return err("issue in removePartitionsOlderThan: " & $error) + + oldestPartition = self.partitionMngr.getOldestPartition().valueOr: + return err( + "could not get partition in removePartitionOlderThan in while loop: " & $error + ) + + ## We reached the partition that contains the target timestamp plus don't want to remove it + return ok() + +proc removeOldestPartition( + self: PostgresDriver, forceRemoval: bool = false, ## To allow cleanup in tests +): Future[ArchiveDriverResult[void]] {.async.} = + ## Indirectly called from the retention policy + + let oldestPartition = self.partitionMngr.getOldestPartition().valueOr: + return err("could not remove oldest partition: " & $error) + + if not forceRemoval: + let now = times.now().toTime().toUnix() + let currentPartitionRes = self.partitionMngr.getPartitionFromDateTime(now) + if currentPartitionRes.isOk(): + ## The database contains a partition that would store current messages. + + if currentPartitionRes.get() == oldestPartition: + debug "Skipping to remove the current partition" + return ok() + + return await self.removePartition(oldestPartition.getName()) + +proc containsAnyPartition*(self: PostgresDriver): bool = + return not self.partitionMngr.isEmpty() + +method decreaseDatabaseSize*( + driver: PostgresDriver, targetSizeInBytes: int64, forceRemoval: bool = false +): Future[ArchiveDriverResult[void]] {.async.} = + var dbSize = (await driver.getDatabaseSize()).valueOr: + return err("decreaseDatabaseSize failed to get database size: " & $error) + + ## database size in bytes + var totalSizeOfDB: int64 = int64(dbSize) + + if totalSizeOfDB <= targetSizeInBytes: + return ok() + + debug "start reducing database size", + targetSize = $targetSizeInBytes, currentSize = $totalSizeOfDB + + while totalSizeOfDB > targetSizeInBytes and driver.containsAnyPartition(): + (await driver.removeOldestPartition(forceRemoval)).isOkOr: + return err( + "decreaseDatabaseSize inside loop failed to remove oldest partition: " & $error + ) + + dbSize = (await driver.getDatabaseSize()).valueOr: + return + err("decreaseDatabaseSize inside loop failed to get database size: " & $error) + + let newCurrentSize = int64(dbSize) + if newCurrentSize == totalSizeOfDB: + return err("the previous partition removal didn't clear database size") + + totalSizeOfDB = newCurrentSize + + debug "reducing database size", + targetSize = $targetSizeInBytes, newCurrentSize = $totalSizeOfDB + + return ok() + +method existsTable*( + s: PostgresDriver, tableName: string +): Future[ArchiveDriverResult[bool]] {.async.} = + let query: string = + fmt""" + SELECT EXISTS ( + SELECT FROM + pg_tables + WHERE + tablename = '{tableName}' + ); + """ + + var exists: string + proc rowCallback(pqResult: ptr PGresult) = + if pqResult.pqnfields() != 1: + error "Wrong number of fields in existsTable" + return + + if pqResult.pqNtuples() != 1: + error "Wrong number of rows in existsTable" + return + + exists = $(pqgetvalue(pqResult, 0, 0)) + + (await s.readConnPool.pgQuery(query, newSeq[string](0), rowCallback)).isOkOr: + return err("existsTable failed in getRow: " & $error) + + return ok(exists == "t") + +proc getCurrentVersion*( + s: PostgresDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + let existsVersionTable = (await s.existsTable("version")).valueOr: + return err("error in getCurrentVersion-existsTable: " & $error) + + if not existsVersionTable: + return ok(0) + + let res = (await s.getInt(fmt"SELECT version FROM version")).valueOr: + return err("error in getMessagesCount: " & $error) + + return ok(res) + +method deleteMessagesOlderThanTimestamp*( + s: PostgresDriver, tsNanoSec: Timestamp +): Future[ArchiveDriverResult[void]] {.async.} = + ## First of all, let's remove the older partitions so that we can reduce + ## the database size. + (await s.removePartitionsOlderThan(tsNanoSec)).isOkOr: + return err("error while removing older partitions: " & $error) + + ( + await s.writeConnPool.pgQuery( + "DELETE FROM messages WHERE timestamp < " & $tsNanoSec + ) + ).isOkOr: + return err("error in deleteMessagesOlderThanTimestamp: " & $error) + + return ok() diff --git a/waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim b/waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim new file mode 100644 index 000000000..ff9dff8f7 --- /dev/null +++ b/waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim @@ -0,0 +1,41 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import chronos, results +import ../../../common/databases/db_postgres, ../../../common/error_handling + +## Simple query to validate that the postgres is working and attending requests +const HealthCheckQuery = "SELECT version();" +const CheckConnectivityInterval = 60.seconds +const MaxNumTrials = 20 +const TrialInterval = 1.seconds + +proc checkConnectivity*( + connPool: PgAsyncPool, onFatalErrorAction: OnFatalErrorHandler +) {.async.} = + while true: + (await connPool.pgQuery(HealthCheckQuery)).isOkOr: + ## The connection failed once. Let's try reconnecting for a while. + ## Notice that the 'exec' proc tries to establish a new connection. + + block errorBlock: + ## Force close all the opened connections. No need to close gracefully. + (await connPool.resetConnPool()).isOkOr: + onFatalErrorAction("checkConnectivity resetConnPool error: " & error) + + var numTrial = 0 + while numTrial < MaxNumTrials: + let res = await connPool.pgQuery(HealthCheckQuery) + if res.isOk(): + ## Connection resumed. Let's go back to the normal healthcheck. + break errorBlock + + await sleepAsync(TrialInterval) + numTrial.inc() + + ## The connection couldn't be resumed. Let's inform the upper layers. + onFatalErrorAction("postgres health check error: " & error) + + await sleepAsync(CheckConnectivityInterval) diff --git a/waku/waku_archive_legacy/driver/queue_driver.nim b/waku/waku_archive_legacy/driver/queue_driver.nim new file mode 100644 index 000000000..1ea8a29d3 --- /dev/null +++ b/waku/waku_archive_legacy/driver/queue_driver.nim @@ -0,0 +1,8 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import ./queue_driver/queue_driver, ./queue_driver/index + +export queue_driver, index diff --git a/waku/waku_archive_legacy/driver/queue_driver/index.nim b/waku/waku_archive_legacy/driver/queue_driver/index.nim new file mode 100644 index 000000000..d34b550c8 --- /dev/null +++ b/waku/waku_archive_legacy/driver/queue_driver/index.nim @@ -0,0 +1,91 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import stew/byteutils, nimcrypto/sha2 +import ../../../waku_core, ../../common + +type Index* = object + ## This type contains the description of an Index used in the pagination of WakuMessages + pubsubTopic*: string + senderTime*: Timestamp # the time at which the message is generated + receiverTime*: Timestamp + digest*: MessageDigest # calculated over payload and content topic + hash*: WakuMessageHash + +proc compute*( + T: type Index, msg: WakuMessage, receivedTime: Timestamp, pubsubTopic: PubsubTopic +): T = + ## Takes a WakuMessage with received timestamp and returns its Index. + let + digest = computeDigest(msg) + senderTime = msg.timestamp + hash = computeMessageHash(pubsubTopic, msg) + + return Index( + pubsubTopic: pubsubTopic, + senderTime: senderTime, + receiverTime: receivedTime, + digest: digest, + hash: hash, + ) + +proc tohistoryCursor*(index: Index): ArchiveCursor = + return ArchiveCursor( + pubsubTopic: index.pubsubTopic, + senderTime: index.senderTime, + storeTime: index.receiverTime, + digest: index.digest, + hash: index.hash, + ) + +proc toIndex*(index: ArchiveCursor): Index = + return Index( + pubsubTopic: index.pubsubTopic, + senderTime: index.senderTime, + receiverTime: index.storeTime, + digest: index.digest, + hash: index.hash, + ) + +proc `==`*(x, y: Index): bool = + ## receiverTime plays no role in index equality + return + ( + (x.senderTime == y.senderTime) and (x.digest == y.digest) and + (x.pubsubTopic == y.pubsubTopic) + ) or (x.hash == y.hash) # this applies to store v3 queries only + +proc cmp*(x, y: Index): int = + ## compares x and y + ## returns 0 if they are equal + ## returns -1 if x < y + ## returns 1 if x > y + ## + ## Default sorting order priority is: + ## 1. senderTimestamp + ## 2. receiverTimestamp (a fallback only if senderTimestamp unset on either side, and all other fields unequal) + ## 3. message digest + ## 4. pubsubTopic + + if x == y: + # Quick exit ensures receiver time does not affect index equality + return 0 + + # Timestamp has a higher priority for comparison + let + # Use receiverTime where senderTime is unset + xTimestamp = if x.senderTime == 0: x.receiverTime else: x.senderTime + yTimestamp = if y.senderTime == 0: y.receiverTime else: y.senderTime + + let timecmp = cmp(xTimestamp, yTimestamp) + if timecmp != 0: + return timecmp + + # Continue only when timestamps are equal + let digestcmp = cmp(x.digest.data, y.digest.data) + if digestcmp != 0: + return digestcmp + + return cmp(x.pubsubTopic, y.pubsubTopic) diff --git a/waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim b/waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim new file mode 100644 index 000000000..85c30823a --- /dev/null +++ b/waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim @@ -0,0 +1,363 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/options, results, stew/sorted_set, chronicles, chronos +import ../../../waku_core, ../../common, ../../driver, ./index + +logScope: + topics = "waku archive queue_store" + +const QueueDriverDefaultMaxCapacity* = 25_000 + +type + QueryFilterMatcher = + proc(index: Index, msg: WakuMessage): bool {.gcsafe, raises: [], closure.} + + QueueDriver* = ref object of ArchiveDriver + ## Bounded repository for indexed messages + ## + ## The store queue will keep messages up to its + ## configured capacity. As soon as this capacity + ## is reached and a new message is added, the oldest + ## item will be removed to make space for the new one. + ## This implies both a `delete` and `add` operation + ## for new items. + + # TODO: a circular/ring buffer may be a more efficient implementation + items: SortedSet[Index, WakuMessage] # sorted set of stored messages + capacity: int # Maximum amount of messages to keep + + QueueDriverErrorKind {.pure.} = enum + INVALID_CURSOR + + QueueDriverGetPageResult = Result[seq[ArchiveRow], QueueDriverErrorKind] + +proc `$`(error: QueueDriverErrorKind): string = + case error + of INVALID_CURSOR: "invalid_cursor" + +### Helpers + +proc walkToCursor( + w: SortedSetWalkRef[Index, WakuMessage], startCursor: Index, forward: bool +): SortedSetResult[Index, WakuMessage] = + ## Walk to util we find the cursor + ## TODO: Improve performance here with a binary/tree search + + var nextItem = + if forward: + w.first() + else: + w.last() + + ## Fast forward until we reach the startCursor + while nextItem.isOk(): + if nextItem.value.key == startCursor: + break + + # Not yet at cursor. Continue advancing + nextItem = + if forward: + w.next() + else: + w.prev() + + return nextItem + +#### API + +proc new*(T: type QueueDriver, capacity: int = QueueDriverDefaultMaxCapacity): T = + var items = SortedSet[Index, WakuMessage].init() + return QueueDriver(items: items, capacity: capacity) + +proc contains*(driver: QueueDriver, index: Index): bool = + ## Return `true` if the store queue already contains the `index`, `false` otherwise. + return driver.items.eq(index).isOk() + +proc len*(driver: QueueDriver): int {.noSideEffect.} = + return driver.items.len + +proc getPage( + driver: QueueDriver, + pageSize: uint = 0, + forward: bool = true, + cursor: Option[Index] = none(Index), + predicate: QueryFilterMatcher = nil, +): QueueDriverGetPageResult {.raises: [].} = + ## Populate a single page in forward direction + ## Start at the `startCursor` (exclusive), or first entry (inclusive) if not defined. + ## Page size must not exceed `maxPageSize` + ## Each entry must match the `pred` + var outSeq: seq[ArchiveRow] + + var w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) + defer: + w.destroy() + + var currentEntry: SortedSetResult[Index, WakuMessage] + + # Find starting entry + if cursor.isSome(): + let cursorEntry = w.walkToCursor(cursor.get(), forward) + if cursorEntry.isErr(): + return err(QueueDriverErrorKind.INVALID_CURSOR) + + # Advance walker once more + currentEntry = + if forward: + w.next() + else: + w.prev() + else: + # Start from the beginning of the queue + currentEntry = + if forward: + w.first() + else: + w.last() + + trace "Starting page query", currentEntry = currentEntry + + ## This loop walks forward over the queue: + ## 1. from the given cursor (or first/last entry, if not provided) + ## 2. adds entries matching the predicate function to output page + ## 3. until either the end of the queue or maxPageSize is reached + var numberOfItems: uint = 0 + while currentEntry.isOk() and numberOfItems < pageSize: + trace "Continuing page query", + currentEntry = currentEntry, numberOfItems = numberOfItems + + let + key = currentEntry.value.key + data = currentEntry.value.data + + if predicate.isNil() or predicate(key, data): + numberOfItems += 1 + + outSeq.add( + (key.pubsubTopic, data, @(key.digest.data), key.receiverTime, key.hash) + ) + + currentEntry = + if forward: + w.next() + else: + w.prev() + + trace "Successfully retrieved page", len = outSeq.len + + return ok(outSeq) + +## --- SortedSet accessors --- + +iterator fwdIterator*(driver: QueueDriver): (Index, WakuMessage) = + ## Forward iterator over the entire store queue + var + w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) + res = w.first() + + while res.isOk(): + yield (res.value.key, res.value.data) + res = w.next() + + w.destroy() + +iterator bwdIterator*(driver: QueueDriver): (Index, WakuMessage) = + ## Backwards iterator over the entire store queue + var + w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) + res = w.last() + + while res.isOk(): + yield (res.value.key, res.value.data) + res = w.prev() + + w.destroy() + +proc first*(driver: QueueDriver): ArchiveDriverResult[Index] = + var + w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) + res = w.first() + w.destroy() + + if res.isErr(): + return err("Not found") + + return ok(res.value.key) + +proc last*(driver: QueueDriver): ArchiveDriverResult[Index] = + var + w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) + res = w.last() + w.destroy() + + if res.isErr(): + return err("Not found") + + return ok(res.value.key) + +## --- Queue API --- + +proc add*( + driver: QueueDriver, index: Index, msg: WakuMessage +): ArchiveDriverResult[void] = + ## Add a message to the queue + ## + ## If we're at capacity, we will be removing, the oldest (first) item + if driver.contains(index): + trace "could not add item to store queue. Index already exists", index = index + return err("duplicate") + + # TODO: the below delete block can be removed if we convert to circular buffer + if driver.items.len >= driver.capacity: + var + w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) + firstItem = w.first + + if cmp(index, firstItem.value.key) < 0: + # When at capacity, we won't add if message index is smaller (older) than our oldest item + w.destroy # Clean up walker + return err("too_old") + + discard driver.items.delete(firstItem.value.key) + w.destroy # better to destroy walker after a delete operation + + driver.items.insert(index).value.data = msg + + return ok() + +method put*( + driver: QueueDriver, + pubsubTopic: PubsubTopic, + message: WakuMessage, + digest: MessageDigest, + messageHash: WakuMessageHash, + receivedTime: Timestamp, +): Future[ArchiveDriverResult[void]] {.async.} = + let index = Index( + pubsubTopic: pubsubTopic, + senderTime: message.timestamp, + receiverTime: receivedTime, + digest: digest, + hash: messageHash, + ) + + return driver.add(index, message) + +method getAllMessages*( + driver: QueueDriver +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + # TODO: Implement this message_store method + return err("interface method not implemented") + +method existsTable*( + driver: QueueDriver, tableName: string +): Future[ArchiveDriverResult[bool]] {.async.} = + return err("interface method not implemented") + +method getMessages*( + driver: QueueDriver, + includeData = true, + contentTopic: seq[ContentTopic] = @[], + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hashes: seq[WakuMessageHash] = @[], + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + let cursor = cursor.map(toIndex) + + let matchesQuery: QueryFilterMatcher = + func (index: Index, msg: WakuMessage): bool = + if pubsubTopic.isSome() and index.pubsubTopic != pubsubTopic.get(): + return false + + if contentTopic.len > 0 and msg.contentTopic notin contentTopic: + return false + + if startTime.isSome() and msg.timestamp < startTime.get(): + return false + + if endTime.isSome() and msg.timestamp > endTime.get(): + return false + + if hashes.len > 0 and index.hash notin hashes: + return false + + return true + + var pageRes: QueueDriverGetPageResult + try: + pageRes = driver.getPage(maxPageSize, ascendingOrder, cursor, matchesQuery) + except CatchableError, Exception: + return err(getCurrentExceptionMsg()) + + if pageRes.isErr(): + return err($pageRes.error) + + return ok(pageRes.value) + +method getMessagesCount*( + driver: QueueDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + return ok(int64(driver.len())) + +method getPagesCount*( + driver: QueueDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + return ok(int64(driver.len())) + +method getPagesSize*( + driver: QueueDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + return ok(int64(driver.len())) + +method getDatabaseSize*( + driver: QueueDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + return ok(int64(driver.len())) + +method performVacuum*( + driver: QueueDriver +): Future[ArchiveDriverResult[void]] {.async.} = + return err("interface method not implemented") + +method getOldestMessageTimestamp*( + driver: QueueDriver +): Future[ArchiveDriverResult[Timestamp]] {.async.} = + return driver.first().map( + proc(index: Index): Timestamp = + index.receiverTime + ) + +method getNewestMessageTimestamp*( + driver: QueueDriver +): Future[ArchiveDriverResult[Timestamp]] {.async.} = + return driver.last().map( + proc(index: Index): Timestamp = + index.receiverTime + ) + +method deleteMessagesOlderThanTimestamp*( + driver: QueueDriver, ts: Timestamp +): Future[ArchiveDriverResult[void]] {.async.} = + # TODO: Implement this message_store method + return err("interface method not implemented") + +method deleteOldestMessagesNotWithinLimit*( + driver: QueueDriver, limit: int +): Future[ArchiveDriverResult[void]] {.async.} = + # TODO: Implement this message_store method + return err("interface method not implemented") + +method decreaseDatabaseSize*( + driver: QueueDriver, targetSizeInBytes: int64, forceRemoval: bool = false +): Future[ArchiveDriverResult[void]] {.async.} = + return err("interface method not implemented") + +method close*(driver: QueueDriver): Future[ArchiveDriverResult[void]] {.async.} = + return ok() diff --git a/waku/waku_archive_legacy/driver/sqlite_driver.nim b/waku/waku_archive_legacy/driver/sqlite_driver.nim new file mode 100644 index 000000000..027e00488 --- /dev/null +++ b/waku/waku_archive_legacy/driver/sqlite_driver.nim @@ -0,0 +1,8 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import ./sqlite_driver/sqlite_driver + +export sqlite_driver diff --git a/waku/waku_archive/driver/sqlite_driver/cursor.nim b/waku/waku_archive_legacy/driver/sqlite_driver/cursor.nim similarity index 67% rename from waku/waku_archive/driver/sqlite_driver/cursor.nim rename to waku/waku_archive_legacy/driver/sqlite_driver/cursor.nim index ada14cc24..9729f0ff7 100644 --- a/waku/waku_archive/driver/sqlite_driver/cursor.nim +++ b/waku/waku_archive_legacy/driver/sqlite_driver/cursor.nim @@ -1,4 +1,7 @@ -{.push raises: [].} +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} import ../../../waku_core, ../../common diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim b/waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim new file mode 100644 index 000000000..4c25ddf3c --- /dev/null +++ b/waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim @@ -0,0 +1,74 @@ +{.push raises: [].} + +import + std/[tables, strutils, os], results, chronicles, sqlite3_abi # sqlite3_column_int64 +import ../../../common/databases/db_sqlite, ../../../common/databases/common + +logScope: + topics = "waku archive migration" + +const SchemaVersion* = 9 # increase this when there is an update in the database schema + +template projectRoot(): string = + currentSourcePath.rsplit(DirSep, 1)[0] / ".." / ".." / ".." / ".." + +const MessageStoreMigrationPath: string = projectRoot / "migrations" / "message_store" + +proc isSchemaVersion7*(db: SqliteDatabase): DatabaseResult[bool] = + ## Temporary proc created to analyse when the table actually belongs to the SchemaVersion 7. + ## + ## During many nwaku versions, 0.14.0 until 0.18.0, the SchemaVersion wasn't set or checked. + ## Docker `nwaku` nodes that start working from these versions, 0.14.0 until 0.18.0, they started + ## with this discrepancy: `user_version`== 0 (not set) but Message table with SchemaVersion 7. + ## + ## We found issues where `user_version` (SchemaVersion) was set to 0 in the database even though + ## its scheme structure reflected SchemaVersion 7. In those cases, when `nwaku` re-started to + ## apply the migration scripts (in 0.19.0) the node didn't start properly because it tried to + ## migrate a database that already had the Schema structure #7, so it failed when changing the PK. + ## + ## TODO: This was added in version 0.20.0. We might remove this in version 0.30.0, as we + ## could consider that many users use +0.20.0. + + var pkColumns = newSeq[string]() + proc queryRowCallback(s: ptr sqlite3_stmt) = + let colName = cstring sqlite3_column_text(s, 0) + pkColumns.add($colName) + + let query = + """SELECT l.name FROM pragma_table_info("Message") as l WHERE l.pk != 0;""" + let res = db.query(query, queryRowCallback) + if res.isErr(): + return err("failed to determine the current SchemaVersion: " & $res.error) + + if pkColumns == @["pubsubTopic", "id", "storedAt"]: + return ok(true) + else: + info "Not considered schema version 7" + return ok(false) + +proc migrate*(db: SqliteDatabase, targetVersion = SchemaVersion): DatabaseResult[void] = + ## Compares the `user_version` of the sqlite database with the provided `targetVersion`, then + ## it runs migration scripts if the `user_version` is outdated. The `migrationScriptsDir` path + ## points to the directory holding the migrations scripts once the db is updated, it sets the + ## `user_version` to the `tragetVersion`. + ## + ## If not `targetVersion` is provided, it defaults to `SchemaVersion`. + ## + ## NOTE: Down migration it is not currently supported + debug "starting message store's sqlite database migration" + + let userVersion = ?db.getUserVersion() + let isSchemaVersion7 = ?db.isSchemaVersion7() + + if userVersion == 0'i64 and isSchemaVersion7: + info "We found user_version 0 but the database schema reflects the user_version 7" + ## Force the correct schema version + ?db.setUserVersion(7) + + let migrationRes = + migrate(db, targetVersion, migrationsScriptsDir = MessageStoreMigrationPath) + if migrationRes.isErr(): + return err("failed to execute migration scripts: " & migrationRes.error) + + debug "finished message store's sqlite database migration" + return ok() diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim b/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim new file mode 100644 index 000000000..76d9755e5 --- /dev/null +++ b/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim @@ -0,0 +1,744 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/[options, sequtils], stew/byteutils, sqlite3_abi, results +import + ../../../common/databases/db_sqlite, + ../../../common/databases/common, + ../../../waku_core, + ./cursor + +const DbTable = "Message" + +type SqlQueryStr = string + +### SQLite column helper methods + +proc queryRowWakuMessageCallback( + s: ptr sqlite3_stmt, + contentTopicCol, payloadCol, versionCol, senderTimestampCol, metaCol: cint, +): WakuMessage = + let + topic = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, contentTopicCol)) + topicLength = sqlite3_column_bytes(s, contentTopicCol) + contentTopic = string.fromBytes(@(toOpenArray(topic, 0, topicLength - 1))) + + p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, payloadCol)) + m = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, metaCol)) + + payloadLength = sqlite3_column_bytes(s, payloadCol) + metaLength = sqlite3_column_bytes(s, metaCol) + payload = @(toOpenArray(p, 0, payloadLength - 1)) + version = sqlite3_column_int64(s, versionCol) + senderTimestamp = sqlite3_column_int64(s, senderTimestampCol) + meta = @(toOpenArray(m, 0, metaLength - 1)) + + return WakuMessage( + contentTopic: ContentTopic(contentTopic), + payload: payload, + version: uint32(version), + timestamp: Timestamp(senderTimestamp), + meta: meta, + ) + +proc queryRowReceiverTimestampCallback( + s: ptr sqlite3_stmt, storedAtCol: cint +): Timestamp = + let storedAt = sqlite3_column_int64(s, storedAtCol) + return Timestamp(storedAt) + +proc queryRowPubsubTopicCallback( + s: ptr sqlite3_stmt, pubsubTopicCol: cint +): PubsubTopic = + let + pubsubTopicPointer = + cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, pubsubTopicCol)) + pubsubTopicLength = sqlite3_column_bytes(s, pubsubTopicCol) + pubsubTopic = + string.fromBytes(@(toOpenArray(pubsubTopicPointer, 0, pubsubTopicLength - 1))) + + return pubsubTopic + +proc queryRowDigestCallback(s: ptr sqlite3_stmt, digestCol: cint): seq[byte] = + let + digestPointer = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, digestCol)) + digestLength = sqlite3_column_bytes(s, digestCol) + digest = @(toOpenArray(digestPointer, 0, digestLength - 1)) + + return digest + +proc queryRowWakuMessageHashCallback( + s: ptr sqlite3_stmt, hashCol: cint +): WakuMessageHash = + let + hashPointer = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, hashCol)) + hashLength = sqlite3_column_bytes(s, hashCol) + hash = fromBytes(toOpenArray(hashPointer, 0, hashLength - 1)) + + return hash + +### SQLite queries + +## Create table + +proc createTableQuery(table: string): SqlQueryStr = + "CREATE TABLE IF NOT EXISTS " & table & " (" & " pubsubTopic BLOB NOT NULL," & + " contentTopic BLOB NOT NULL," & " payload BLOB," & " version INTEGER NOT NULL," & + " timestamp INTEGER NOT NULL," & " id BLOB," & " messageHash BLOB," & + " storedAt INTEGER NOT NULL," & " meta BLOB," & + " CONSTRAINT messageIndex PRIMARY KEY (messageHash)" & ") WITHOUT ROWID;" + +proc createTable*(db: SqliteDatabase): DatabaseResult[void] = + let query = createTableQuery(DbTable) + discard + ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard + , + ) + return ok() + +## Create indices + +proc createOldestMessageTimestampIndexQuery(table: string): SqlQueryStr = + "CREATE INDEX IF NOT EXISTS i_ts ON " & table & " (storedAt);" + +proc createOldestMessageTimestampIndex*(db: SqliteDatabase): DatabaseResult[void] = + let query = createOldestMessageTimestampIndexQuery(DbTable) + discard + ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard + , + ) + return ok() + +proc createHistoryQueryIndexQuery(table: string): SqlQueryStr = + "CREATE INDEX IF NOT EXISTS i_query ON " & table & + " (contentTopic, pubsubTopic, storedAt, id);" + +proc createHistoryQueryIndex*(db: SqliteDatabase): DatabaseResult[void] = + let query = createHistoryQueryIndexQuery(DbTable) + discard + ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard + , + ) + return ok() + +## Insert message +type InsertMessageParams* = ( + seq[byte], + seq[byte], + Timestamp, + seq[byte], + seq[byte], + seq[byte], + int64, + Timestamp, + seq[byte], +) + +proc insertMessageQuery(table: string): SqlQueryStr = + return + "INSERT INTO " & table & + "(id, messageHash, storedAt, contentTopic, payload, pubsubTopic, version, timestamp, meta)" & + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);" + +proc prepareInsertMessageStmt*( + db: SqliteDatabase +): SqliteStmt[InsertMessageParams, void] = + let query = insertMessageQuery(DbTable) + return + db.prepareStmt(query, InsertMessageParams, void).expect("this is a valid statement") + +## Count table messages + +proc countMessagesQuery(table: string): SqlQueryStr = + return "SELECT COUNT(*) FROM " & table + +proc getMessageCount*(db: SqliteDatabase): DatabaseResult[int64] = + var count: int64 + proc queryRowCallback(s: ptr sqlite3_stmt) = + count = sqlite3_column_int64(s, 0) + + let query = countMessagesQuery(DbTable) + let res = db.query(query, queryRowCallback) + if res.isErr(): + return err("failed to count number of messages in the database") + + return ok(count) + +## Get oldest message receiver timestamp + +proc selectOldestMessageTimestampQuery(table: string): SqlQueryStr = + return "SELECT MIN(storedAt) FROM " & table + +proc selectOldestReceiverTimestamp*( + db: SqliteDatabase +): DatabaseResult[Timestamp] {.inline.} = + var timestamp: Timestamp + proc queryRowCallback(s: ptr sqlite3_stmt) = + timestamp = queryRowReceiverTimestampCallback(s, 0) + + let query = selectOldestMessageTimestampQuery(DbTable) + let res = db.query(query, queryRowCallback) + if res.isErr(): + return err("failed to get the oldest receiver timestamp from the database") + + return ok(timestamp) + +## Get newest message receiver timestamp + +proc selectNewestMessageTimestampQuery(table: string): SqlQueryStr = + return "SELECT MAX(storedAt) FROM " & table + +proc selectNewestReceiverTimestamp*( + db: SqliteDatabase +): DatabaseResult[Timestamp] {.inline.} = + var timestamp: Timestamp + proc queryRowCallback(s: ptr sqlite3_stmt) = + timestamp = queryRowReceiverTimestampCallback(s, 0) + + let query = selectNewestMessageTimestampQuery(DbTable) + let res = db.query(query, queryRowCallback) + if res.isErr(): + return err("failed to get the newest receiver timestamp from the database") + + return ok(timestamp) + +## Delete messages older than timestamp + +proc deleteMessagesOlderThanTimestampQuery(table: string, ts: Timestamp): SqlQueryStr = + return "DELETE FROM " & table & " WHERE storedAt < " & $ts + +proc deleteMessagesOlderThanTimestamp*( + db: SqliteDatabase, ts: int64 +): DatabaseResult[void] = + let query = deleteMessagesOlderThanTimestampQuery(DbTable, ts) + discard + ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard + , + ) + return ok() + +## Delete oldest messages not within limit + +proc deleteOldestMessagesNotWithinLimitQuery(table: string, limit: int): SqlQueryStr = + return + "DELETE FROM " & table & " WHERE (storedAt, id, pubsubTopic) NOT IN (" & + " SELECT storedAt, id, pubsubTopic FROM " & table & + " ORDER BY storedAt DESC, id DESC" & " LIMIT " & $limit & ");" + +proc deleteOldestMessagesNotWithinLimit*( + db: SqliteDatabase, limit: int +): DatabaseResult[void] = + # NOTE: The word `limit` here refers the store capacity/maximum number-of-messages allowed limit + let query = deleteOldestMessagesNotWithinLimitQuery(DbTable, limit = limit) + discard + ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard + , + ) + return ok() + +## Select all messages + +proc selectAllMessagesQuery(table: string): SqlQueryStr = + return + "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" & + " FROM " & table & " ORDER BY storedAt ASC" + +proc selectAllMessages*( + db: SqliteDatabase +): DatabaseResult[ + seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] +] {.gcsafe.} = + ## Retrieve all messages from the store. + var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] + proc queryRowCallback(s: ptr sqlite3_stmt) = + let + pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) + wakuMessage = queryRowWakuMessageCallback( + s, + contentTopicCol = 1, + payloadCol = 2, + versionCol = 4, + senderTimestampCol = 5, + metaCol = 8, + ) + digest = queryRowDigestCallback(s, digestCol = 6) + storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) + hash = queryRowWakuMessageHashCallback(s, hashCol = 7) + + rows.add((pubsubTopic, wakuMessage, digest, storedAt, hash)) + + let query = selectAllMessagesQuery(DbTable) + let res = db.query(query, queryRowCallback) + if res.isErr(): + return err(res.error()) + + return ok(rows) + +## Select messages by history query with limit + +proc combineClauses(clauses: varargs[Option[string]]): Option[string] = + let whereSeq = @clauses.filterIt(it.isSome()).mapIt(it.get()) + if whereSeq.len <= 0: + return none(string) + + var where: string = whereSeq[0] + for clause in whereSeq[1 ..^ 1]: + where &= " AND " & clause + return some(where) + +proc whereClausev2( + cursor: bool, + pubsubTopic: Option[PubsubTopic], + contentTopic: seq[ContentTopic], + startTime: Option[Timestamp], + endTime: Option[Timestamp], + ascending: bool, +): Option[string] {.deprecated.} = + let cursorClause = + if cursor: + let comp = if ascending: ">" else: "<" + + some("(storedAt, id) " & comp & " (?, ?)") + else: + none(string) + + let pubsubTopicClause = + if pubsubTopic.isNone(): + none(string) + else: + some("pubsubTopic = (?)") + + let contentTopicClause = + if contentTopic.len <= 0: + none(string) + else: + var where = "contentTopic IN (" + where &= "?" + for _ in 1 ..< contentTopic.len: + where &= ", ?" + where &= ")" + some(where) + + let startTimeClause = + if startTime.isNone(): + none(string) + else: + some("storedAt >= (?)") + + let endTimeClause = + if endTime.isNone(): + none(string) + else: + some("storedAt <= (?)") + + return combineClauses( + cursorClause, pubsubTopicClause, contentTopicClause, startTimeClause, endTimeClause + ) + +proc selectMessagesWithLimitQueryv2( + table: string, where: Option[string], limit: uint, ascending = true, v3 = false +): SqlQueryStr {.deprecated.} = + let order = if ascending: "ASC" else: "DESC" + + var query: string + + query = + "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" + query &= " FROM " & table + + if where.isSome(): + query &= " WHERE " & where.get() + + query &= " ORDER BY storedAt " & order & ", id " & order + + query &= " LIMIT " & $limit & ";" + + return query + +proc prepareStmt( + db: SqliteDatabase, stmt: string +): DatabaseResult[SqliteStmt[void, void]] = + var s: RawStmtPtr + checkErr sqlite3_prepare_v2(db.env, stmt, stmt.len.cint, addr s, nil) + return ok(SqliteStmt[void, void](s)) + +proc execSelectMessagesV2WithLimitStmt( + s: SqliteStmt, + cursor: Option[DbCursor], + pubsubTopic: Option[PubsubTopic], + contentTopic: seq[ContentTopic], + startTime: Option[Timestamp], + endTime: Option[Timestamp], + onRowCallback: DataProc, +): DatabaseResult[void] {.deprecated.} = + let s = RawStmtPtr(s) + + # Bind params + var paramIndex = 1 + + if cursor.isSome(): + let (storedAt, id, _) = cursor.get() + checkErr bindParam(s, paramIndex, storedAt) + paramIndex += 1 + checkErr bindParam(s, paramIndex, id) + paramIndex += 1 + + if pubsubTopic.isSome(): + let pubsubTopic = toBytes(pubsubTopic.get()) + checkErr bindParam(s, paramIndex, pubsubTopic) + paramIndex += 1 + + for topic in contentTopic: + checkErr bindParam(s, paramIndex, topic.toBytes()) + paramIndex += 1 + + if startTime.isSome(): + let time = startTime.get() + checkErr bindParam(s, paramIndex, time) + paramIndex += 1 + + if endTime.isSome(): + let time = endTime.get() + checkErr bindParam(s, paramIndex, time) + paramIndex += 1 + + try: + while true: + let v = sqlite3_step(s) + case v + of SQLITE_ROW: + onRowCallback(s) + of SQLITE_DONE: + return ok() + else: + return err($sqlite3_errstr(v)) + except Exception, CatchableError: + # release implicit transaction + discard sqlite3_reset(s) # same return information as step + discard sqlite3_clear_bindings(s) # no errors possible + +proc selectMessagesByHistoryQueryWithLimit*( + db: SqliteDatabase, + contentTopic: seq[ContentTopic], + pubsubTopic: Option[PubsubTopic], + cursor: Option[DbCursor], + startTime: Option[Timestamp], + endTime: Option[Timestamp], + limit: uint, + ascending: bool, +): DatabaseResult[ + seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] +] {.deprecated.} = + var messages: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] = + @[] + + proc queryRowCallback(s: ptr sqlite3_stmt) = + let + pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) + message = queryRowWakuMessageCallback( + s, + contentTopicCol = 1, + payloadCol = 2, + versionCol = 4, + senderTimestampCol = 5, + metaCol = 8, + ) + digest = queryRowDigestCallback(s, digestCol = 6) + storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) + hash = queryRowWakuMessageHashCallback(s, hashCol = 7) + + messages.add((pubsubTopic, message, digest, storedAt, hash)) + + let query = block: + let where = whereClausev2( + cursor.isSome(), pubsubTopic, contentTopic, startTime, endTime, ascending + ) + + selectMessagesWithLimitQueryv2(DbTable, where, limit, ascending) + + let dbStmt = ?db.prepareStmt(query) + ?dbStmt.execSelectMessagesV2WithLimitStmt( + cursor, pubsubTopic, contentTopic, startTime, endTime, queryRowCallback + ) + dbStmt.dispose() + + return ok(messages) + +### Store v3 ### + +proc execSelectMessageByHash( + s: SqliteStmt, hash: WakuMessageHash, onRowCallback: DataProc +): DatabaseResult[void] = + let s = RawStmtPtr(s) + + checkErr bindParam(s, 1, toSeq(hash)) + + try: + while true: + let v = sqlite3_step(s) + case v + of SQLITE_ROW: + onRowCallback(s) + of SQLITE_DONE: + return ok() + else: + return err($sqlite3_errstr(v)) + except Exception, CatchableError: + # release implicit transaction + discard sqlite3_reset(s) # same return information as step + discard sqlite3_clear_bindings(s) # no errors possible + +proc selectMessageByHashQuery(): SqlQueryStr = + var query: string + + query = "SELECT contentTopic, payload, version, timestamp, meta, messageHash" + query &= " FROM " & DbTable + query &= " WHERE messageHash = (?)" + + return query + +proc whereClause( + cursor: bool, + pubsubTopic: Option[PubsubTopic], + contentTopic: seq[ContentTopic], + startTime: Option[Timestamp], + endTime: Option[Timestamp], + hashes: seq[WakuMessageHash], + ascending: bool, +): Option[string] = + let cursorClause = + if cursor: + let comp = if ascending: ">" else: "<" + + some("(timestamp, messageHash) " & comp & " (?, ?)") + else: + none(string) + + let pubsubTopicClause = + if pubsubTopic.isNone(): + none(string) + else: + some("pubsubTopic = (?)") + + let contentTopicClause = + if contentTopic.len <= 0: + none(string) + else: + var where = "contentTopic IN (" + where &= "?" + for _ in 1 ..< contentTopic.len: + where &= ", ?" + where &= ")" + some(where) + + let startTimeClause = + if startTime.isNone(): + none(string) + else: + some("storedAt >= (?)") + + let endTimeClause = + if endTime.isNone(): + none(string) + else: + some("storedAt <= (?)") + + let hashesClause = + if hashes.len <= 0: + none(string) + else: + var where = "messageHash IN (" + where &= "?" + for _ in 1 ..< hashes.len: + where &= ", ?" + where &= ")" + some(where) + + return combineClauses( + cursorClause, pubsubTopicClause, contentTopicClause, startTimeClause, endTimeClause, + hashesClause, + ) + +proc execSelectMessagesWithLimitStmt( + s: SqliteStmt, + cursor: Option[(Timestamp, WakuMessageHash)], + pubsubTopic: Option[PubsubTopic], + contentTopic: seq[ContentTopic], + startTime: Option[Timestamp], + endTime: Option[Timestamp], + hashes: seq[WakuMessageHash], + onRowCallback: DataProc, +): DatabaseResult[void] = + let s = RawStmtPtr(s) + + # Bind params + var paramIndex = 1 + + if cursor.isSome(): + let (time, hash) = cursor.get() + checkErr bindParam(s, paramIndex, time) + paramIndex += 1 + checkErr bindParam(s, paramIndex, toSeq(hash)) + paramIndex += 1 + + if pubsubTopic.isSome(): + let pubsubTopic = toBytes(pubsubTopic.get()) + checkErr bindParam(s, paramIndex, pubsubTopic) + paramIndex += 1 + + for topic in contentTopic: + checkErr bindParam(s, paramIndex, topic.toBytes()) + paramIndex += 1 + + for hash in hashes: + checkErr bindParam(s, paramIndex, toSeq(hash)) + paramIndex += 1 + + if startTime.isSome(): + let time = startTime.get() + checkErr bindParam(s, paramIndex, time) + paramIndex += 1 + + if endTime.isSome(): + let time = endTime.get() + checkErr bindParam(s, paramIndex, time) + paramIndex += 1 + + try: + while true: + let v = sqlite3_step(s) + case v + of SQLITE_ROW: + onRowCallback(s) + of SQLITE_DONE: + return ok() + else: + return err($sqlite3_errstr(v)) + except Exception, CatchableError: + # release implicit transaction + discard sqlite3_reset(s) # same return information as step + discard sqlite3_clear_bindings(s) # no errors possible + +proc selectMessagesWithLimitQuery( + table: string, where: Option[string], limit: uint, ascending = true, v3 = false +): SqlQueryStr = + let order = if ascending: "ASC" else: "DESC" + + var query: string + + query = + "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" + query &= " FROM " & table + + if where.isSome(): + query &= " WHERE " & where.get() + + query &= " ORDER BY storedAt " & order & ", messageHash " & order + + query &= " LIMIT " & $limit & ";" + + return query + +proc selectMessagesByStoreQueryWithLimit*( + db: SqliteDatabase, + contentTopic: seq[ContentTopic], + pubsubTopic: Option[PubsubTopic], + cursor: Option[WakuMessageHash], + startTime: Option[Timestamp], + endTime: Option[Timestamp], + hashes: seq[WakuMessageHash], + limit: uint, + ascending: bool, +): DatabaseResult[ + seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] +] = + # Must first get the message timestamp before paginating by time + let newCursor = + if cursor.isSome() and cursor.get() != EmptyWakuMessageHash: + let hash: WakuMessageHash = cursor.get() + + var wakuMessage: Option[WakuMessage] + + proc queryRowCallback(s: ptr sqlite3_stmt) = + wakuMessage = some( + queryRowWakuMessageCallback( + s, + contentTopicCol = 0, + payloadCol = 1, + versionCol = 2, + senderTimestampCol = 3, + metaCol = 4, + ) + ) + + let query = selectMessageByHashQuery() + let dbStmt = ?db.prepareStmt(query) + ?dbStmt.execSelectMessageByHash(hash, queryRowCallback) + dbStmt.dispose() + + if wakuMessage.isSome(): + let time = wakuMessage.get().timestamp + + some((time, hash)) + else: + return err("cursor not found") + else: + none((Timestamp, WakuMessageHash)) + + var messages: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] = + @[] + + proc queryRowCallback(s: ptr sqlite3_stmt) = + let + pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) + message = queryRowWakuMessageCallback( + s, + contentTopicCol = 1, + payloadCol = 2, + versionCol = 4, + senderTimestampCol = 5, + metaCol = 8, + ) + digest = queryRowDigestCallback(s, digestCol = 6) + storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) + hash = queryRowWakuMessageHashCallback(s, hashCol = 7) + + messages.add((pubsubTopic, message, digest, storedAt, hash)) + + let query = block: + let where = whereClause( + newCursor.isSome(), + pubsubTopic, + contentTopic, + startTime, + endTime, + hashes, + ascending, + ) + + selectMessagesWithLimitQuery(DbTable, where, limit, ascending, true) + + let dbStmt = ?db.prepareStmt(query) + ?dbStmt.execSelectMessagesWithLimitStmt( + newCursor, pubsubTopic, contentTopic, startTime, endTime, hashes, queryRowCallback + ) + dbStmt.dispose() + + return ok(messages) diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim b/waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim new file mode 100644 index 000000000..4e0450aab --- /dev/null +++ b/waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim @@ -0,0 +1,225 @@ +# The code in this file is an adaptation of the Sqlite KV Store found in nim-eth. +# https://github.com/status-im/nim-eth/blob/master/eth/db/kvstore_sqlite3.nim +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/options, stew/byteutils, chronicles, chronos, results +import + ../../../common/databases/db_sqlite, + ../../../waku_core, + ../../../waku_core/message/digest, + ../../common, + ../../driver, + ./cursor, + ./queries + +logScope: + topics = "waku archive sqlite" + +proc init(db: SqliteDatabase): ArchiveDriverResult[void] = + ## Misconfiguration can lead to nil DB + if db.isNil(): + return err("db not initialized") + + # Create table, if doesn't exist + let resCreate = createTable(db) + if resCreate.isErr(): + return err("failed to create table: " & resCreate.error()) + + # Create indices, if don't exist + let resRtIndex = createOldestMessageTimestampIndex(db) + if resRtIndex.isErr(): + return err("failed to create i_rt index: " & resRtIndex.error()) + + let resMsgIndex = createHistoryQueryIndex(db) + if resMsgIndex.isErr(): + return err("failed to create i_query index: " & resMsgIndex.error()) + + return ok() + +type SqliteDriver* = ref object of ArchiveDriver + db: SqliteDatabase + insertStmt: SqliteStmt[InsertMessageParams, void] + +proc new*(T: type SqliteDriver, db: SqliteDatabase): ArchiveDriverResult[T] = + # Database initialization + let resInit = init(db) + if resInit.isErr(): + return err(resInit.error()) + + # General initialization + let insertStmt = db.prepareInsertMessageStmt() + return ok(SqliteDriver(db: db, insertStmt: insertStmt)) + +method put*( + s: SqliteDriver, + pubsubTopic: PubsubTopic, + message: WakuMessage, + digest: MessageDigest, + messageHash: WakuMessageHash, + receivedTime: Timestamp, +): Future[ArchiveDriverResult[void]] {.async.} = + ## Inserts a message into the store + let res = s.insertStmt.exec( + ( + @(digest.data), # id + @(messageHash), # messageHash + receivedTime, # storedAt + toBytes(message.contentTopic), # contentTopic + message.payload, # payload + toBytes(pubsubTopic), # pubsubTopic + int64(message.version), # version + message.timestamp, # senderTimestamp + message.meta, # meta + ) + ) + + return res + +method getAllMessages*( + s: SqliteDriver +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + ## Retrieve all messages from the store. + return s.db.selectAllMessages() + +method getMessagesV2*( + s: SqliteDriver, + contentTopic = newSeq[ContentTopic](0), + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = + echo "here" + + let cursor = cursor.map(toDbCursor) + + let rowsRes = s.db.selectMessagesByHistoryQueryWithLimit( + contentTopic, + pubsubTopic, + cursor, + startTime, + endTime, + limit = maxPageSize, + ascending = ascendingOrder, + ) + + return rowsRes + +method getMessages*( + s: SqliteDriver, + includeData = true, + contentTopic = newSeq[ContentTopic](0), + pubsubTopic = none(PubsubTopic), + cursor = none(ArchiveCursor), + startTime = none(Timestamp), + endTime = none(Timestamp), + hashes = newSeq[WakuMessageHash](0), + maxPageSize = DefaultPageSize, + ascendingOrder = true, +): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = + let cursor = + if cursor.isSome(): + some(cursor.get().hash) + else: + none(WakuMessageHash) + + let rowsRes = s.db.selectMessagesByStoreQueryWithLimit( + contentTopic, + pubsubTopic, + cursor, + startTime, + endTime, + hashes, + limit = maxPageSize, + ascending = ascendingOrder, + ) + + return rowsRes + +method getMessagesCount*( + s: SqliteDriver +): Future[ArchiveDriverResult[int64]] {.async.} = + return s.db.getMessageCount() + +method getPagesCount*(s: SqliteDriver): Future[ArchiveDriverResult[int64]] {.async.} = + return s.db.getPageCount() + +method getPagesSize*(s: SqliteDriver): Future[ArchiveDriverResult[int64]] {.async.} = + return s.db.getPageSize() + +method getDatabaseSize*(s: SqliteDriver): Future[ArchiveDriverResult[int64]] {.async.} = + return s.db.getDatabaseSize() + +method performVacuum*(s: SqliteDriver): Future[ArchiveDriverResult[void]] {.async.} = + return s.db.performSqliteVacuum() + +method getOldestMessageTimestamp*( + s: SqliteDriver +): Future[ArchiveDriverResult[Timestamp]] {.async.} = + return s.db.selectOldestReceiverTimestamp() + +method getNewestMessageTimestamp*( + s: SqliteDriver +): Future[ArchiveDriverResult[Timestamp]] {.async.} = + return s.db.selectnewestReceiverTimestamp() + +method deleteMessagesOlderThanTimestamp*( + s: SqliteDriver, ts: Timestamp +): Future[ArchiveDriverResult[void]] {.async.} = + return s.db.deleteMessagesOlderThanTimestamp(ts) + +method deleteOldestMessagesNotWithinLimit*( + s: SqliteDriver, limit: int +): Future[ArchiveDriverResult[void]] {.async.} = + return s.db.deleteOldestMessagesNotWithinLimit(limit) + +method decreaseDatabaseSize*( + driver: SqliteDriver, targetSizeInBytes: int64, forceRemoval: bool = false +): Future[ArchiveDriverResult[void]] {.async.} = + ## To remove 20% of the outdated data from database + const DeleteLimit = 0.80 + + ## when db size overshoots the database limit, shread 20% of outdated messages + ## get size of database + let dbSize = (await driver.getDatabaseSize()).valueOr: + return err("failed to get database size: " & $error) + + ## database size in bytes + let totalSizeOfDB: int64 = int64(dbSize) + + if totalSizeOfDB < targetSizeInBytes: + return ok() + + ## to shread/delete messsges, get the total row/message count + let numMessages = (await driver.getMessagesCount()).valueOr: + return err("failed to get messages count: " & error) + + ## NOTE: Using SQLite vacuuming is done manually, we delete a percentage of rows + ## if vacumming is done automatically then we aim to check DB size periodially for efficient + ## retention policy implementation. + + ## 80% of the total messages are to be kept, delete others + let pageDeleteWindow = int(float(numMessages) * DeleteLimit) + + (await driver.deleteOldestMessagesNotWithinLimit(limit = pageDeleteWindow)).isOkOr: + return err("deleting oldest messages failed: " & error) + + return ok() + +method close*(s: SqliteDriver): Future[ArchiveDriverResult[void]] {.async.} = + ## Close the database connection + # Dispose statements + s.insertStmt.dispose() + # Close connection + s.db.close() + return ok() + +method existsTable*( + s: SqliteDriver, tableName: string +): Future[ArchiveDriverResult[bool]] {.async.} = + return err("existsTable method not implemented in sqlite_driver") diff --git a/waku/waku_archive_legacy/retention_policy.nim b/waku/waku_archive_legacy/retention_policy.nim new file mode 100644 index 000000000..26916d0dd --- /dev/null +++ b/waku/waku_archive_legacy/retention_policy.nim @@ -0,0 +1,16 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import results, chronos +import ./driver + +type RetentionPolicyResult*[T] = Result[T, string] + +type RetentionPolicy* = ref object of RootObj + +method execute*( + p: RetentionPolicy, store: ArchiveDriver +): Future[RetentionPolicyResult[void]] {.base, async.} = + discard diff --git a/waku/waku_archive_legacy/retention_policy/builder.nim b/waku/waku_archive_legacy/retention_policy/builder.nim new file mode 100644 index 000000000..b7469220f --- /dev/null +++ b/waku/waku_archive_legacy/retention_policy/builder.nim @@ -0,0 +1,88 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/[strutils, options], regex, results +import + ../retention_policy, + ./retention_policy_time, + ./retention_policy_capacity, + ./retention_policy_size + +proc new*( + T: type RetentionPolicy, retPolicy: string +): RetentionPolicyResult[Option[RetentionPolicy]] = + let retPolicy = retPolicy.toLower + + # Validate the retention policy format + if retPolicy == "" or retPolicy == "none": + return ok(none(RetentionPolicy)) + + const StoreMessageRetentionPolicyRegex = re2"^\w+:\d*\.?\d+((g|m)b)?$" + if not retPolicy.match(StoreMessageRetentionPolicyRegex): + return err("invalid 'store message retention policy' format: " & retPolicy) + + # Apply the retention policy, if any + let rententionPolicyParts = retPolicy.split(":", 1) + let + policy = rententionPolicyParts[0] + policyArgs = rententionPolicyParts[1] + + if policy == "time": + var retentionTimeSeconds: int64 + try: + retentionTimeSeconds = parseInt(policyArgs) + except ValueError: + return err("invalid time retention policy argument") + + let retPolicy: RetentionPolicy = TimeRetentionPolicy.new(retentionTimeSeconds) + return ok(some(retPolicy)) + elif policy == "capacity": + var retentionCapacity: int + try: + retentionCapacity = parseInt(policyArgs) + except ValueError: + return err("invalid capacity retention policy argument") + + let retPolicy: RetentionPolicy = CapacityRetentionPolicy.new(retentionCapacity) + return ok(some(retPolicy)) + elif policy == "size": + var retentionSize: string + retentionSize = policyArgs + + # captures the size unit such as GB or MB + let sizeUnit = retentionSize.substr(retentionSize.len - 2) + # captures the string type number data of the size provided + let sizeQuantityStr = retentionSize.substr(0, retentionSize.len - 3) + # to hold the numeric value data of size + var inptSizeQuantity: float + var sizeQuantity: int64 + var sizeMultiplier: float + + try: + inptSizeQuantity = parseFloat(sizeQuantityStr) + except ValueError: + return err("invalid size retention policy argument: " & getCurrentExceptionMsg()) + + case sizeUnit + of "gb": + sizeMultiplier = 1024.0 * 1024.0 * 1024.0 + of "mb": + sizeMultiplier = 1024.0 * 1024.0 + else: + return err ( + """invalid size retention value unit: expected "Mb" or "Gb" but got """ & + sizeUnit + ) + + # quantity is converted into bytes for uniform processing + sizeQuantity = int64(inptSizeQuantity * sizeMultiplier) + + if sizeQuantity <= 0: + return err("invalid size retention policy argument: a non-zero value is required") + + let retPolicy: RetentionPolicy = SizeRetentionPolicy.new(sizeQuantity) + return ok(some(retPolicy)) + else: + return err("unknown retention policy") diff --git a/waku/waku_archive_legacy/retention_policy/retention_policy_capacity.nim b/waku/waku_archive_legacy/retention_policy/retention_policy_capacity.nim new file mode 100644 index 000000000..e679e9f16 --- /dev/null +++ b/waku/waku_archive_legacy/retention_policy/retention_policy_capacity.nim @@ -0,0 +1,68 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import results, chronicles, chronos +import ../driver, ../retention_policy + +logScope: + topics = "waku archive retention_policy" + +const DefaultCapacity*: int = 25_000 + +const MaxOverflow = 1.3 + +type + # CapacityRetentionPolicy implements auto deletion as follows: + # - The sqlite DB will driver up to `totalCapacity = capacity` * `MaxOverflow` messages, + # giving an overflowWindow of `capacity * (MaxOverflow - 1) = overflowWindow`. + # + # - In case of an overflow, messages are sorted by `receiverTimestamp` and the oldest ones are + # deleted. The number of messages that get deleted is `(overflowWindow / 2) = deleteWindow`, + # bringing the total number of driverd messages back to `capacity + (overflowWindow / 2)`. + # + # The rationale for batch deleting is efficiency. We keep half of the overflow window in addition + # to `capacity` because we delete the oldest messages with respect to `receiverTimestamp` instead of + # `senderTimestamp`. `ReceiverTimestamp` is guaranteed to be set, while senders could omit setting + # `senderTimestamp`. However, `receiverTimestamp` can differ from node to node for the same message. + # So sorting by `receiverTimestamp` might (slightly) prioritize some actually older messages and we + # compensate that by keeping half of the overflow window. + CapacityRetentionPolicy* = ref object of RetentionPolicy + capacity: int + # represents both the number of messages that are persisted in the sqlite DB (excl. the overflow window explained above), and the number of messages that get loaded via `getAll`. + totalCapacity: int # = capacity * MaxOverflow + deleteWindow: int + # = capacity * (MaxOverflow - 1) / 2; half of the overflow window, the amount of messages deleted when overflow occurs + +proc calculateTotalCapacity(capacity: int, overflow: float): int = + int(float(capacity) * overflow) + +proc calculateOverflowWindow(capacity: int, overflow: float): int = + int(float(capacity) * (overflow - 1)) + +proc calculateDeleteWindow(capacity: int, overflow: float): int = + calculateOverflowWindow(capacity, overflow) div 2 + +proc new*(T: type CapacityRetentionPolicy, capacity = DefaultCapacity): T = + let + totalCapacity = calculateTotalCapacity(capacity, MaxOverflow) + deleteWindow = calculateDeleteWindow(capacity, MaxOverflow) + + CapacityRetentionPolicy( + capacity: capacity, totalCapacity: totalCapacity, deleteWindow: deleteWindow + ) + +method execute*( + p: CapacityRetentionPolicy, driver: ArchiveDriver +): Future[RetentionPolicyResult[void]] {.async.} = + let numMessages = (await driver.getMessagesCount()).valueOr: + return err("failed to get messages count: " & error) + + if numMessages < p.totalCapacity: + return ok() + + (await driver.deleteOldestMessagesNotWithinLimit(limit = p.capacity + p.deleteWindow)).isOkOr: + return err("deleting oldest messages failed: " & error) + + return ok() diff --git a/waku/waku_archive_legacy/retention_policy/retention_policy_size.nim b/waku/waku_archive_legacy/retention_policy/retention_policy_size.nim new file mode 100644 index 000000000..9f710f028 --- /dev/null +++ b/waku/waku_archive_legacy/retention_policy/retention_policy_size.nim @@ -0,0 +1,27 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import results, chronicles, chronos +import ../driver, ../retention_policy + +logScope: + topics = "waku archive retention_policy" + +# default size is 30 GiB or 32212254720.0 in bytes +const DefaultRetentionSize*: int64 = 32212254720 + +type SizeRetentionPolicy* = ref object of RetentionPolicy + sizeLimit: int64 + +proc new*(T: type SizeRetentionPolicy, size = DefaultRetentionSize): T = + SizeRetentionPolicy(sizeLimit: size) + +method execute*( + p: SizeRetentionPolicy, driver: ArchiveDriver +): Future[RetentionPolicyResult[void]] {.async.} = + (await driver.decreaseDatabaseSize(p.sizeLimit)).isOkOr: + return err("decreaseDatabaseSize failed: " & $error) + + return ok() diff --git a/waku/waku_archive_legacy/retention_policy/retention_policy_time.nim b/waku/waku_archive_legacy/retention_policy/retention_policy_time.nim new file mode 100644 index 000000000..b5f096e64 --- /dev/null +++ b/waku/waku_archive_legacy/retention_policy/retention_policy_time.nim @@ -0,0 +1,40 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + +import std/times, results, chronicles, chronos +import ../../waku_core, ../driver, ../retention_policy + +logScope: + topics = "waku archive retention_policy" + +const DefaultRetentionTime*: int64 = 30.days.seconds + +type TimeRetentionPolicy* = ref object of RetentionPolicy + retentionTime: chronos.Duration + +proc new*(T: type TimeRetentionPolicy, retentionTime = DefaultRetentionTime): T = + TimeRetentionPolicy(retentionTime: retentionTime.seconds) + +method execute*( + p: TimeRetentionPolicy, driver: ArchiveDriver +): Future[RetentionPolicyResult[void]] {.async.} = + ## Delete messages that exceed the retention time by 10% and more (batch delete for efficiency) + + let omtRes = await driver.getOldestMessageTimestamp() + if omtRes.isErr(): + return err("failed to get oldest message timestamp: " & omtRes.error) + + let now = getNanosecondTime(getTime().toUnixFloat()) + let retentionTimestamp = now - p.retentionTime.nanoseconds + let thresholdTimestamp = retentionTimestamp - p.retentionTime.nanoseconds div 10 + + if thresholdTimestamp <= omtRes.value: + return ok() + + let res = await driver.deleteMessagesOlderThanTimestamp(ts = retentionTimestamp) + if res.isErr(): + return err("failed to delete oldest messages: " & res.error) + + return ok()