From fdf4e839ffcd40542b4def2207290f405948131c Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Fri, 13 Mar 2026 23:22:53 +0100 Subject: [PATCH 01/19] use fix branch from nim-http-utils --- vendor/nim-http-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils index 79cbab146..08db60946 160000 --- a/vendor/nim-http-utils +++ b/vendor/nim-http-utils @@ -1 +1 @@ -Subproject commit 79cbab1460f4c0cdde2084589d017c43a3d7b4f1 +Subproject commit 08db609467a0e2b5a6e8ce118bb83dba7a8a9375 From 614f17162638eb178b1b119c86c6c7fa040d018c Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Tue, 17 Mar 2026 10:15:35 -0300 Subject: [PATCH 02/19] nim nph 0.7.0 formatting (#3759) --- apps/chat2/chat2.nim | 23 +- apps/chat2bridge/chat2bridge.nim | 14 +- apps/chat2mix/chat2mix.nim | 23 +- apps/chat2mix/config_chat2mix.nim | 21 +- apps/wakucanary/wakucanary.nim | 9 +- .../logos_delivery_api/debug_api.nim | 4 +- .../json_connection_status_change_event.nim | 10 +- library/libwaku.nim | 2 +- .../content_script_version_1.nim | 3 +- .../content_script_version_2.nim | 3 +- .../content_script_version_3.nim | 3 +- .../content_script_version_4.nim | 3 +- .../content_script_version_5.nim | 3 +- .../content_script_version_6.nim | 3 +- .../content_script_version_7.nim | 3 +- .../pg_migration_manager.nim | 19 +- tests/api/test_api_send.nim | 2 +- tests/api/test_entry_nodes.nim | 64 +- tests/common/test_base64_codec.nim | 31 +- tests/common/test_request_broker.nim | 378 ++--- tests/common/test_sqlite_migrations.nim | 33 +- .../peer_store/test_waku_peer_storage.nim | 77 +- tests/node/test_wakunode_health_monitor.nim | 68 +- tests/node/test_wakunode_legacy_store.nim | 112 +- tests/node/test_wakunode_relay_rln.nim | 48 +- tests/node/test_wakunode_sharding.nim | 44 +- tests/node/test_wakunode_store.nim | 138 +- tests/resources/payloads.nim | 6 +- tests/test_peer_manager.nim | 60 +- tests/test_waku.nim | 23 +- tests/test_waku_keystore_keyfile.nim | 43 +- tests/test_waku_noise.nim | 9 +- tests/test_waku_rendezvous.nim | 9 +- tests/test_waku_switch.nim | 16 +- .../test_driver_postgres_query.nim | 1273 ++++++++--------- .../waku_archive/test_driver_queue_query.nim | 1145 +++++++-------- .../waku_archive/test_driver_sqlite_query.nim | 1180 +++++++-------- tests/waku_archive/test_retention_policy.nim | 19 +- tests/waku_archive/test_waku_archive.nim | 105 +- .../test_driver_postgres_query.nim | 1273 ++++++++--------- .../test_driver_queue_query.nim | 1135 +++++++-------- .../test_driver_sqlite_query.nim | 1180 +++++++-------- .../waku_archive_legacy/test_waku_archive.nim | 105 +- tests/waku_enr/test_sharding.nim | 15 +- tests/waku_rln_relay/test_waku_rln_relay.nim | 17 +- .../test_wakunode_rln_relay.nim | 15 +- tests/waku_rln_relay/utils_onchain.nim | 31 +- tests/waku_store/test_client.nim | 35 +- tests/waku_store/test_resume.nim | 25 +- tests/waku_store/test_wakunode_store.nim | 25 +- tests/waku_store_legacy/test_resume.nim | 178 ++- tests/waku_store_legacy/test_rpc_codec.nim | 9 +- .../waku_store_legacy/test_wakunode_store.nim | 25 +- tests/waku_store_sync/test_range_split.nim | 42 +- .../waku_store_sync/test_state_transition.nim | 40 +- tests/wakunode_rest/test_rest_filter.nim | 11 +- tests/wakunode_rest/test_rest_relay.nim | 48 +- tests/wakunode_rest/test_rest_store.nim | 151 +- tools/confutils/cli_args.nim | 3 +- vendor/nph | 2 +- waku/api/api_conf.nim | 7 +- waku/common/broker/event_broker.nim | 15 +- waku/common/databases/db_postgres/dbconn.nim | 9 +- waku/common/rate_limit/timed_map.nim | 12 +- waku/factory/builder.nim | 27 +- waku/factory/networks_config.nim | 28 +- waku/factory/node_factory.nim | 9 +- waku/node/peer_manager/peer_manager.nim | 8 +- waku/node/waku_node.nim | 11 +- waku/rest_api/endpoint/builder.nim | 13 +- waku/rest_api/endpoint/server.nim | 33 +- .../postgres_driver/postgres_driver.nim | 36 +- .../driver/sqlite_driver/queries.nim | 44 +- .../driver/sqlite_driver/queries.nim | 55 +- waku/waku_core/peers.nim | 36 +- waku/waku_core/topics/pubsub_topic.nim | 18 +- waku/waku_enr/sharding.nim | 7 +- waku/waku_keystore/protocol_types.nim | 7 +- .../waku_noise/noise_handshake_processing.nim | 17 +- waku/waku_noise/noise_types.nim | 64 +- .../group_manager/on_chain/group_manager.nim | 4 +- waku/waku_rln_relay/rln/wrappers.nim | 13 +- 82 files changed, 4464 insertions(+), 5403 deletions(-) diff --git a/apps/chat2/chat2.nim b/apps/chat2/chat2.nim index 71d8a4e6a..4102cf074 100644 --- a/apps/chat2/chat2.nim +++ b/apps/chat2/chat2.nim @@ -50,8 +50,7 @@ import import libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub import ../../waku/waku_rln_relay -const Help = - """ +const Help = """ Commands: /[?|help|connect|nick|exit] help: Prints this help connect: dials a remote peer @@ -337,16 +336,16 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} = builder.withRecord(record) builder - .withNetworkConfigurationDetails( - conf.listenAddress, - Port(uint16(conf.tcpPort) + conf.portsShift), - extIp, - extTcpPort, - wsBindPort = Port(uint16(conf.websocketPort) + conf.portsShift), - wsEnabled = conf.websocketSupport, - wssEnabled = conf.websocketSecureSupport, - ) - .tryGet() + .withNetworkConfigurationDetails( + conf.listenAddress, + Port(uint16(conf.tcpPort) + conf.portsShift), + extIp, + extTcpPort, + wsBindPort = Port(uint16(conf.websocketPort) + conf.portsShift), + wsEnabled = conf.websocketSupport, + wssEnabled = conf.websocketSecureSupport, + ) + .tryGet() builder.build().tryGet() await node.start() diff --git a/apps/chat2bridge/chat2bridge.nim b/apps/chat2bridge/chat2bridge.nim index 77c9e553e..53eb5d04b 100644 --- a/apps/chat2bridge/chat2bridge.nim +++ b/apps/chat2bridge/chat2bridge.nim @@ -127,8 +127,10 @@ proc toMatterbridge( assert chat2Msg.isOk if not cmb.mbClient - .postMessage(text = string.fromBytes(chat2Msg[].payload), username = chat2Msg[].nick) - .containsValue(true): + .postMessage( + text = string.fromBytes(chat2Msg[].payload), username = chat2Msg[].nick + ) + .containsValue(true): chat2_mb_dropped.inc(labelValues = ["duplicate"]) error "Matterbridge host unreachable. Dropping message." @@ -175,10 +177,10 @@ proc new*( builder.withNodeKey(nodev2Key) builder - .withNetworkConfigurationDetails( - nodev2BindIp, nodev2BindPort, nodev2ExtIp, nodev2ExtPort - ) - .tryGet() + .withNetworkConfigurationDetails( + nodev2BindIp, nodev2BindPort, nodev2ExtIp, nodev2ExtPort + ) + .tryGet() builder.build().tryGet() return Chat2MatterBridge( diff --git a/apps/chat2mix/chat2mix.nim b/apps/chat2mix/chat2mix.nim index 558454307..8b786d7b6 100644 --- a/apps/chat2mix/chat2mix.nim +++ b/apps/chat2mix/chat2mix.nim @@ -57,8 +57,7 @@ import ../../waku/waku_rln_relay logScope: topics = "chat2 mix" -const Help = - """ +const Help = """ Commands: /[?|help|connect|nick|exit] help: Prints this help connect: dials a remote peer @@ -429,16 +428,16 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} = builder.withRecord(record) builder - .withNetworkConfigurationDetails( - conf.listenAddress, - Port(uint16(conf.tcpPort) + conf.portsShift), - extIp, - extTcpPort, - wsBindPort = Port(uint16(conf.websocketPort) + conf.portsShift), - wsEnabled = conf.websocketSupport, - wssEnabled = conf.websocketSecureSupport, - ) - .tryGet() + .withNetworkConfigurationDetails( + conf.listenAddress, + Port(uint16(conf.tcpPort) + conf.portsShift), + extIp, + extTcpPort, + wsBindPort = Port(uint16(conf.websocketPort) + conf.portsShift), + wsEnabled = conf.websocketSupport, + wssEnabled = conf.websocketSecureSupport, + ) + .tryGet() builder.build().tryGet() node.mountAutoSharding(conf.clusterId, conf.numShardsInNetwork).isOkOr: diff --git a/apps/chat2mix/config_chat2mix.nim b/apps/chat2mix/config_chat2mix.nim index 46cd481d7..4e5a32e6d 100644 --- a/apps/chat2mix/config_chat2mix.nim +++ b/apps/chat2mix/config_chat2mix.nim @@ -113,17 +113,16 @@ type shards* {. desc: "Shards index to subscribe to [0..NUM_SHARDS_IN_NETWORK-1]. Argument may be repeated.", - defaultValue: - @[ - uint16(0), - uint16(1), - uint16(2), - uint16(3), - uint16(4), - uint16(5), - uint16(6), - uint16(7), - ], + defaultValue: @[ + uint16(0), + uint16(1), + uint16(2), + uint16(3), + uint16(4), + uint16(5), + uint16(6), + uint16(7), + ], name: "shard" .}: seq[uint16] diff --git a/apps/wakucanary/wakucanary.nim b/apps/wakucanary/wakucanary.nim index 40bf4db45..bb68f7237 100644 --- a/apps/wakucanary/wakucanary.nim +++ b/apps/wakucanary/wakucanary.nim @@ -161,11 +161,10 @@ proc main(rng: ref HmacDrbgContext): Future[int] {.async.} = # create dns resolver let - nameServers = - @[ - initTAddress(parseIpAddress("1.1.1.1"), Port(53)), - initTAddress(parseIpAddress("1.0.0.1"), Port(53)), - ] + nameServers = @[ + initTAddress(parseIpAddress("1.1.1.1"), Port(53)), + initTAddress(parseIpAddress("1.0.0.1"), Port(53)), + ] resolver: DnsResolver = DnsResolver.new(nameServers) if conf.logLevel != LogLevel.NONE: diff --git a/liblogosdelivery/logos_delivery_api/debug_api.nim b/liblogosdelivery/logos_delivery_api/debug_api.nim index 623b3b08f..bb66a0e3f 100644 --- a/liblogosdelivery/logos_delivery_api/debug_api.nim +++ b/liblogosdelivery/logos_delivery_api/debug_api.nim @@ -45,7 +45,9 @@ proc logosdelivery_get_available_configs( for meta in optionMetas: configOptionDetails.add( - %*{meta.fieldName: meta.typeName & "(" & meta.defaultValue & ")", "desc": meta.desc} + %*{ + meta.fieldName: meta.typeName & "(" & meta.defaultValue & ")", "desc": meta.desc + } ) var jsonNode = newJObject() diff --git a/library/events/json_connection_status_change_event.nim b/library/events/json_connection_status_change_event.nim index 347a84c48..86bfda780 100644 --- a/library/events/json_connection_status_change_event.nim +++ b/library/events/json_connection_status_change_event.nim @@ -7,13 +7,9 @@ import ../../waku/api/types type JsonConnectionStatusChangeEvent* = ref object of JsonEvent status*: ConnectionStatus -proc new*( - T: type JsonConnectionStatusChangeEvent, status: ConnectionStatus -): T = - return JsonConnectionStatusChangeEvent( - eventType: "node_health_change", - status: status - ) +proc new*(T: type JsonConnectionStatusChangeEvent, status: ConnectionStatus): T = + return + JsonConnectionStatusChangeEvent(eventType: "node_health_change", status: status) method `$`*(event: JsonConnectionStatusChangeEvent): string = $(%*event) diff --git a/library/libwaku.nim b/library/libwaku.nim index eb3cdff5e..dd1ee9fd9 100644 --- a/library/libwaku.nim +++ b/library/libwaku.nim @@ -72,7 +72,7 @@ proc waku_new( relayHandler: onReceivedMessage(ctx), topicHealthChangeHandler: onTopicHealthChange(ctx), connectionChangeHandler: onConnectionChange(ctx), - connectionStatusChangeHandler: onConnectionStatusChange(ctx) + connectionStatusChangeHandler: onConnectionStatusChange(ctx), ) ffi.sendRequestToFFIThread( diff --git a/migrations/message_store_postgres/content_script_version_1.nim b/migrations/message_store_postgres/content_script_version_1.nim index 18133bdca..37c6bf2ec 100644 --- a/migrations/message_store_postgres/content_script_version_1.nim +++ b/migrations/message_store_postgres/content_script_version_1.nim @@ -1,5 +1,4 @@ -const ContentScriptVersion_1* = - """ +const ContentScriptVersion_1* = """ CREATE TABLE IF NOT EXISTS messages ( pubsubTopic VARCHAR NOT NULL, contentTopic VARCHAR NOT NULL, diff --git a/migrations/message_store_postgres/content_script_version_2.nim b/migrations/message_store_postgres/content_script_version_2.nim index 8c3656e64..4065a26c6 100644 --- a/migrations/message_store_postgres/content_script_version_2.nim +++ b/migrations/message_store_postgres/content_script_version_2.nim @@ -1,5 +1,4 @@ -const ContentScriptVersion_2* = - """ +const ContentScriptVersion_2* = """ ALTER TABLE IF EXISTS messages_backup RENAME TO messages; ALTER TABLE messages RENAME TO messages_backup; ALTER TABLE messages_backup DROP CONSTRAINT messageIndex; diff --git a/migrations/message_store_postgres/content_script_version_3.nim b/migrations/message_store_postgres/content_script_version_3.nim index 2938087cc..22e7308aa 100644 --- a/migrations/message_store_postgres/content_script_version_3.nim +++ b/migrations/message_store_postgres/content_script_version_3.nim @@ -1,5 +1,4 @@ -const ContentScriptVersion_3* = - """ +const ContentScriptVersion_3* = """ CREATE INDEX IF NOT EXISTS i_query ON messages (contentTopic, pubsubTopic, storedAt, id); diff --git a/migrations/message_store_postgres/content_script_version_4.nim b/migrations/message_store_postgres/content_script_version_4.nim index 50ee269f6..6412371e5 100644 --- a/migrations/message_store_postgres/content_script_version_4.nim +++ b/migrations/message_store_postgres/content_script_version_4.nim @@ -1,5 +1,4 @@ -const ContentScriptVersion_4* = - """ +const ContentScriptVersion_4* = """ ALTER TABLE messages ADD meta VARCHAR default null; CREATE INDEX IF NOT EXISTS i_query ON messages (contentTopic, pubsubTopic, storedAt, id); diff --git a/migrations/message_store_postgres/content_script_version_5.nim b/migrations/message_store_postgres/content_script_version_5.nim index a59b2da87..8210be4ac 100644 --- a/migrations/message_store_postgres/content_script_version_5.nim +++ b/migrations/message_store_postgres/content_script_version_5.nim @@ -1,5 +1,4 @@ -const ContentScriptVersion_5* = - """ +const ContentScriptVersion_5* = """ CREATE INDEX IF NOT EXISTS i_query_storedAt ON messages (storedAt, id); UPDATE version SET version = 5 WHERE version = 4; diff --git a/migrations/message_store_postgres/content_script_version_6.nim b/migrations/message_store_postgres/content_script_version_6.nim index 126ec6da1..88e92ed34 100644 --- a/migrations/message_store_postgres/content_script_version_6.nim +++ b/migrations/message_store_postgres/content_script_version_6.nim @@ -1,5 +1,4 @@ -const ContentScriptVersion_6* = - """ +const ContentScriptVersion_6* = """ -- we can drop the timestamp column because this data is also kept in the storedAt column ALTER TABLE messages DROP COLUMN timestamp; diff --git a/migrations/message_store_postgres/content_script_version_7.nim b/migrations/message_store_postgres/content_script_version_7.nim index 01d7ad84e..e2aba854d 100644 --- a/migrations/message_store_postgres/content_script_version_7.nim +++ b/migrations/message_store_postgres/content_script_version_7.nim @@ -1,5 +1,4 @@ -const ContentScriptVersion_7* = - """ +const ContentScriptVersion_7* = """ -- Create lookup table CREATE TABLE IF NOT EXISTS messages_lookup ( diff --git a/migrations/message_store_postgres/pg_migration_manager.nim b/migrations/message_store_postgres/pg_migration_manager.nim index 051ac9e79..89a443609 100644 --- a/migrations/message_store_postgres/pg_migration_manager.nim +++ b/migrations/message_store_postgres/pg_migration_manager.nim @@ -10,16 +10,15 @@ type MigrationScript* = object proc init*(T: type MigrationScript, targetVersion: int, scriptContent: string): T = return MigrationScript(targetVersion: targetVersion, scriptContent: scriptContent) -const PgMigrationScripts* = - @[ - MigrationScript(version: 1, scriptContent: ContentScriptVersion_1), - MigrationScript(version: 2, scriptContent: ContentScriptVersion_2), - MigrationScript(version: 3, scriptContent: ContentScriptVersion_3), - MigrationScript(version: 4, scriptContent: ContentScriptVersion_4), - MigrationScript(version: 5, scriptContent: ContentScriptVersion_5), - MigrationScript(version: 6, scriptContent: ContentScriptVersion_6), - MigrationScript(version: 7, scriptContent: ContentScriptVersion_7), - ] +const PgMigrationScripts* = @[ + MigrationScript(version: 1, scriptContent: ContentScriptVersion_1), + MigrationScript(version: 2, scriptContent: ContentScriptVersion_2), + MigrationScript(version: 3, scriptContent: ContentScriptVersion_3), + MigrationScript(version: 4, scriptContent: ContentScriptVersion_4), + MigrationScript(version: 5, scriptContent: ContentScriptVersion_5), + MigrationScript(version: 6, scriptContent: ContentScriptVersion_6), + MigrationScript(version: 7, scriptContent: ContentScriptVersion_7), +] proc getMigrationScripts*(currentVersion: int64, targetVersion: int64): seq[string] = var ret = newSeq[string]() diff --git a/tests/api/test_api_send.nim b/tests/api/test_api_send.nim index 30a176119..28f0ca2ff 100644 --- a/tests/api/test_api_send.nim +++ b/tests/api/test_api_send.nim @@ -88,7 +88,7 @@ proc waitForEvents( return await allFutures( manager.sentFuture, manager.propagatedFuture, manager.errorFuture ) - .withTimeout(timeout) + .withTimeout(timeout) proc outcomes(manager: SendEventListenerManager): set[SendEventOutcome] = if manager.sentFuture.completed(): diff --git a/tests/api/test_entry_nodes.nim b/tests/api/test_entry_nodes.nim index 38dc38ba4..a6faf58c8 100644 --- a/tests/api/test_entry_nodes.nim +++ b/tests/api/test_entry_nodes.nim @@ -126,12 +126,11 @@ suite "Entry Nodes Classification": suite "Entry Nodes Processing": test "Process mixed entry nodes": - let entryNodes = - @[ - "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", - "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", - "enr:-QESuEC1p_s3xJzAC_XlOuuNrhVUETmfhbm1wxRGis0f7DlqGSw2FM-p2Vn7gmfkTTnAe8Ys2cgGBN8ufJnvzKQFZqFMBgmlkgnY0iXNlY3AyNTZrMaEDS8-D878DrdbNwcuY-3p1qdDp5MOoCurhdsNPJTXZ3c5g3RjcIJ2X4N1ZHCCd2g", - ] + let entryNodes = @[ + "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", + "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", + "enr:-QESuEC1p_s3xJzAC_XlOuuNrhVUETmfhbm1wxRGis0f7DlqGSw2FM-p2Vn7gmfkTTnAe8Ys2cgGBN8ufJnvzKQFZqFMBgmlkgnY0iXNlY3AyNTZrMaEDS8-D878DrdbNwcuY-3p1qdDp5MOoCurhdsNPJTXZ3c5g3RjcIJ2X4N1ZHCCd2g", + ] let result = processEntryNodes(entryNodes) check: @@ -147,11 +146,10 @@ suite "Entry Nodes Processing": staticNodes[0] == entryNodes[1] # multiaddr added to static test "Process only ENRTree nodes": - let entryNodes = - @[ - "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", - "enrtree://ANOTHER_TREE@example.com", - ] + let entryNodes = @[ + "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", + "enrtree://ANOTHER_TREE@example.com", + ] let result = processEntryNodes(entryNodes) check: @@ -165,11 +163,10 @@ suite "Entry Nodes Processing": enrTreeUrls == entryNodes test "Process only multiaddresses": - let entryNodes = - @[ - "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", - "/ip4/192.168.1.1/tcp/60001/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYd", - ] + let entryNodes = @[ + "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", + "/ip4/192.168.1.1/tcp/60001/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYd", + ] let result = processEntryNodes(entryNodes) check: @@ -183,11 +180,10 @@ suite "Entry Nodes Processing": staticNodes == entryNodes test "Process only ENR nodes": - let entryNodes = - @[ - "enr:-QESuEC1p_s3xJzAC_XlOuuNrhVUETmfhbm1wxRGis0f7DlqGSw2FM-p2Vn7gmfkTTnAe8Ys2cgGBN8ufJnvzKQFZqFMBgmlkgnY0iXNlY3AyNTZrMaEDS8-D878DrdbNwcuY-3p1qdDp5MOoCurhdsNPJTXZ3c5g3RjcIJ2X4N1ZHCCd2g", - "enr:-QEkuECnZ3IbVAgkOzv-QLnKC4dRKAPRY80m1-R7G8jZ7yfT3ipEfBrhKN7ARcQgQ-vg-h40AQzyvAkPYlHPaFKk6u9MBgmlkgnY0iXNlY3AyNTZrMaEDk49D8JjMSns4p1XVNBvJquOUzT4PENSJknkROspfAFGg3RjcIJ2X4N1ZHCCd2g", - ] + let entryNodes = @[ + "enr:-QESuEC1p_s3xJzAC_XlOuuNrhVUETmfhbm1wxRGis0f7DlqGSw2FM-p2Vn7gmfkTTnAe8Ys2cgGBN8ufJnvzKQFZqFMBgmlkgnY0iXNlY3AyNTZrMaEDS8-D878DrdbNwcuY-3p1qdDp5MOoCurhdsNPJTXZ3c5g3RjcIJ2X4N1ZHCCd2g", + "enr:-QEkuECnZ3IbVAgkOzv-QLnKC4dRKAPRY80m1-R7G8jZ7yfT3ipEfBrhKN7ARcQgQ-vg-h40AQzyvAkPYlHPaFKk6u9MBgmlkgnY0iXNlY3AyNTZrMaEDk49D8JjMSns4p1XVNBvJquOUzT4PENSJknkROspfAFGg3RjcIJ2X4N1ZHCCd2g", + ] let result = processEntryNodes(entryNodes) check: @@ -224,13 +220,12 @@ suite "Entry Nodes Processing": "Entry node error: Unrecognized entry node format. Must start with 'enrtree:', 'enr:', or '/'" test "Process different multiaddr formats": - let entryNodes = - @[ - "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", - "/ip6/::1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYd", - "/dns4/example.com/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYe", - "/dns/node.example.org/tcp/443/wss/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYf", - ] + let entryNodes = @[ + "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", + "/ip6/::1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYd", + "/dns4/example.com/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYe", + "/dns/node.example.org/tcp/443/wss/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYf", + ] let result = processEntryNodes(entryNodes) check: @@ -244,13 +239,12 @@ suite "Entry Nodes Processing": staticNodes == entryNodes test "Process with duplicate entries": - let entryNodes = - @[ - "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", - "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", - "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", - "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", - ] + let entryNodes = @[ + "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", + "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", + "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", + "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", + ] let result = processEntryNodes(entryNodes) check: diff --git a/tests/common/test_base64_codec.nim b/tests/common/test_base64_codec.nim index 1c2d04c45..b9ac9e464 100644 --- a/tests/common/test_base64_codec.nim +++ b/tests/common/test_base64_codec.nim @@ -4,23 +4,22 @@ import std/strutils, results, stew/byteutils, testutils/unittests import waku/common/base64 suite "Waku Common - stew base64 wrapper": - const TestData = - @[ - # Test vectors from RFC 4648 - # See: https://datatracker.ietf.org/doc/html/rfc4648#section-10 - ("", Base64String("")), - ("f", Base64String("Zg==")), - ("fo", Base64String("Zm8=")), - ("foo", Base64String("Zm9v")), - ("foob", Base64String("Zm9vYg==")), - ("fooba", Base64String("Zm9vYmE=")), - ("foobar", Base64String("Zm9vYmFy")), + const TestData = @[ + # Test vectors from RFC 4648 + # See: https://datatracker.ietf.org/doc/html/rfc4648#section-10 + ("", Base64String("")), + ("f", Base64String("Zg==")), + ("fo", Base64String("Zm8=")), + ("foo", Base64String("Zm9v")), + ("foob", Base64String("Zm9vYg==")), + ("fooba", Base64String("Zm9vYmE=")), + ("foobar", Base64String("Zm9vYmFy")), - # Custom test vectors - ("\x01", Base64String("AQ==")), - ("\x13", Base64String("Ew==")), - ("\x01\x02\x03\x04", Base64String("AQIDBA==")), - ] + # Custom test vectors + ("\x01", Base64String("AQ==")), + ("\x13", Base64String("Ew==")), + ("\x01\x02\x03\x04", Base64String("AQIDBA==")), + ] for (plaintext, encoded) in TestData: test "encode into base64 (" & escape(plaintext) & " -> \"" & string(encoded) & "\")": diff --git a/tests/common/test_request_broker.nim b/tests/common/test_request_broker.nim index 87065a916..b1e16979b 100644 --- a/tests/common/test_request_broker.nim +++ b/tests/common/test_request_broker.nim @@ -45,11 +45,11 @@ static: suite "RequestBroker macro (async mode)": test "serves zero-argument providers": check SimpleResponse - .setProvider( - proc(): Future[Result[SimpleResponse, string]] {.async.} = - ok(SimpleResponse(value: "hi")) - ) - .isOk() + .setProvider( + proc(): Future[Result[SimpleResponse, string]] {.async.} = + ok(SimpleResponse(value: "hi")) + ) + .isOk() let res = waitFor SimpleResponse.request() check res.isOk() @@ -65,12 +65,14 @@ suite "RequestBroker macro (async mode)": test "serves input-based providers": var seen: seq[string] = @[] check KeyedResponse - .setProvider( - proc(key: string, subKey: int): Future[Result[KeyedResponse, string]] {.async.} = - seen.add(key) - ok(KeyedResponse(key: key, payload: key & "-payload+" & $subKey)) - ) - .isOk() + .setProvider( + proc( + key: string, subKey: int + ): Future[Result[KeyedResponse, string]] {.async.} = + seen.add(key) + ok(KeyedResponse(key: key, payload: key & "-payload+" & $subKey)) + ) + .isOk() let res = waitFor KeyedResponse.request("topic", 1) check res.isOk() @@ -82,11 +84,13 @@ suite "RequestBroker macro (async mode)": test "catches provider exception": check KeyedResponse - .setProvider( - proc(key: string, subKey: int): Future[Result[KeyedResponse, string]] {.async.} = - raise newException(ValueError, "simulated failure") - ) - .isOk() + .setProvider( + proc( + key: string, subKey: int + ): Future[Result[KeyedResponse, string]] {.async.} = + raise newException(ValueError, "simulated failure") + ) + .isOk() let res = waitFor KeyedResponse.request("neglected", 11) check res.isErr() @@ -101,18 +105,18 @@ suite "RequestBroker macro (async mode)": test "supports both provider types simultaneously": check DualResponse - .setProvider( - proc(): Future[Result[DualResponse, string]] {.async.} = - ok(DualResponse(note: "base", count: 1)) - ) - .isOk() + .setProvider( + proc(): Future[Result[DualResponse, string]] {.async.} = + ok(DualResponse(note: "base", count: 1)) + ) + .isOk() check DualResponse - .setProvider( - proc(suffix: string): Future[Result[DualResponse, string]] {.async.} = - ok(DualResponse(note: "base" & suffix, count: suffix.len)) - ) - .isOk() + .setProvider( + proc(suffix: string): Future[Result[DualResponse, string]] {.async.} = + ok(DualResponse(note: "base" & suffix, count: suffix.len)) + ) + .isOk() let noInput = waitFor DualResponse.request() check noInput.isOk() @@ -127,11 +131,11 @@ suite "RequestBroker macro (async mode)": test "clearProvider resets both entries": check DualResponse - .setProvider( - proc(): Future[Result[DualResponse, string]] {.async.} = - ok(DualResponse(note: "temp", count: 0)) - ) - .isOk() + .setProvider( + proc(): Future[Result[DualResponse, string]] {.async.} = + ok(DualResponse(note: "temp", count: 0)) + ) + .isOk() DualResponse.clearProvider() let res = waitFor DualResponse.request() @@ -139,11 +143,11 @@ suite "RequestBroker macro (async mode)": test "implicit zero-argument provider works by default": check ImplicitResponse - .setProvider( - proc(): Future[Result[ImplicitResponse, string]] {.async.} = - ok(ImplicitResponse(note: "auto")) - ) - .isOk() + .setProvider( + proc(): Future[Result[ImplicitResponse, string]] {.async.} = + ok(ImplicitResponse(note: "auto")) + ) + .isOk() let res = waitFor ImplicitResponse.request() check res.isOk() @@ -158,18 +162,18 @@ suite "RequestBroker macro (async mode)": test "no provider override": check DualResponse - .setProvider( - proc(): Future[Result[DualResponse, string]] {.async.} = - ok(DualResponse(note: "base", count: 1)) - ) - .isOk() + .setProvider( + proc(): Future[Result[DualResponse, string]] {.async.} = + ok(DualResponse(note: "base", count: 1)) + ) + .isOk() check DualResponse - .setProvider( - proc(suffix: string): Future[Result[DualResponse, string]] {.async.} = - ok(DualResponse(note: "base" & suffix, count: suffix.len)) - ) - .isOk() + .setProvider( + proc(suffix: string): Future[Result[DualResponse, string]] {.async.} = + ok(DualResponse(note: "base" & suffix, count: suffix.len)) + ) + .isOk() let overrideProc = proc(): Future[Result[DualResponse, string]] {.async.} = ok(DualResponse(note: "something else", count: 1)) @@ -207,27 +211,27 @@ suite "RequestBroker macro (async mode)": SimpleResponse.clearProvider() check SimpleResponse - .setProvider( - proc(): Future[Result[SimpleResponse, string]] {.async.} = - ok(SimpleResponse(value: "default")) - ) - .isOk() + .setProvider( + proc(): Future[Result[SimpleResponse, string]] {.async.} = + ok(SimpleResponse(value: "default")) + ) + .isOk() check SimpleResponse - .setProvider( - BrokerContext(0x11111111'u32), - proc(): Future[Result[SimpleResponse, string]] {.async.} = - ok(SimpleResponse(value: "one")), - ) - .isOk() + .setProvider( + BrokerContext(0x11111111'u32), + proc(): Future[Result[SimpleResponse, string]] {.async.} = + ok(SimpleResponse(value: "one")), + ) + .isOk() check SimpleResponse - .setProvider( - BrokerContext(0x22222222'u32), - proc(): Future[Result[SimpleResponse, string]] {.async.} = - ok(SimpleResponse(value: "two")), - ) - .isOk() + .setProvider( + BrokerContext(0x22222222'u32), + proc(): Future[Result[SimpleResponse, string]] {.async.} = + ok(SimpleResponse(value: "two")), + ) + .isOk() let defaultRes = waitFor SimpleResponse.request() check defaultRes.isOk() @@ -246,12 +250,12 @@ suite "RequestBroker macro (async mode)": check missing.error.contains("no provider registered for broker context") check SimpleResponse - .setProvider( - BrokerContext(0x11111111'u32), - proc(): Future[Result[SimpleResponse, string]] {.async.} = - ok(SimpleResponse(value: "dup")), - ) - .isErr() + .setProvider( + BrokerContext(0x11111111'u32), + proc(): Future[Result[SimpleResponse, string]] {.async.} = + ok(SimpleResponse(value: "dup")), + ) + .isErr() SimpleResponse.clearProvider() @@ -259,27 +263,33 @@ suite "RequestBroker macro (async mode)": KeyedResponse.clearProvider() check KeyedResponse - .setProvider( - proc(key: string, subKey: int): Future[Result[KeyedResponse, string]] {.async.} = - ok(KeyedResponse(key: "default-" & key, payload: $subKey)) - ) - .isOk() + .setProvider( + proc( + key: string, subKey: int + ): Future[Result[KeyedResponse, string]] {.async.} = + ok(KeyedResponse(key: "default-" & key, payload: $subKey)) + ) + .isOk() check KeyedResponse - .setProvider( - BrokerContext(0xABCDEF01'u32), - proc(key: string, subKey: int): Future[Result[KeyedResponse, string]] {.async.} = - ok(KeyedResponse(key: "k1-" & key, payload: "p" & $subKey)), - ) - .isOk() + .setProvider( + BrokerContext(0xABCDEF01'u32), + proc( + key: string, subKey: int + ): Future[Result[KeyedResponse, string]] {.async.} = + ok(KeyedResponse(key: "k1-" & key, payload: "p" & $subKey)), + ) + .isOk() check KeyedResponse - .setProvider( - BrokerContext(0xABCDEF02'u32), - proc(key: string, subKey: int): Future[Result[KeyedResponse, string]] {.async.} = - ok(KeyedResponse(key: "k2-" & key, payload: "q" & $subKey)), - ) - .isOk() + .setProvider( + BrokerContext(0xABCDEF02'u32), + proc( + key: string, subKey: int + ): Future[Result[KeyedResponse, string]] {.async.} = + ok(KeyedResponse(key: "k2-" & key, payload: "q" & $subKey)), + ) + .isOk() let d = waitFor KeyedResponse.request("topic", 7) check d.isOk() @@ -343,11 +353,11 @@ static: suite "RequestBroker macro (sync mode)": test "serves zero-argument providers (sync)": check SimpleResponseSync - .setProvider( - proc(): Result[SimpleResponseSync, string] = - ok(SimpleResponseSync(value: "hi")) - ) - .isOk() + .setProvider( + proc(): Result[SimpleResponseSync, string] = + ok(SimpleResponseSync(value: "hi")) + ) + .isOk() let res = SimpleResponseSync.request() check res.isOk() @@ -363,12 +373,12 @@ suite "RequestBroker macro (sync mode)": test "serves input-based providers (sync)": var seen: seq[string] = @[] check KeyedResponseSync - .setProvider( - proc(key: string, subKey: int): Result[KeyedResponseSync, string] = - seen.add(key) - ok(KeyedResponseSync(key: key, payload: key & "-payload+" & $subKey)) - ) - .isOk() + .setProvider( + proc(key: string, subKey: int): Result[KeyedResponseSync, string] = + seen.add(key) + ok(KeyedResponseSync(key: key, payload: key & "-payload+" & $subKey)) + ) + .isOk() let res = KeyedResponseSync.request("topic", 1) check res.isOk() @@ -380,11 +390,11 @@ suite "RequestBroker macro (sync mode)": test "catches provider exception (sync)": check KeyedResponseSync - .setProvider( - proc(key: string, subKey: int): Result[KeyedResponseSync, string] = - raise newException(ValueError, "simulated failure") - ) - .isOk() + .setProvider( + proc(key: string, subKey: int): Result[KeyedResponseSync, string] = + raise newException(ValueError, "simulated failure") + ) + .isOk() let res = KeyedResponseSync.request("neglected", 11) check res.isErr() @@ -399,18 +409,18 @@ suite "RequestBroker macro (sync mode)": test "supports both provider types simultaneously (sync)": check DualResponseSync - .setProvider( - proc(): Result[DualResponseSync, string] = - ok(DualResponseSync(note: "base", count: 1)) - ) - .isOk() + .setProvider( + proc(): Result[DualResponseSync, string] = + ok(DualResponseSync(note: "base", count: 1)) + ) + .isOk() check DualResponseSync - .setProvider( - proc(suffix: string): Result[DualResponseSync, string] = - ok(DualResponseSync(note: "base" & suffix, count: suffix.len)) - ) - .isOk() + .setProvider( + proc(suffix: string): Result[DualResponseSync, string] = + ok(DualResponseSync(note: "base" & suffix, count: suffix.len)) + ) + .isOk() let noInput = DualResponseSync.request() check noInput.isOk() @@ -425,11 +435,11 @@ suite "RequestBroker macro (sync mode)": test "clearProvider resets both entries (sync)": check DualResponseSync - .setProvider( - proc(): Result[DualResponseSync, string] = - ok(DualResponseSync(note: "temp", count: 0)) - ) - .isOk() + .setProvider( + proc(): Result[DualResponseSync, string] = + ok(DualResponseSync(note: "temp", count: 0)) + ) + .isOk() DualResponseSync.clearProvider() let res = DualResponseSync.request() @@ -437,11 +447,11 @@ suite "RequestBroker macro (sync mode)": test "implicit zero-argument provider works by default (sync)": check ImplicitResponseSync - .setProvider( - proc(): Result[ImplicitResponseSync, string] = - ok(ImplicitResponseSync(note: "auto")) - ) - .isOk() + .setProvider( + proc(): Result[ImplicitResponseSync, string] = + ok(ImplicitResponseSync(note: "auto")) + ) + .isOk() let res = ImplicitResponseSync.request() check res.isOk() @@ -456,11 +466,11 @@ suite "RequestBroker macro (sync mode)": test "implicit zero-argument provider raises error (sync)": check ImplicitResponseSync - .setProvider( - proc(): Result[ImplicitResponseSync, string] = - raise newException(ValueError, "simulated failure") - ) - .isOk() + .setProvider( + proc(): Result[ImplicitResponseSync, string] = + raise newException(ValueError, "simulated failure") + ) + .isOk() let res = ImplicitResponseSync.request() check res.isErr() @@ -472,19 +482,19 @@ suite "RequestBroker macro (sync mode)": SimpleResponseSync.clearProvider() check SimpleResponseSync - .setProvider( - proc(): Result[SimpleResponseSync, string] = - ok(SimpleResponseSync(value: "default")) - ) - .isOk() + .setProvider( + proc(): Result[SimpleResponseSync, string] = + ok(SimpleResponseSync(value: "default")) + ) + .isOk() check SimpleResponseSync - .setProvider( - BrokerContext(0x10101010'u32), - proc(): Result[SimpleResponseSync, string] = - ok(SimpleResponseSync(value: "ten")), - ) - .isOk() + .setProvider( + BrokerContext(0x10101010'u32), + proc(): Result[SimpleResponseSync, string] = + ok(SimpleResponseSync(value: "ten")), + ) + .isOk() let defaultRes = SimpleResponseSync.request() check defaultRes.isOk() @@ -504,19 +514,19 @@ suite "RequestBroker macro (sync mode)": KeyedResponseSync.clearProvider() check KeyedResponseSync - .setProvider( - proc(key: string, subKey: int): Result[KeyedResponseSync, string] = - ok(KeyedResponseSync(key: "default-" & key, payload: $subKey)) - ) - .isOk() + .setProvider( + proc(key: string, subKey: int): Result[KeyedResponseSync, string] = + ok(KeyedResponseSync(key: "default-" & key, payload: $subKey)) + ) + .isOk() check KeyedResponseSync - .setProvider( - BrokerContext(0xA0A0A0A0'u32), - proc(key: string, subKey: int): Result[KeyedResponseSync, string] = - ok(KeyedResponseSync(key: "k-" & key, payload: "p" & $subKey)), - ) - .isOk() + .setProvider( + BrokerContext(0xA0A0A0A0'u32), + proc(key: string, subKey: int): Result[KeyedResponseSync, string] = + ok(KeyedResponseSync(key: "k-" & key, payload: "p" & $subKey)), + ) + .isOk() let d = KeyedResponseSync.request("topic", 2) check d.isOk() @@ -576,11 +586,11 @@ RequestBroker(sync): suite "RequestBroker macro (POD/external types)": test "supports non-object response types (async)": check PodResponse - .setProvider( - proc(): Future[Result[PodResponse, string]] {.async.} = - ok(PodResponse(123)) - ) - .isOk() + .setProvider( + proc(): Future[Result[PodResponse, string]] {.async.} = + ok(PodResponse(123)) + ) + .isOk() let res = waitFor PodResponse.request() check res.isOk() @@ -590,11 +600,11 @@ suite "RequestBroker macro (POD/external types)": test "supports aliased external types (async)": check ExternalAliasedResponse - .setProvider( - proc(): Future[Result[ExternalAliasedResponse, string]] {.async.} = - ok(ExternalAliasedResponse(ExternalDefinedTypeAsync(label: "ext"))) - ) - .isOk() + .setProvider( + proc(): Future[Result[ExternalAliasedResponse, string]] {.async.} = + ok(ExternalAliasedResponse(ExternalDefinedTypeAsync(label: "ext"))) + ) + .isOk() let res = waitFor ExternalAliasedResponse.request() check res.isOk() @@ -604,11 +614,11 @@ suite "RequestBroker macro (POD/external types)": test "supports aliased external types (sync)": check ExternalAliasedResponseSync - .setProvider( - proc(): Result[ExternalAliasedResponseSync, string] = - ok(ExternalAliasedResponseSync(ExternalDefinedTypeSync(label: "ext"))) - ) - .isOk() + .setProvider( + proc(): Result[ExternalAliasedResponseSync, string] = + ok(ExternalAliasedResponseSync(ExternalDefinedTypeSync(label: "ext"))) + ) + .isOk() let res = ExternalAliasedResponseSync.request() check res.isOk() @@ -618,32 +628,32 @@ suite "RequestBroker macro (POD/external types)": test "distinct response types avoid overload ambiguity (sync)": check DistinctStringResponseA - .setProvider( - proc(): Result[DistinctStringResponseA, string] = - ok(DistinctStringResponseA("a")) - ) - .isOk() + .setProvider( + proc(): Result[DistinctStringResponseA, string] = + ok(DistinctStringResponseA("a")) + ) + .isOk() check DistinctStringResponseB - .setProvider( - proc(): Result[DistinctStringResponseB, string] = - ok(DistinctStringResponseB("b")) - ) - .isOk() + .setProvider( + proc(): Result[DistinctStringResponseB, string] = + ok(DistinctStringResponseB("b")) + ) + .isOk() check ExternalDistinctResponseA - .setProvider( - proc(): Result[ExternalDistinctResponseA, string] = - ok(ExternalDistinctResponseA(ExternalDefinedTypeShared(label: "ea"))) - ) - .isOk() + .setProvider( + proc(): Result[ExternalDistinctResponseA, string] = + ok(ExternalDistinctResponseA(ExternalDefinedTypeShared(label: "ea"))) + ) + .isOk() check ExternalDistinctResponseB - .setProvider( - proc(): Result[ExternalDistinctResponseB, string] = - ok(ExternalDistinctResponseB(ExternalDefinedTypeShared(label: "eb"))) - ) - .isOk() + .setProvider( + proc(): Result[ExternalDistinctResponseB, string] = + ok(ExternalDistinctResponseB(ExternalDefinedTypeShared(label: "eb"))) + ) + .isOk() let resA = DistinctStringResponseA.request() let resB = DistinctStringResponseB.request() diff --git a/tests/common/test_sqlite_migrations.nim b/tests/common/test_sqlite_migrations.nim index 9e67fb9c8..2a9cae609 100644 --- a/tests/common/test_sqlite_migrations.nim +++ b/tests/common/test_sqlite_migrations.nim @@ -29,17 +29,16 @@ suite "SQLite - migrations": test "filter and order migration script file paths": ## Given - let paths = - @[ - sourceDir / "00001_valid.up.sql", - sourceDir / "00002_alsoValidWithUpperCaseExtension.UP.SQL", - sourceDir / "00007_unorderedValid.up.sql", - sourceDir / "00003_validRepeated.up.sql", - sourceDir / "00003_validRepeated.up.sql", - sourceDir / "00666_noMigrationScript.bmp", - sourceDir / "00X00_invalidVersion.down.sql", - sourceDir / "00008_notWithinVersionRange.up.sql", - ] + let paths = @[ + sourceDir / "00001_valid.up.sql", + sourceDir / "00002_alsoValidWithUpperCaseExtension.UP.SQL", + sourceDir / "00007_unorderedValid.up.sql", + sourceDir / "00003_validRepeated.up.sql", + sourceDir / "00003_validRepeated.up.sql", + sourceDir / "00666_noMigrationScript.bmp", + sourceDir / "00X00_invalidVersion.down.sql", + sourceDir / "00008_notWithinVersionRange.up.sql", + ] let lowerVersion = 0 @@ -64,16 +63,14 @@ suite "SQLite - migrations": test "break migration scripts into queries": ## Given - let statement1 = - """CREATE TABLE contacts1 ( + let statement1 = """CREATE TABLE contacts1 ( contact_id INTEGER PRIMARY KEY, first_name TEXT NOT NULL, last_name TEXT NOT NULL, email TEXT NOT NULL UNIQUE, phone TEXT NOT NULL UNIQUE );""" - let statement2 = - """CREATE TABLE contacts2 ( + let statement2 = """CREATE TABLE contacts2 ( contact_id INTEGER PRIMARY KEY, first_name TEXT NOT NULL, last_name TEXT NOT NULL, @@ -91,16 +88,14 @@ suite "SQLite - migrations": test "break statements script into queries - empty statements": ## Given - let statement1 = - """CREATE TABLE contacts1 ( + let statement1 = """CREATE TABLE contacts1 ( contact_id INTEGER PRIMARY KEY, first_name TEXT NOT NULL, last_name TEXT NOT NULL, email TEXT NOT NULL UNIQUE, phone TEXT NOT NULL UNIQUE );""" - let statement2 = - """CREATE TABLE contacts2 ( + let statement2 = """CREATE TABLE contacts2 ( contact_id INTEGER PRIMARY KEY, first_name TEXT NOT NULL, last_name TEXT NOT NULL, diff --git a/tests/node/peer_manager/peer_store/test_waku_peer_storage.nim b/tests/node/peer_manager/peer_store/test_waku_peer_storage.nim index c0e25ec6a..bf052205b 100644 --- a/tests/node/peer_manager/peer_store/test_waku_peer_storage.nim +++ b/tests/node/peer_manager/peer_store/test_waku_peer_storage.nim @@ -10,22 +10,21 @@ import import waku/waku_core/peers, waku/node/peer_manager/peer_store/waku_peer_storage proc `==`(a, b: RemotePeerInfo): bool = - let comparisons = - @[ - a.peerId == b.peerId, - a.addrs == b.addrs, - a.enr == b.enr, - a.protocols == b.protocols, - a.agent == b.agent, - a.protoVersion == b.protoVersion, - a.publicKey == b.publicKey, - a.connectedness == b.connectedness, - a.disconnectTime == b.disconnectTime, - a.origin == b.origin, - a.direction == b.direction, - a.lastFailedConn == b.lastFailedConn, - a.numberFailedConn == b.numberFailedConn, - ] + let comparisons = @[ + a.peerId == b.peerId, + a.addrs == b.addrs, + a.enr == b.enr, + a.protocols == b.protocols, + a.agent == b.agent, + a.protoVersion == b.protoVersion, + a.publicKey == b.publicKey, + a.connectedness == b.connectedness, + a.disconnectTime == b.disconnectTime, + a.origin == b.origin, + a.direction == b.direction, + a.lastFailedConn == b.lastFailedConn, + a.numberFailedConn == b.numberFailedConn, + ] allIt(comparisons, it == true) @@ -61,18 +60,17 @@ suite "Protobuf Serialisation": suite "encode": test "simple": # Given the expected bytes representation of a valid RemotePeerInfo - let expectedBuffer: seq[byte] = - @[ - 10, 39, 0, 37, 8, 2, 18, 33, 3, 43, 246, 238, 219, 109, 147, 79, 129, 40, 145, - 217, 209, 109, 105, 185, 186, 200, 180, 203, 72, 166, 220, 196, 232, 170, 74, - 141, 125, 255, 112, 238, 204, 18, 8, 4, 192, 168, 0, 1, 6, 31, 144, 34, 95, 8, - 3, 18, 91, 48, 89, 48, 19, 6, 7, 42, 134, 72, 206, 61, 2, 1, 6, 8, 42, 134, - 72, 206, 61, 3, 1, 7, 3, 66, 0, 4, 222, 61, 48, 15, 163, 106, 224, 232, 245, - 213, 48, 137, 157, 131, 171, 171, 68, 171, 243, 22, 31, 22, 42, 75, 201, 1, - 216, 230, 236, 218, 2, 14, 139, 109, 95, 141, 163, 5, 37, 231, 29, 104, 81, - 81, 12, 9, 142, 92, 71, 198, 70, 165, 151, 251, 77, 206, 192, 52, 233, 247, - 124, 64, 158, 98, 40, 0, 48, 0, - ] + let expectedBuffer: seq[byte] = @[ + 10, 39, 0, 37, 8, 2, 18, 33, 3, 43, 246, 238, 219, 109, 147, 79, 129, 40, 145, + 217, 209, 109, 105, 185, 186, 200, 180, 203, 72, 166, 220, 196, 232, 170, 74, + 141, 125, 255, 112, 238, 204, 18, 8, 4, 192, 168, 0, 1, 6, 31, 144, 34, 95, 8, + 3, 18, 91, 48, 89, 48, 19, 6, 7, 42, 134, 72, 206, 61, 2, 1, 6, 8, 42, 134, 72, + 206, 61, 3, 1, 7, 3, 66, 0, 4, 222, 61, 48, 15, 163, 106, 224, 232, 245, 213, + 48, 137, 157, 131, 171, 171, 68, 171, 243, 22, 31, 22, 42, 75, 201, 1, 216, 230, + 236, 218, 2, 14, 139, 109, 95, 141, 163, 5, 37, 231, 29, 104, 81, 81, 12, 9, + 142, 92, 71, 198, 70, 165, 151, 251, 77, 206, 192, 52, 233, 247, 124, 64, 158, + 98, 40, 0, 48, 0, + ] # When converting a valid RemotePeerInfo to a ProtoBuffer let encodedRemotePeerInfo = encode(remotePeerInfo).get() @@ -87,18 +85,17 @@ suite "Protobuf Serialisation": suite "decode": test "simple": # Given the bytes representation of a valid RemotePeerInfo - let buffer: seq[byte] = - @[ - 10, 39, 0, 37, 8, 2, 18, 33, 3, 43, 246, 238, 219, 109, 147, 79, 129, 40, 145, - 217, 209, 109, 105, 185, 186, 200, 180, 203, 72, 166, 220, 196, 232, 170, 74, - 141, 125, 255, 112, 238, 204, 18, 8, 4, 192, 168, 0, 1, 6, 31, 144, 34, 95, 8, - 3, 18, 91, 48, 89, 48, 19, 6, 7, 42, 134, 72, 206, 61, 2, 1, 6, 8, 42, 134, - 72, 206, 61, 3, 1, 7, 3, 66, 0, 4, 222, 61, 48, 15, 163, 106, 224, 232, 245, - 213, 48, 137, 157, 131, 171, 171, 68, 171, 243, 22, 31, 22, 42, 75, 201, 1, - 216, 230, 236, 218, 2, 14, 139, 109, 95, 141, 163, 5, 37, 231, 29, 104, 81, - 81, 12, 9, 142, 92, 71, 198, 70, 165, 151, 251, 77, 206, 192, 52, 233, 247, - 124, 64, 158, 98, 40, 0, 48, 0, - ] + let buffer: seq[byte] = @[ + 10, 39, 0, 37, 8, 2, 18, 33, 3, 43, 246, 238, 219, 109, 147, 79, 129, 40, 145, + 217, 209, 109, 105, 185, 186, 200, 180, 203, 72, 166, 220, 196, 232, 170, 74, + 141, 125, 255, 112, 238, 204, 18, 8, 4, 192, 168, 0, 1, 6, 31, 144, 34, 95, 8, + 3, 18, 91, 48, 89, 48, 19, 6, 7, 42, 134, 72, 206, 61, 2, 1, 6, 8, 42, 134, 72, + 206, 61, 3, 1, 7, 3, 66, 0, 4, 222, 61, 48, 15, 163, 106, 224, 232, 245, 213, + 48, 137, 157, 131, 171, 171, 68, 171, 243, 22, 31, 22, 42, 75, 201, 1, 216, 230, + 236, 218, 2, 14, 139, 109, 95, 141, 163, 5, 37, 231, 29, 104, 81, 81, 12, 9, + 142, 92, 71, 198, 70, 165, 151, 251, 77, 206, 192, 52, 233, 247, 124, 64, 158, + 98, 40, 0, 48, 0, + ] # When converting a valid buffer to RemotePeerInfo let decodedRemotePeerInfo = RemotePeerInfo.decode(buffer).get() diff --git a/tests/node/test_wakunode_health_monitor.nim b/tests/node/test_wakunode_health_monitor.nim index 8be9c444d..416dc9dda 100644 --- a/tests/node/test_wakunode_health_monitor.nim +++ b/tests/node/test_wakunode_health_monitor.nim @@ -35,13 +35,12 @@ proc protoHealthMock(kind: WakuProtocol, health: HealthStatus): ProtocolHealth = suite "Health Monitor - health state calculation": test "Disconnected, zero peers": - let protocols = - @[ - protoHealthMock(RelayProtocol, HealthStatus.NOT_READY), - protoHealthMock(StoreClientProtocol, HealthStatus.NOT_READY), - protoHealthMock(FilterClientProtocol, HealthStatus.NOT_READY), - protoHealthMock(LightpushClientProtocol, HealthStatus.NOT_READY), - ] + let protocols = @[ + protoHealthMock(RelayProtocol, HealthStatus.NOT_READY), + protoHealthMock(StoreClientProtocol, HealthStatus.NOT_READY), + protoHealthMock(FilterClientProtocol, HealthStatus.NOT_READY), + protoHealthMock(LightpushClientProtocol, HealthStatus.NOT_READY), + ] let strength = initTable[WakuProtocol, int]() let state = calculateConnectionState(protocols, strength, some(MockDLow)) check state == ConnectionStatus.Disconnected @@ -64,13 +63,12 @@ suite "Health Monitor - health state calculation": check state == ConnectionStatus.Connected test "Connected, robust edge": - let protocols = - @[ - protoHealthMock(RelayProtocol, HealthStatus.NOT_MOUNTED), - protoHealthMock(LightpushClientProtocol, HealthStatus.READY), - protoHealthMock(FilterClientProtocol, HealthStatus.READY), - protoHealthMock(StoreClientProtocol, HealthStatus.READY), - ] + let protocols = @[ + protoHealthMock(RelayProtocol, HealthStatus.NOT_MOUNTED), + protoHealthMock(LightpushClientProtocol, HealthStatus.READY), + protoHealthMock(FilterClientProtocol, HealthStatus.READY), + protoHealthMock(StoreClientProtocol, HealthStatus.READY), + ] var strength = initTable[WakuProtocol, int]() strength[LightpushClientProtocol] = HealthyThreshold strength[FilterClientProtocol] = HealthyThreshold @@ -79,12 +77,11 @@ suite "Health Monitor - health state calculation": check state == ConnectionStatus.Connected test "Disconnected, edge missing store": - let protocols = - @[ - protoHealthMock(LightpushClientProtocol, HealthStatus.READY), - protoHealthMock(FilterClientProtocol, HealthStatus.READY), - protoHealthMock(StoreClientProtocol, HealthStatus.NOT_READY), - ] + let protocols = @[ + protoHealthMock(LightpushClientProtocol, HealthStatus.READY), + protoHealthMock(FilterClientProtocol, HealthStatus.READY), + protoHealthMock(StoreClientProtocol, HealthStatus.NOT_READY), + ] var strength = initTable[WakuProtocol, int]() strength[LightpushClientProtocol] = HealthyThreshold strength[FilterClientProtocol] = HealthyThreshold @@ -94,12 +91,11 @@ suite "Health Monitor - health state calculation": test "PartiallyConnected, edge meets minimum failover requirement": let weakCount = max(1, HealthyThreshold - 1) - let protocols = - @[ - protoHealthMock(LightpushClientProtocol, HealthStatus.READY), - protoHealthMock(FilterClientProtocol, HealthStatus.READY), - protoHealthMock(StoreClientProtocol, HealthStatus.READY), - ] + let protocols = @[ + protoHealthMock(LightpushClientProtocol, HealthStatus.READY), + protoHealthMock(FilterClientProtocol, HealthStatus.READY), + protoHealthMock(StoreClientProtocol, HealthStatus.READY), + ] var strength = initTable[WakuProtocol, int]() strength[LightpushClientProtocol] = weakCount strength[FilterClientProtocol] = weakCount @@ -108,11 +104,10 @@ suite "Health Monitor - health state calculation": check state == ConnectionStatus.PartiallyConnected test "Connected, robust relay ignores store server": - let protocols = - @[ - protoHealthMock(RelayProtocol, HealthStatus.READY), - protoHealthMock(StoreProtocol, HealthStatus.READY), - ] + let protocols = @[ + protoHealthMock(RelayProtocol, HealthStatus.READY), + protoHealthMock(StoreProtocol, HealthStatus.READY), + ] var strength = initTable[WakuProtocol, int]() strength[RelayProtocol] = MockDLow strength[StoreProtocol] = 0 @@ -120,12 +115,11 @@ suite "Health Monitor - health state calculation": check state == ConnectionStatus.Connected test "Connected, robust relay ignores store client": - let protocols = - @[ - protoHealthMock(RelayProtocol, HealthStatus.READY), - protoHealthMock(StoreProtocol, HealthStatus.READY), - protoHealthMock(StoreClientProtocol, HealthStatus.NOT_READY), - ] + let protocols = @[ + protoHealthMock(RelayProtocol, HealthStatus.READY), + protoHealthMock(StoreProtocol, HealthStatus.READY), + protoHealthMock(StoreClientProtocol, HealthStatus.NOT_READY), + ] var strength = initTable[WakuProtocol, int]() strength[RelayProtocol] = MockDLow strength[StoreProtocol] = 0 diff --git a/tests/node/test_wakunode_legacy_store.nim b/tests/node/test_wakunode_legacy_store.nim index bf8003743..e9b0c9170 100644 --- a/tests/node/test_wakunode_legacy_store.nim +++ b/tests/node/test_wakunode_legacy_store.nim @@ -37,19 +37,18 @@ suite "Waku Store - End to End - Sorted Archive": contentTopicSeq = @[contentTopic] let timeOrigin = now() - archiveMessages = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] + archiveMessages = @[ + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + ] historyQuery = HistoryQuery( pubsubTopic: some(pubsubTopic), @@ -500,19 +499,18 @@ suite "Waku Store - End to End - Unsorted Archive": ) let timeOrigin = now() - unsortedArchiveMessages = - @[ # SortIndex (by timestamp and digest) - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), # 1 - fakeWakuMessage(@[byte 03], ts = ts(00, timeOrigin)), # 2 - fakeWakuMessage(@[byte 08], ts = ts(00, timeOrigin)), # 0 - fakeWakuMessage(@[byte 07], ts = ts(10, timeOrigin)), # 4 - fakeWakuMessage(@[byte 02], ts = ts(10, timeOrigin)), # 3 - fakeWakuMessage(@[byte 09], ts = ts(10, timeOrigin)), # 5 - fakeWakuMessage(@[byte 06], ts = ts(20, timeOrigin)), # 6 - fakeWakuMessage(@[byte 01], ts = ts(20, timeOrigin)), # 9 - fakeWakuMessage(@[byte 04], ts = ts(20, timeOrigin)), # 7 - fakeWakuMessage(@[byte 05], ts = ts(20, timeOrigin)), # 8 - ] + unsortedArchiveMessages = @[ # SortIndex (by timestamp and digest) + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), # 1 + fakeWakuMessage(@[byte 03], ts = ts(00, timeOrigin)), # 2 + fakeWakuMessage(@[byte 08], ts = ts(00, timeOrigin)), # 0 + fakeWakuMessage(@[byte 07], ts = ts(10, timeOrigin)), # 4 + fakeWakuMessage(@[byte 02], ts = ts(10, timeOrigin)), # 3 + fakeWakuMessage(@[byte 09], ts = ts(10, timeOrigin)), # 5 + fakeWakuMessage(@[byte 06], ts = ts(20, timeOrigin)), # 6 + fakeWakuMessage(@[byte 01], ts = ts(20, timeOrigin)), # 9 + fakeWakuMessage(@[byte 04], ts = ts(20, timeOrigin)), # 7 + fakeWakuMessage(@[byte 05], ts = ts(20, timeOrigin)), # 8 + ] let serverKey = generateSecp256k1Key() @@ -654,21 +652,20 @@ suite "Waku Store - End to End - Archive with Multiple Topics": originTs = proc(offset = 0): Timestamp {.gcsafe, raises: [].} = ts(offset, timeOrigin) - archiveMessages = - @[ - fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic), - fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic), - fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic), - fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC), - fakeWakuMessage( - @[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials - ), - ] + archiveMessages = @[ + fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic), + fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic), + fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic), + fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC), + fakeWakuMessage( + @[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials + ), + ] let serverKey = generateSecp256k1Key() @@ -910,12 +907,11 @@ suite "Waku Store - End to End - Archive with Multiple Topics": xasyncTest "Only ephemeral Messages:": # Given an archive with only ephemeral messages let - ephemeralMessages = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), - fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), - fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), - ] + ephemeralMessages = @[ + fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), + fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), + fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), + ] ephemeralArchiveDriver = newSqliteArchiveDriver().put(pubsubTopic, ephemeralMessages) @@ -946,18 +942,16 @@ suite "Waku Store - End to End - Archive with Multiple Topics": xasyncTest "Mixed messages": # Given an archive with both ephemeral and non-ephemeral messages let - ephemeralMessages = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), - fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), - fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), - ] - nonEphemeralMessages = - @[ - fakeWakuMessage(@[byte 03], ts = ts(30), ephemeral = false), - fakeWakuMessage(@[byte 04], ts = ts(40), ephemeral = false), - fakeWakuMessage(@[byte 05], ts = ts(50), ephemeral = false), - ] + ephemeralMessages = @[ + fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), + fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), + fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), + ] + nonEphemeralMessages = @[ + fakeWakuMessage(@[byte 03], ts = ts(30), ephemeral = false), + fakeWakuMessage(@[byte 04], ts = ts(40), ephemeral = false), + fakeWakuMessage(@[byte 05], ts = ts(50), ephemeral = false), + ] mixedArchiveDriver = newSqliteArchiveDriver() .put(pubsubTopic, ephemeralMessages) .put(pubsubTopic, nonEphemeralMessages) diff --git a/tests/node/test_wakunode_relay_rln.nim b/tests/node/test_wakunode_relay_rln.nim index 9c5c928f0..c8ca9b43d 100644 --- a/tests/node/test_wakunode_relay_rln.nim +++ b/tests/node/test_wakunode_relay_rln.nim @@ -283,31 +283,31 @@ suite "Waku RlnRelay - End to End - Static": doAssert( client.wakuRlnRelay - .appendRLNProof( - message1b, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 0) - ) - .isOk() + .appendRLNProof( + message1b, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 0) + ) + .isOk() ) doAssert( client.wakuRlnRelay - .appendRLNProof( - message1kib, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 1) - ) - .isOk() + .appendRLNProof( + message1kib, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 1) + ) + .isOk() ) doAssert( client.wakuRlnRelay - .appendRLNProof( - message150kib, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 2) - ) - .isOk() + .appendRLNProof( + message150kib, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 2) + ) + .isOk() ) doAssert( client.wakuRlnRelay - .appendRLNProof( - message151kibPlus, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 3) - ) - .isOk() + .appendRLNProof( + message151kibPlus, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 3) + ) + .isOk() ) # When sending the 1B message @@ -372,10 +372,10 @@ suite "Waku RlnRelay - End to End - Static": doAssert( client.wakuRlnRelay - .appendRLNProof( - message151kibPlus, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 3) - ) - .isOk() + .appendRLNProof( + message151kibPlus, epoch + float64(client.wakuRlnRelay.rlnEpochSizeSec * 3) + ) + .isOk() ) # When sending the 150KiB plus message @@ -496,11 +496,11 @@ suite "Waku RlnRelay - End to End - OnChain": # However, it doesn't reduce the retries against the blockchain that the mounting rln process attempts (until it accepts failure). # Note: These retries might be an unintended library issue. discard await server - .setupRelayWithOnChainRln(@[pubsubTopic], wakuRlnConfig1) - .withTimeout(FUTURE_TIMEOUT) + .setupRelayWithOnChainRln(@[pubsubTopic], wakuRlnConfig1) + .withTimeout(FUTURE_TIMEOUT) discard await client - .setupRelayWithOnChainRln(@[pubsubTopic], wakuRlnConfig2) - .withTimeout(FUTURE_TIMEOUT) + .setupRelayWithOnChainRln(@[pubsubTopic], wakuRlnConfig2) + .withTimeout(FUTURE_TIMEOUT) check: (await serverErrorFuture.waitForResult()).get() == diff --git a/tests/node/test_wakunode_sharding.nim b/tests/node/test_wakunode_sharding.nim index 261077e36..236510fe1 100644 --- a/tests/node/test_wakunode_sharding.nim +++ b/tests/node/test_wakunode_sharding.nim @@ -434,18 +434,16 @@ suite "Sharding": contentTopicShort = "/toychat/2/huilong/proto" contentTopicFull = "/0/toychat/2/huilong/proto" pubsubTopic = "/waku/2/rs/0/58355" - archiveMessages1 = - @[ - fakeWakuMessage( - @[byte 00], ts = ts(00, timeOrigin), contentTopic = contentTopicShort - ) - ] - archiveMessages2 = - @[ - fakeWakuMessage( - @[byte 01], ts = ts(10, timeOrigin), contentTopic = contentTopicFull - ) - ] + archiveMessages1 = @[ + fakeWakuMessage( + @[byte 00], ts = ts(00, timeOrigin), contentTopic = contentTopicShort + ) + ] + archiveMessages2 = @[ + fakeWakuMessage( + @[byte 01], ts = ts(10, timeOrigin), contentTopic = contentTopicFull + ) + ] archiveDriver = newArchiveDriverWithMessages(pubsubTopic, archiveMessages1) discard archiveDriver.put(pubsubTopic, archiveMessages2) let mountArchiveResult = server.mountArchive(archiveDriver) @@ -597,18 +595,16 @@ suite "Sharding": contentTopic2 = "/0/toychat2/2/huilong/proto" pubsubTopic2 = "/waku/2/rs/0/23286" # Automatically generated from the contentTopic above - archiveMessages1 = - @[ - fakeWakuMessage( - @[byte 00], ts = ts(00, timeOrigin), contentTopic = contentTopic1 - ) - ] - archiveMessages2 = - @[ - fakeWakuMessage( - @[byte 01], ts = ts(10, timeOrigin), contentTopic = contentTopic2 - ) - ] + archiveMessages1 = @[ + fakeWakuMessage( + @[byte 00], ts = ts(00, timeOrigin), contentTopic = contentTopic1 + ) + ] + archiveMessages2 = @[ + fakeWakuMessage( + @[byte 01], ts = ts(10, timeOrigin), contentTopic = contentTopic2 + ) + ] archiveDriver = newArchiveDriverWithMessages(pubsubTopic1, archiveMessages1) discard archiveDriver.put(pubsubTopic2, archiveMessages2) let mountArchiveResult = server.mountArchive(archiveDriver) diff --git a/tests/node/test_wakunode_store.nim b/tests/node/test_wakunode_store.nim index 9f312afd5..01deb2903 100644 --- a/tests/node/test_wakunode_store.nim +++ b/tests/node/test_wakunode_store.nim @@ -38,19 +38,18 @@ suite "Waku Store - End to End - Sorted Archive": contentTopicSeq = @[contentTopic] let timeOrigin = now() - let messages = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] + let messages = @[ + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + ] archiveMessages = messages.mapIt( WakuMessageKeyValue( messageHash: computeMessageHash(pubsubTopic, it), @@ -542,19 +541,18 @@ suite "Waku Store - End to End - Unsorted Archive": ) let timeOrigin = now() - let messages = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(20, timeOrigin)), - ] + let messages = @[ + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(20, timeOrigin)), + ] unsortedArchiveMessages = messages.mapIt( WakuMessageKeyValue( messageHash: computeMessageHash(pubsubTopic, it), @@ -759,19 +757,19 @@ suite "Waku Store - End to End - Unsorted Archive without provided Timestamp": paginationLimit: some(uint64(5)), ) - let messages = - @[ # Not providing explicit timestamp means it will be set in "arrive" order - fakeWakuMessage(@[byte 09]), - fakeWakuMessage(@[byte 07]), - fakeWakuMessage(@[byte 05]), - fakeWakuMessage(@[byte 03]), - fakeWakuMessage(@[byte 01]), - fakeWakuMessage(@[byte 00]), - fakeWakuMessage(@[byte 02]), - fakeWakuMessage(@[byte 04]), - fakeWakuMessage(@[byte 06]), - fakeWakuMessage(@[byte 08]), - ] + let messages = @[ + # Not providing explicit timestamp means it will be set in "arrive" order + fakeWakuMessage(@[byte 09]), + fakeWakuMessage(@[byte 07]), + fakeWakuMessage(@[byte 05]), + fakeWakuMessage(@[byte 03]), + fakeWakuMessage(@[byte 01]), + fakeWakuMessage(@[byte 00]), + fakeWakuMessage(@[byte 02]), + fakeWakuMessage(@[byte 04]), + fakeWakuMessage(@[byte 06]), + fakeWakuMessage(@[byte 08]), + ] unsortedArchiveMessages = messages.mapIt( WakuMessageKeyValue( messageHash: computeMessageHash(pubsubTopic, it), @@ -900,21 +898,20 @@ suite "Waku Store - End to End - Archive with Multiple Topics": originTs = proc(offset = 0): Timestamp {.gcsafe, raises: [].} = ts(offset, timeOrigin) - let messages = - @[ - fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic), - fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic), - fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic), - fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC), - fakeWakuMessage( - @[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials - ), - ] + let messages = @[ + fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic), + fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic), + fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC), + fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic), + fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB), + fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC), + fakeWakuMessage( + @[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials + ), + ] archiveMessages = messages.mapIt( WakuMessageKeyValue( @@ -1172,12 +1169,11 @@ suite "Waku Store - End to End - Archive with Multiple Topics": xasyncTest "Only ephemeral Messages:": # Given an archive with only ephemeral messages let - ephemeralMessages = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), - fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), - fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), - ] + ephemeralMessages = @[ + fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), + fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), + fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), + ] ephemeralArchiveDriver = newSqliteArchiveDriver().put(pubsubTopic, ephemeralMessages) @@ -1207,18 +1203,16 @@ suite "Waku Store - End to End - Archive with Multiple Topics": xasyncTest "Mixed messages": # Given an archive with both ephemeral and non-ephemeral messages let - ephemeralMessages = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), - fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), - fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), - ] - nonEphemeralMessages = - @[ - fakeWakuMessage(@[byte 03], ts = ts(30), ephemeral = false), - fakeWakuMessage(@[byte 04], ts = ts(40), ephemeral = false), - fakeWakuMessage(@[byte 05], ts = ts(50), ephemeral = false), - ] + ephemeralMessages = @[ + fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), + fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), + fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), + ] + nonEphemeralMessages = @[ + fakeWakuMessage(@[byte 03], ts = ts(30), ephemeral = false), + fakeWakuMessage(@[byte 04], ts = ts(40), ephemeral = false), + fakeWakuMessage(@[byte 05], ts = ts(50), ephemeral = false), + ] mixedArchiveDriver = newSqliteArchiveDriver() .put(pubsubTopic, ephemeralMessages) .put(pubsubTopic, nonEphemeralMessages) diff --git a/tests/resources/payloads.nim b/tests/resources/payloads.nim index 723bf788c..0fc1eaebb 100644 --- a/tests/resources/payloads.nim +++ b/tests/resources/payloads.nim @@ -8,8 +8,7 @@ const EMOJI* = "😀 😃 😄 😁 😆 😅 🤣 😂 🙂 🙃 😉 😊 😇 🥰 😍 🤩 😘 😗 😚 😙" CODE* = "def main():\n\tprint('Hello, world!')" - QUERY* = - """ + QUERY* = """ SELECT u.id, u.name, @@ -30,8 +29,7 @@ const u.id = 1 """ TEXT_SMALL* = "Lorem ipsum dolor sit amet, consectetur adipiscing elit." - TEXT_LARGE* = - """ + TEXT_LARGE* = """ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras gravida vulputate semper. Proin eleifend varius cursus. Morbi lacinia posuere quam sit amet pretium. Sed non metus fermentum, venenatis nisl id, vestibulum eros. Quisque non lorem sit amet lectus faucibus elementum eu diff --git a/tests/test_peer_manager.nim b/tests/test_peer_manager.nim index c96f21b6e..f78c3831f 100644 --- a/tests/test_peer_manager.nim +++ b/tests/test_peer_manager.nim @@ -1216,30 +1216,29 @@ procSuite "Peer Manager": shardId1 = 1.uint16 # Create 3 nodes with different shards - let nodes = - @[ - newTestWakuNode( - generateSecp256k1Key(), - parseIpAddress("0.0.0.0"), - Port(0), - clusterId = clusterId, - subscribeShards = @[shardId0], - ), - newTestWakuNode( - generateSecp256k1Key(), - parseIpAddress("0.0.0.0"), - Port(0), - clusterId = clusterId, - subscribeShards = @[shardId1], - ), - newTestWakuNode( - generateSecp256k1Key(), - parseIpAddress("0.0.0.0"), - Port(0), - clusterId = clusterId, - subscribeShards = @[shardId0], - ), - ] + let nodes = @[ + newTestWakuNode( + generateSecp256k1Key(), + parseIpAddress("0.0.0.0"), + Port(0), + clusterId = clusterId, + subscribeShards = @[shardId0], + ), + newTestWakuNode( + generateSecp256k1Key(), + parseIpAddress("0.0.0.0"), + Port(0), + clusterId = clusterId, + subscribeShards = @[shardId1], + ), + newTestWakuNode( + generateSecp256k1Key(), + parseIpAddress("0.0.0.0"), + Port(0), + clusterId = clusterId, + subscribeShards = @[shardId0], + ), + ] await allFutures(nodes.mapIt(it.start())) for node in nodes: @@ -1364,13 +1363,12 @@ procSuite "Peer Manager": node.peerManager.switch.peerStore[ProtoBook][peerInfo.peerId] = @[WakuRelayCodec] ## When: selectPeer is called with malformed pubsub topic - let invalidTopics = - @[ - some(PubsubTopic("invalid-topic")), - some(PubsubTopic("/waku/2/invalid")), - some(PubsubTopic("/waku/2/rs/abc/0")), # non-numeric cluster - some(PubsubTopic("")), # empty topic - ] + let invalidTopics = @[ + some(PubsubTopic("invalid-topic")), + some(PubsubTopic("/waku/2/invalid")), + some(PubsubTopic("/waku/2/rs/abc/0")), # non-numeric cluster + some(PubsubTopic("")), # empty topic + ] ## Then: Returns none(RemotePeerInfo) without crashing for invalidTopic in invalidTopics: diff --git a/tests/test_waku.nim b/tests/test_waku.nim index dabd65af7..cf5675716 100644 --- a/tests/test_waku.nim +++ b/tests/test_waku.nim @@ -35,14 +35,12 @@ suite "Waku API - Create node": nodeConf.rest = false nodeConf.numShardsInNetwork = 16 nodeConf.maxMessageSize = "1024 KiB" - nodeConf.entryNodes = - @[ - "enr:-QESuEC1p_s3xJzAC_XlOuuNrhVUETmfhbm1wxRGis0f7DlqGSw2FM-p2Vn7gmfkTTnAe8Ys2cgGBN8ufJnvzKQFZqFMBgmlkgnY0iXNlY3AyNTZrMaEDS8-D878DrdbNwcuY-3p1qdDp5MOoCurhdsNPJTXZ3c5g3RjcIJ2X4N1ZHCCd2g" - ] - nodeConf.staticnodes = - @[ - "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc" - ] + nodeConf.entryNodes = @[ + "enr:-QESuEC1p_s3xJzAC_XlOuuNrhVUETmfhbm1wxRGis0f7DlqGSw2FM-p2Vn7gmfkTTnAe8Ys2cgGBN8ufJnvzKQFZqFMBgmlkgnY0iXNlY3AyNTZrMaEDS8-D878DrdbNwcuY-3p1qdDp5MOoCurhdsNPJTXZ3c5g3RjcIJ2X4N1ZHCCd2g" + ] + nodeConf.staticnodes = @[ + "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc" + ] ## When let node = (await createNode(nodeConf)).valueOr: @@ -67,11 +65,10 @@ suite "Waku API - Create node": nodeConf.mode = Core nodeConf.clusterId = 42'u16 nodeConf.rest = false - nodeConf.entryNodes = - @[ - "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", - "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", - ] + nodeConf.entryNodes = @[ + "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im", + "/ip4/127.0.0.1/tcp/60000/p2p/16Uuu2HBmAcHvhLqQKwSSbX6BG5JLWUDRcaLVrehUVqpw7fz1hbYc", + ] ## When let node = (await createNode(nodeConf)).valueOr: diff --git a/tests/test_waku_keystore_keyfile.nim b/tests/test_waku_keystore_keyfile.nim index 5f4c74591..afdb7e44b 100644 --- a/tests/test_waku_keystore_keyfile.nim +++ b/tests/test_waku_keystore_keyfile.nim @@ -307,30 +307,29 @@ suite "KeyFile test suite (adapted from nim-eth keyfile tests)": # but the last byte of mac is changed to 00. # While ciphertext is the correct encryption of priv under password, # mac verfication should fail and nothing will be decrypted - let keyfileWrongMac = - %*{ - "keyfile": { - "crypto": { - "cipher": "aes-128-ctr", - "cipherparams": {"iv": "6087dab2f9fdbbfaddc31a909735c1e6"}, - "ciphertext": - "5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46", - "kdf": "pbkdf2", - "kdfparams": { - "c": 262144, - "dklen": 32, - "prf": "hmac-sha256", - "salt": "ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd", - }, - "mac": "517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e900", + let keyfileWrongMac = %*{ + "keyfile": { + "crypto": { + "cipher": "aes-128-ctr", + "cipherparams": {"iv": "6087dab2f9fdbbfaddc31a909735c1e6"}, + "ciphertext": + "5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46", + "kdf": "pbkdf2", + "kdfparams": { + "c": 262144, + "dklen": 32, + "prf": "hmac-sha256", + "salt": "ae3cd4e7013836a3df6bd7241b12db061dbe2c6785853cce422d148a624ce0bd", }, - "id": "3198bc9c-6672-5ab3-d995-4942343ae5b6", - "version": 3, + "mac": "517ead924a9d0dc3124507e3393d175ce3ff7c1e96529c6c555ce9e51205e900", }, - "name": "test1", - "password": "testpassword", - "priv": "7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d", - } + "id": "3198bc9c-6672-5ab3-d995-4942343ae5b6", + "version": 3, + }, + "name": "test1", + "password": "testpassword", + "priv": "7a28b5ba57c53603b0b07b56bba752f7784bf506fa95edc395f5cf6c7514fe9d", + } # Decryption with correct password let expectedSecret = decodeHex(keyfileWrongMac.getOrDefault("priv").getStr()) diff --git a/tests/test_waku_noise.nim b/tests/test_waku_noise.nim index 980e752f5..6566f9eed 100644 --- a/tests/test_waku_noise.nim +++ b/tests/test_waku_noise.nim @@ -669,11 +669,10 @@ procSuite "Waku Noise": # <- s # ... # So we define accordingly the sequence of the pre-message public keys - let preMessagePKs: seq[NoisePublicKey] = - @[ - toNoisePublicKey(getPublicKey(aliceStaticKey)), - toNoisePublicKey(getPublicKey(bobStaticKey)), - ] + let preMessagePKs: seq[NoisePublicKey] = @[ + toNoisePublicKey(getPublicKey(aliceStaticKey)), + toNoisePublicKey(getPublicKey(bobStaticKey)), + ] var aliceHS = initialize( hsPattern = hsPattern, diff --git a/tests/test_waku_rendezvous.nim b/tests/test_waku_rendezvous.nim index 07113ca4a..88845dc25 100644 --- a/tests/test_waku_rendezvous.nim +++ b/tests/test_waku_rendezvous.nim @@ -117,11 +117,10 @@ procSuite "Waku Rendezvous": ## Given: A light client node with no relay protocol let clusterId = 10.uint16 - configuredShards = - @[ - RelayShard(clusterId: clusterId, shardId: 0), - RelayShard(clusterId: clusterId, shardId: 1), - ] + configuredShards = @[ + RelayShard(clusterId: clusterId, shardId: 0), + RelayShard(clusterId: clusterId, shardId: 1), + ] let lightClient = newTestWakuNode( generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0), clusterId = clusterId diff --git a/tests/test_waku_switch.nim b/tests/test_waku_switch.nim index 3e6fd08eb..9f11a41a1 100644 --- a/tests/test_waku_switch.nim +++ b/tests/test_waku_switch.nim @@ -12,14 +12,14 @@ import waku/node/waku_switch, ./testlib/common, ./testlib/wakucore proc newCircuitRelayClientSwitch(relayClient: RelayClient): Switch = SwitchBuilder - .new() - .withRng(rng()) - .withAddresses(@[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()]) - .withTcpTransport() - .withMplex() - .withNoise() - .withCircuitRelay(relayClient) - .build() + .new() + .withRng(rng()) + .withAddresses(@[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()]) + .withTcpTransport() + .withMplex() + .withNoise() + .withCircuitRelay(relayClient) + .build() suite "Waku Switch": asyncTest "Waku Switch works with AutoNat": diff --git a/tests/waku_archive/test_driver_postgres_query.nim b/tests/waku_archive/test_driver_postgres_query.nim index 8bbdc52c0..240ac28dd 100644 --- a/tests/waku_archive/test_driver_postgres_query.nim +++ b/tests/waku_archive/test_driver_postgres_query.nim @@ -49,17 +49,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -85,17 +84,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -124,29 +122,28 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), - fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" - ), - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" - ), - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" - ), - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" - ), - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" - ), - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" - ), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), + fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" + ), + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" + ), + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" + ), + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" + ), + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" + ), + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" + ), + ] var messages = expected shuffle(messages) @@ -175,17 +172,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -216,17 +212,16 @@ suite "Postgres driver - queries": const contentTopic2 = "test-content-topic-2" const contentTopic3 = "test-content-topic-3" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -275,14 +270,13 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] var messages = expected shuffle(messages) @@ -338,35 +332,34 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" const pubsubTopic = "test-pubsub-topic" - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -395,35 +388,34 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" const pubsubTopic = "test-pubsub-topic" - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -450,35 +442,34 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" const pubsubTopic = "test-pubsub-topic" - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -509,18 +500,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -551,18 +540,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -593,17 +580,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - var messages = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + var messages = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] shuffle(messages) info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) @@ -640,18 +626,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -685,18 +669,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -732,59 +714,42 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -819,59 +784,42 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -903,19 +851,18 @@ suite "Postgres driver - queries": asyncTest "only hashes - descending order": ## Given let timeOrigin = now() - var expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + var expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -942,17 +889,16 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -982,17 +928,16 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1023,61 +968,44 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1109,18 +1037,17 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1153,17 +1080,16 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1195,20 +1121,19 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1240,21 +1165,20 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1289,21 +1213,20 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1339,52 +1262,39 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1421,51 +1331,38 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1502,52 +1399,39 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1585,52 +1469,39 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1667,16 +1538,15 @@ suite "Postgres driver - queries": let timeOrigin = now() let oldestTime = ts(00, timeOrigin) let newestTime = ts(100, timeOrigin) - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = oldestTime), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = newestTime), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = oldestTime), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = newestTime), + ] var messages = expected shuffle(messages) @@ -1714,16 +1584,15 @@ suite "Postgres driver - queries": let timeOrigin = now() let targetTime = ts(40, timeOrigin) - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = targetTime), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = targetTime), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1751,16 +1620,15 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1793,12 +1661,11 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ] var messages = expected var hashes = newSeq[WakuMessageHash](0) diff --git a/tests/waku_archive/test_driver_queue_query.nim b/tests/waku_archive/test_driver_queue_query.nim index 11b39a3f8..b93e79bf6 100644 --- a/tests/waku_archive/test_driver_queue_query.nim +++ b/tests/waku_archive/test_driver_queue_query.nim @@ -3,13 +3,9 @@ import std/[options, sequtils, random, algorithm], testutils/unittests, chronos, chronicles import - waku/ - [ - waku_archive, - waku_archive/driver/queue_driver, - waku_core, - waku_core/message/digest, - ], + waku/[ + waku_archive, waku_archive/driver/queue_driver, waku_core, waku_core/message/digest + ], ../testlib/common, ../testlib/wakucore @@ -29,17 +25,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -71,17 +66,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -115,17 +109,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -161,17 +154,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -207,14 +199,13 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] var messages = expected shuffle(messages) @@ -281,35 +272,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newTestSqliteDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -345,35 +335,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newTestSqliteDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -407,35 +396,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newTestSqliteDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -474,18 +462,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -521,18 +507,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -568,17 +552,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - var messages = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + var messages = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] shuffle(messages) info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) @@ -618,18 +601,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -668,18 +649,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -720,59 +699,42 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -814,59 +776,42 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -908,17 +853,16 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -953,17 +897,16 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -999,61 +942,44 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1092,18 +1018,17 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1141,17 +1066,16 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1188,20 +1112,19 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1238,21 +1161,20 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1292,21 +1214,20 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1347,52 +1268,39 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1436,51 +1344,38 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1524,52 +1419,39 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1614,52 +1496,39 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) diff --git a/tests/waku_archive/test_driver_sqlite_query.nim b/tests/waku_archive/test_driver_sqlite_query.nim index 9812a50f3..6ae7c5b9d 100644 --- a/tests/waku_archive/test_driver_sqlite_query.nim +++ b/tests/waku_archive/test_driver_sqlite_query.nim @@ -22,17 +22,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -65,17 +64,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -110,29 +108,28 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), - fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" - ), - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" - ), - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" - ), - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" - ), - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" - ), - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" - ), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), + fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" + ), + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" + ), + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" + ), + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" + ), + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" + ), + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" + ), + ] var messages = expected shuffle(messages) @@ -167,17 +164,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -214,17 +210,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -261,14 +256,13 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] var messages = expected shuffle(messages) @@ -337,35 +331,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -400,35 +393,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -461,35 +453,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -527,18 +518,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -575,18 +564,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -623,17 +610,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - var messages = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + var messages = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] shuffle(messages) info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) @@ -674,18 +660,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -725,18 +709,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -778,59 +760,42 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -871,59 +836,42 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -964,17 +912,16 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1010,17 +957,16 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1057,61 +1003,44 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1149,18 +1078,17 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1199,17 +1127,16 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1247,20 +1174,19 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1298,21 +1224,20 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1353,21 +1278,20 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1409,52 +1333,39 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1497,51 +1408,38 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1584,52 +1482,39 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1673,52 +1558,39 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) diff --git a/tests/waku_archive/test_retention_policy.nim b/tests/waku_archive/test_retention_policy.nim index ea86e1d69..394d5711d 100644 --- a/tests/waku_archive/test_retention_policy.nim +++ b/tests/waku_archive/test_retention_policy.nim @@ -121,16 +121,15 @@ suite "Waku Archive - Retention policy": retentionPolicy: RetentionPolicy = CapacityRetentionPolicy.new(capacity = capacity) - let messages = - @[ - fakeWakuMessage(contentTopic = DefaultContentTopic, ts = ts(0)), - fakeWakuMessage(contentTopic = DefaultContentTopic, ts = ts(1)), - fakeWakuMessage(contentTopic = contentTopic, ts = ts(2)), - fakeWakuMessage(contentTopic = contentTopic, ts = ts(3)), - fakeWakuMessage(contentTopic = contentTopic, ts = ts(4)), - fakeWakuMessage(contentTopic = contentTopic, ts = ts(5)), - fakeWakuMessage(contentTopic = contentTopic, ts = ts(6)), - ] + let messages = @[ + fakeWakuMessage(contentTopic = DefaultContentTopic, ts = ts(0)), + fakeWakuMessage(contentTopic = DefaultContentTopic, ts = ts(1)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(2)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(3)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(4)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(5)), + fakeWakuMessage(contentTopic = contentTopic, ts = ts(6)), + ] ## When for msg in messages: diff --git a/tests/waku_archive/test_waku_archive.nim b/tests/waku_archive/test_waku_archive.nim index 802473d64..9b235dd57 100644 --- a/tests/waku_archive/test_waku_archive.nim +++ b/tests/waku_archive/test_waku_archive.nim @@ -36,14 +36,13 @@ suite "Waku Archive - message handling": let archive = newWakuArchive(driver) ## Given - let msgList = - @[ - fakeWakuMessage(ephemeral = false, payload = "1"), - fakeWakuMessage(ephemeral = true, payload = "2"), - fakeWakuMessage(ephemeral = true, payload = "3"), - fakeWakuMessage(ephemeral = true, payload = "4"), - fakeWakuMessage(ephemeral = false, payload = "5"), - ] + let msgList = @[ + fakeWakuMessage(ephemeral = false, payload = "1"), + fakeWakuMessage(ephemeral = true, payload = "2"), + fakeWakuMessage(ephemeral = true, payload = "3"), + fakeWakuMessage(ephemeral = true, payload = "4"), + fakeWakuMessage(ephemeral = false, payload = "5"), + ] ## When for msg in msgList: @@ -127,39 +126,38 @@ suite "Waku Archive - message handling": procSuite "Waku Archive - find messages": ## Fixtures let timeOrigin = now() - let msgListA = - @[ - fakeWakuMessage( - @[byte 00], contentTopic = ContentTopic("2"), ts = ts(00, timeOrigin) - ), - fakeWakuMessage( - @[byte 01], contentTopic = ContentTopic("1"), ts = ts(10, timeOrigin) - ), - fakeWakuMessage( - @[byte 02], contentTopic = ContentTopic("2"), ts = ts(20, timeOrigin) - ), - fakeWakuMessage( - @[byte 03], contentTopic = ContentTopic("1"), ts = ts(30, timeOrigin) - ), - fakeWakuMessage( - @[byte 04], contentTopic = ContentTopic("2"), ts = ts(40, timeOrigin) - ), - fakeWakuMessage( - @[byte 05], contentTopic = ContentTopic("1"), ts = ts(50, timeOrigin) - ), - fakeWakuMessage( - @[byte 06], contentTopic = ContentTopic("2"), ts = ts(60, timeOrigin) - ), - fakeWakuMessage( - @[byte 07], contentTopic = ContentTopic("1"), ts = ts(70, timeOrigin) - ), - fakeWakuMessage( - @[byte 08], contentTopic = ContentTopic("2"), ts = ts(80, timeOrigin) - ), - fakeWakuMessage( - @[byte 09], contentTopic = ContentTopic("1"), ts = ts(90, timeOrigin) - ), - ] + let msgListA = @[ + fakeWakuMessage( + @[byte 00], contentTopic = ContentTopic("2"), ts = ts(00, timeOrigin) + ), + fakeWakuMessage( + @[byte 01], contentTopic = ContentTopic("1"), ts = ts(10, timeOrigin) + ), + fakeWakuMessage( + @[byte 02], contentTopic = ContentTopic("2"), ts = ts(20, timeOrigin) + ), + fakeWakuMessage( + @[byte 03], contentTopic = ContentTopic("1"), ts = ts(30, timeOrigin) + ), + fakeWakuMessage( + @[byte 04], contentTopic = ContentTopic("2"), ts = ts(40, timeOrigin) + ), + fakeWakuMessage( + @[byte 05], contentTopic = ContentTopic("1"), ts = ts(50, timeOrigin) + ), + fakeWakuMessage( + @[byte 06], contentTopic = ContentTopic("2"), ts = ts(60, timeOrigin) + ), + fakeWakuMessage( + @[byte 07], contentTopic = ContentTopic("1"), ts = ts(70, timeOrigin) + ), + fakeWakuMessage( + @[byte 08], contentTopic = ContentTopic("2"), ts = ts(80, timeOrigin) + ), + fakeWakuMessage( + @[byte 09], contentTopic = ContentTopic("1"), ts = ts(90, timeOrigin) + ), + ] let archiveA = block: let @@ -446,19 +444,18 @@ procSuite "Waku Archive - find messages": driver = newSqliteArchiveDriver() archive = newWakuArchive(driver) - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2")), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 5], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 6], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 7], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 8], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2")), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2")), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 5], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 6], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 7], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 8], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2")), + ] for msg in msgList: require ( diff --git a/tests/waku_archive_legacy/test_driver_postgres_query.nim b/tests/waku_archive_legacy/test_driver_postgres_query.nim index e164a63a8..ff513de76 100644 --- a/tests/waku_archive_legacy/test_driver_postgres_query.nim +++ b/tests/waku_archive_legacy/test_driver_postgres_query.nim @@ -76,17 +76,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -117,17 +116,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -160,29 +158,28 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), - fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" - ), - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" - ), - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" - ), - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" - ), - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" - ), - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" - ), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), + fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" + ), + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" + ), + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" + ), + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" + ), + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" + ), + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" + ), + ] var messages = expected shuffle(messages) @@ -215,17 +212,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -260,17 +256,16 @@ suite "Postgres driver - queries": const contentTopic2 = "test-content-topic-2" const contentTopic3 = "test-content-topic-3" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -323,14 +318,13 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] var messages = expected shuffle(messages) @@ -394,35 +388,34 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" const pubsubTopic = "test-pubsub-topic" - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -455,35 +448,34 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" const pubsubTopic = "test-pubsub-topic" - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -514,35 +506,34 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" const pubsubTopic = "test-pubsub-topic" - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -577,18 +568,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -623,18 +612,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -669,17 +656,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - var messages = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + var messages = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] shuffle(messages) info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) @@ -721,18 +707,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -770,18 +754,16 @@ suite "Postgres driver - queries": ## Given const contentTopic = "test-content-topic" - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -821,59 +803,42 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -912,59 +877,42 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1000,19 +948,18 @@ suite "Postgres driver - queries": asyncTest "only hashes - descending order": ## Given let timeOrigin = now() - var expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + var expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1043,17 +990,16 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1087,17 +1033,16 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1132,61 +1077,44 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1222,18 +1150,17 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1270,17 +1197,16 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1316,20 +1242,19 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1365,21 +1290,20 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1418,21 +1342,20 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1472,52 +1395,39 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1558,51 +1468,38 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1643,52 +1540,39 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1730,52 +1614,39 @@ suite "Postgres driver - queries": const pubsubTopic = "test-pubsub-topic" let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1817,16 +1688,15 @@ suite "Postgres driver - queries": let timeOrigin = now() let oldestTime = ts(00, timeOrigin) let newestTime = ts(100, timeOrigin) - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = oldestTime), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = newestTime), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = oldestTime), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = newestTime), + ] var messages = expected shuffle(messages) @@ -1869,16 +1739,15 @@ suite "Postgres driver - queries": let timeOrigin = now() let targetTime = ts(40, timeOrigin) - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = targetTime), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = targetTime), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1911,16 +1780,15 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1957,12 +1825,11 @@ suite "Postgres driver - queries": const contentTopic = "test-content-topic" let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ] var messages = expected var hashes = newSeq[WakuMessageHash](0) diff --git a/tests/waku_archive_legacy/test_driver_queue_query.nim b/tests/waku_archive_legacy/test_driver_queue_query.nim index 6ebe5963a..0726d1931 100644 --- a/tests/waku_archive_legacy/test_driver_queue_query.nim +++ b/tests/waku_archive_legacy/test_driver_queue_query.nim @@ -35,17 +35,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -81,17 +80,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -129,17 +127,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -179,17 +176,16 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -229,14 +225,13 @@ suite "Queue driver - query by content topic": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] var messages = expected shuffle(messages) @@ -311,35 +306,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newTestSqliteDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -377,35 +371,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newTestSqliteDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -441,35 +434,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newTestSqliteDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -510,18 +502,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -561,18 +551,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -612,17 +600,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - var messages = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + var messages = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] shuffle(messages) info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) @@ -667,18 +654,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -721,18 +706,16 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -777,59 +760,42 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -873,59 +839,42 @@ suite "Queue driver - query by cursor": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -969,17 +918,16 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1018,17 +966,16 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1068,61 +1015,44 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1163,18 +1093,17 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1216,17 +1145,16 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1267,20 +1195,19 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1321,21 +1248,20 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1379,21 +1305,20 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1438,52 +1363,39 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1529,51 +1441,38 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1619,52 +1518,39 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1711,52 +1597,39 @@ suite "Queue driver - query by time range": let driver = newTestSqliteDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) diff --git a/tests/waku_archive_legacy/test_driver_sqlite_query.nim b/tests/waku_archive_legacy/test_driver_sqlite_query.nim index 4143decf6..3c3b55232 100644 --- a/tests/waku_archive_legacy/test_driver_sqlite_query.nim +++ b/tests/waku_archive_legacy/test_driver_sqlite_query.nim @@ -24,17 +24,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -71,17 +70,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -120,29 +118,28 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), - fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" - ), - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" - ), - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" - ), - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" - ), - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" - ), - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" - ), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), + fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), + fakeWakuMessage( + @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" + ), + fakeWakuMessage( + @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" + ), + fakeWakuMessage( + @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" + ), + fakeWakuMessage( + @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" + ), + fakeWakuMessage( + @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" + ), + fakeWakuMessage( + @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" + ), + ] var messages = expected shuffle(messages) @@ -181,17 +178,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -232,17 +228,16 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -283,14 +278,13 @@ suite "SQLite driver - query by content topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), + ] var messages = expected shuffle(messages) @@ -367,35 +361,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -434,35 +427,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -499,35 +491,34 @@ suite "SQLite driver - query by pubsub topic": let driver = newSqliteArchiveDriver() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ), + ] var messages = expected shuffle(messages) @@ -569,18 +560,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -621,18 +610,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -673,17 +660,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - var messages = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + var messages = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] shuffle(messages) info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) @@ -729,18 +715,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), + fakeWakuMessage(@[byte 7], ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -784,18 +768,16 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() - let expected = - @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], ts = ts(00)), + fakeWakuMessage(@[byte 1], ts = ts(10)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), + ] var messages = expected shuffle(messages) @@ -841,59 +823,42 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -938,59 +903,42 @@ suite "SQLite driver - query by cursor": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), # << cursor + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1035,17 +983,16 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1085,17 +1032,16 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1136,61 +1082,44 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # start_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1232,18 +1161,17 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + # end_time + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1286,17 +1214,16 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1338,20 +1265,19 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1393,21 +1319,20 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1452,21 +1377,20 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] + let expected = @[ + fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), + # start_time + fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), + # << cursor + fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ] var messages = expected shuffle(messages) @@ -1512,52 +1436,39 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + # start_time + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + # end_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1604,51 +1515,38 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1695,52 +1593,39 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) @@ -1788,52 +1673,39 @@ suite "SQLite driver - query by time range": let driver = newSqliteArchiveDriver() let timeOrigin = now() - let expected = - @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin) - ), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin) - ), - ), - ( - pubsubTopic, - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin) - ), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin) - ), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage( - @[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin) - ), - ), - ] + let expected = @[ + (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), + (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), + # << cursor + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), + ), + # start_time + ( + pubsubTopic, + fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), + ), + ( + pubsubTopic, + fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), + ), + (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), + (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), + ), + # end_time + ( + DefaultPubsubTopic, + fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), + ), + ] var messages = expected shuffle(messages) diff --git a/tests/waku_archive_legacy/test_waku_archive.nim b/tests/waku_archive_legacy/test_waku_archive.nim index e58b2cfc9..f6373608d 100644 --- a/tests/waku_archive_legacy/test_waku_archive.nim +++ b/tests/waku_archive_legacy/test_waku_archive.nim @@ -33,14 +33,13 @@ suite "Waku Archive - message handling": let archive = newWakuArchive(driver) ## Given - let msgList = - @[ - fakeWakuMessage(ephemeral = false, payload = "1"), - fakeWakuMessage(ephemeral = true, payload = "2"), - fakeWakuMessage(ephemeral = true, payload = "3"), - fakeWakuMessage(ephemeral = true, payload = "4"), - fakeWakuMessage(ephemeral = false, payload = "5"), - ] + let msgList = @[ + fakeWakuMessage(ephemeral = false, payload = "1"), + fakeWakuMessage(ephemeral = true, payload = "2"), + fakeWakuMessage(ephemeral = true, payload = "3"), + fakeWakuMessage(ephemeral = true, payload = "4"), + fakeWakuMessage(ephemeral = false, payload = "5"), + ] ## When for msg in msgList: @@ -108,39 +107,38 @@ suite "Waku Archive - message handling": procSuite "Waku Archive - find messages": ## Fixtures let timeOrigin = now() - let msgListA = - @[ - fakeWakuMessage( - @[byte 00], contentTopic = ContentTopic("2"), ts = ts(00, timeOrigin) - ), - fakeWakuMessage( - @[byte 01], contentTopic = ContentTopic("1"), ts = ts(10, timeOrigin) - ), - fakeWakuMessage( - @[byte 02], contentTopic = ContentTopic("2"), ts = ts(20, timeOrigin) - ), - fakeWakuMessage( - @[byte 03], contentTopic = ContentTopic("1"), ts = ts(30, timeOrigin) - ), - fakeWakuMessage( - @[byte 04], contentTopic = ContentTopic("2"), ts = ts(40, timeOrigin) - ), - fakeWakuMessage( - @[byte 05], contentTopic = ContentTopic("1"), ts = ts(50, timeOrigin) - ), - fakeWakuMessage( - @[byte 06], contentTopic = ContentTopic("2"), ts = ts(60, timeOrigin) - ), - fakeWakuMessage( - @[byte 07], contentTopic = ContentTopic("1"), ts = ts(70, timeOrigin) - ), - fakeWakuMessage( - @[byte 08], contentTopic = ContentTopic("2"), ts = ts(80, timeOrigin) - ), - fakeWakuMessage( - @[byte 09], contentTopic = ContentTopic("1"), ts = ts(90, timeOrigin) - ), - ] + let msgListA = @[ + fakeWakuMessage( + @[byte 00], contentTopic = ContentTopic("2"), ts = ts(00, timeOrigin) + ), + fakeWakuMessage( + @[byte 01], contentTopic = ContentTopic("1"), ts = ts(10, timeOrigin) + ), + fakeWakuMessage( + @[byte 02], contentTopic = ContentTopic("2"), ts = ts(20, timeOrigin) + ), + fakeWakuMessage( + @[byte 03], contentTopic = ContentTopic("1"), ts = ts(30, timeOrigin) + ), + fakeWakuMessage( + @[byte 04], contentTopic = ContentTopic("2"), ts = ts(40, timeOrigin) + ), + fakeWakuMessage( + @[byte 05], contentTopic = ContentTopic("1"), ts = ts(50, timeOrigin) + ), + fakeWakuMessage( + @[byte 06], contentTopic = ContentTopic("2"), ts = ts(60, timeOrigin) + ), + fakeWakuMessage( + @[byte 07], contentTopic = ContentTopic("1"), ts = ts(70, timeOrigin) + ), + fakeWakuMessage( + @[byte 08], contentTopic = ContentTopic("2"), ts = ts(80, timeOrigin) + ), + fakeWakuMessage( + @[byte 09], contentTopic = ContentTopic("1"), ts = ts(90, timeOrigin) + ), + ] let archiveA = block: let @@ -433,19 +431,18 @@ procSuite "Waku Archive - find messages": driver = newSqliteArchiveDriver() archive = newWakuArchive(driver) - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2")), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 5], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 6], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 7], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 8], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2")), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2")), + fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 5], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 6], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 7], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 8], contentTopic = DefaultContentTopic), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2")), + ] for msg in msgList: require ( diff --git a/tests/waku_enr/test_sharding.nim b/tests/waku_enr/test_sharding.nim index 0984b7d8d..344436d0e 100644 --- a/tests/waku_enr/test_sharding.nim +++ b/tests/waku_enr/test_sharding.nim @@ -140,14 +140,13 @@ suite "Discovery Mechanisms for Shards": test "Bit Vector Representation": # Given a valid bit vector and its representation let - bitVector: seq[byte] = - @[ - 0, 73, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ] + bitVector: seq[byte] = @[ + 0, 73, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ] clusterId: uint16 = 73 # bitVector's clusterId shardIds: seq[uint16] = @[1u16, 10u16] # bitVector's shardIds diff --git a/tests/waku_rln_relay/test_waku_rln_relay.nim b/tests/waku_rln_relay/test_waku_rln_relay.nim index e41b79608..08d3daedb 100644 --- a/tests/waku_rln_relay/test_waku_rln_relay.nim +++ b/tests/waku_rln_relay/test_waku_rln_relay.nim @@ -79,11 +79,10 @@ suite "Waku rln relay": let rln = rlnInstance.get() # prepare the input - let msg = - @[ - "126f4c026cd731979365f79bd345a46d673c5a3f6f588bdc718e6356d02b6fdc".toBytes(), - "1f0e5db2b69d599166ab16219a97b82b662085c93220382b39f9f911d3b943b1".toBytes(), - ] + let msg = @[ + "126f4c026cd731979365f79bd345a46d673c5a3f6f588bdc718e6356d02b6fdc".toBytes(), + "1f0e5db2b69d599166ab16219a97b82b662085c93220382b39f9f911d3b943b1".toBytes(), + ] let hashRes = poseidon(msg) @@ -348,8 +347,7 @@ suite "Waku rln relay": let idCredentials1 = generateCredentials() (waitFor manager1.register(idCredentials1, UserMessageLimit(20))).isOkOr: - assert false, - "error returned when calling register: " & error + assert false, "error returned when calling register: " & error let index2 = MembershipIndex(6) let rlnConf2 = getWakuRlnConfig(manager = manager, index = index2) @@ -362,8 +360,7 @@ suite "Waku rln relay": let idCredentials2 = generateCredentials() (waitFor manager2.register(idCredentials2, UserMessageLimit(20))).isOkOr: - assert false, - "error returned when calling register: " & error + assert false, "error returned when calling register: " & error # get the current epoch time let epoch = wakuRlnRelay1.getCurrentEpoch() @@ -447,7 +444,7 @@ suite "Waku rln relay": password = password, appInfo = RLNAppInfo, ) - .isOk() + .isOk() let readKeystoreRes = getMembershipCredentials( path = filepath, diff --git a/tests/waku_rln_relay/test_wakunode_rln_relay.nim b/tests/waku_rln_relay/test_wakunode_rln_relay.nim index 79a4d6711..19a47e1aa 100644 --- a/tests/waku_rln_relay/test_wakunode_rln_relay.nim +++ b/tests/waku_rln_relay/test_wakunode_rln_relay.nim @@ -135,8 +135,10 @@ procSuite "WakuNode - RLN relay": WakuMessage(payload: @payload, contentTopic: contentTopic, timestamp: now()) doAssert( node1.wakuRlnRelay - .unsafeAppendRLNProof(message, node1.wakuRlnRelay.getCurrentEpoch(), MessageId(0)) - .isOk() + .unsafeAppendRLNProof( + message, node1.wakuRlnRelay.getCurrentEpoch(), MessageId(0) + ) + .isOk() ) info " Nodes participating in the test", @@ -211,11 +213,10 @@ procSuite "WakuNode - RLN relay": let shards = @[RelayShard(clusterId: 0, shardId: 0), RelayShard(clusterId: 0, shardId: 1)] - let contentTopics = - @[ - ContentTopic("/waku/2/content-topic-a/proto"), - ContentTopic("/waku/2/content-topic-b/proto"), - ] + let contentTopics = @[ + ContentTopic("/waku/2/content-topic-a/proto"), + ContentTopic("/waku/2/content-topic-b/proto"), + ] # connect them together await node1.connectToNodes(@[node2.switch.peerInfo.toRemotePeerInfo()]) diff --git a/tests/waku_rln_relay/utils_onchain.nim b/tests/waku_rln_relay/utils_onchain.nim index 9f1048097..db07d3cd6 100644 --- a/tests/waku_rln_relay/utils_onchain.nim +++ b/tests/waku_rln_relay/utils_onchain.nim @@ -535,22 +535,21 @@ proc runAnvil*( let anvilPath = getAnvilPath() info "Anvil path", anvilPath - var args = - @[ - "--port", - $port, - "--gas-limit", - "30000000", - "--gas-price", - "7", - "--base-fee", - "7", - "--balance", - "10000000000", - "--chain-id", - $chainId, - "--disable-min-priority-fee", - ] + var args = @[ + "--port", + $port, + "--gas-limit", + "30000000", + "--gas-price", + "7", + "--base-fee", + "7", + "--balance", + "10000000000", + "--chain-id", + $chainId, + "--disable-min-priority-fee", + ] # Add state file argument if provided if stateFile.isSome(): diff --git a/tests/waku_store/test_client.nim b/tests/waku_store/test_client.nim index 38b07bdf4..ec462eb56 100644 --- a/tests/waku_store/test_client.nim +++ b/tests/waku_store/test_client.nim @@ -35,24 +35,23 @@ suite "Store Client": hash1 = computeMessageHash(DefaultPubsubTopic, message1) hash2 = computeMessageHash(DefaultPubsubTopic, message2) hash3 = computeMessageHash(DefaultPubsubTopic, message3) - messageSeq = - @[ - WakuMessageKeyValue( - messageHash: hash1, - message: some(message1), - pubsubTopic: some(DefaultPubsubTopic), - ), - WakuMessageKeyValue( - messageHash: hash2, - message: some(message2), - pubsubTopic: some(DefaultPubsubTopic), - ), - WakuMessageKeyValue( - messageHash: hash3, - message: some(message3), - pubsubTopic: some(DefaultPubsubTopic), - ), - ] + messageSeq = @[ + WakuMessageKeyValue( + messageHash: hash1, + message: some(message1), + pubsubTopic: some(DefaultPubsubTopic), + ), + WakuMessageKeyValue( + messageHash: hash2, + message: some(message2), + pubsubTopic: some(DefaultPubsubTopic), + ), + WakuMessageKeyValue( + messageHash: hash3, + message: some(message3), + pubsubTopic: some(DefaultPubsubTopic), + ), + ] handlerFuture = newHistoryFuture() handler = proc(req: StoreQueryRequest): Future[StoreQueryResult] {.async, gcsafe.} = var request = req diff --git a/tests/waku_store/test_resume.nim b/tests/waku_store/test_resume.nim index 93e07ec0e..eb11b8f8e 100644 --- a/tests/waku_store/test_resume.nim +++ b/tests/waku_store/test_resume.nim @@ -50,19 +50,18 @@ suite "Store Resume - End to End": var clientDriver {.threadvar.}: ArchiveDriver asyncSetup: - let messages = - @[ - fakeWakuMessage(@[byte 00]), - fakeWakuMessage(@[byte 01]), - fakeWakuMessage(@[byte 02]), - fakeWakuMessage(@[byte 03]), - fakeWakuMessage(@[byte 04]), - fakeWakuMessage(@[byte 05]), - fakeWakuMessage(@[byte 06]), - fakeWakuMessage(@[byte 07]), - fakeWakuMessage(@[byte 08]), - fakeWakuMessage(@[byte 09]), - ] + let messages = @[ + fakeWakuMessage(@[byte 00]), + fakeWakuMessage(@[byte 01]), + fakeWakuMessage(@[byte 02]), + fakeWakuMessage(@[byte 03]), + fakeWakuMessage(@[byte 04]), + fakeWakuMessage(@[byte 05]), + fakeWakuMessage(@[byte 06]), + fakeWakuMessage(@[byte 07]), + fakeWakuMessage(@[byte 08]), + fakeWakuMessage(@[byte 09]), + ] let serverKey = generateSecp256k1Key() diff --git a/tests/waku_store/test_wakunode_store.nim b/tests/waku_store/test_wakunode_store.nim index 9239435af..fa73cd16d 100644 --- a/tests/waku_store/test_wakunode_store.nim +++ b/tests/waku_store/test_wakunode_store.nim @@ -32,19 +32,18 @@ import procSuite "WakuNode - Store": ## Fixtures let timeOrigin = now() - let msgListA = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] + let msgListA = @[ + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + ] let hashes = msgListA.mapIt(computeMessageHash(DefaultPubsubTopic, it)) diff --git a/tests/waku_store_legacy/test_resume.nim b/tests/waku_store_legacy/test_resume.nim index 53e48834e..0f8132f08 100644 --- a/tests/waku_store_legacy/test_resume.nim +++ b/tests/waku_store_legacy/test_resume.nim @@ -29,93 +29,91 @@ when defined(waku_exp_store_resume): ## Fixtures let storeA = block: let store = newTestMessageStore() - let msgList = - @[ - fakeWakuMessage( - payload = @[byte 0], contentTopic = ContentTopic("2"), ts = ts(0) - ), - fakeWakuMessage( - payload = @[byte 1], contentTopic = ContentTopic("1"), ts = ts(1) - ), - fakeWakuMessage( - payload = @[byte 2], contentTopic = ContentTopic("2"), ts = ts(2) - ), - fakeWakuMessage( - payload = @[byte 3], contentTopic = ContentTopic("1"), ts = ts(3) - ), - fakeWakuMessage( - payload = @[byte 4], contentTopic = ContentTopic("2"), ts = ts(4) - ), - fakeWakuMessage( - payload = @[byte 5], contentTopic = ContentTopic("1"), ts = ts(5) - ), - fakeWakuMessage( - payload = @[byte 6], contentTopic = ContentTopic("2"), ts = ts(6) - ), - fakeWakuMessage( - payload = @[byte 7], contentTopic = ContentTopic("1"), ts = ts(7) - ), - fakeWakuMessage( - payload = @[byte 8], contentTopic = ContentTopic("2"), ts = ts(8) - ), - fakeWakuMessage( - payload = @[byte 9], contentTopic = ContentTopic("1"), ts = ts(9) - ), - ] + let msgList = @[ + fakeWakuMessage( + payload = @[byte 0], contentTopic = ContentTopic("2"), ts = ts(0) + ), + fakeWakuMessage( + payload = @[byte 1], contentTopic = ContentTopic("1"), ts = ts(1) + ), + fakeWakuMessage( + payload = @[byte 2], contentTopic = ContentTopic("2"), ts = ts(2) + ), + fakeWakuMessage( + payload = @[byte 3], contentTopic = ContentTopic("1"), ts = ts(3) + ), + fakeWakuMessage( + payload = @[byte 4], contentTopic = ContentTopic("2"), ts = ts(4) + ), + fakeWakuMessage( + payload = @[byte 5], contentTopic = ContentTopic("1"), ts = ts(5) + ), + fakeWakuMessage( + payload = @[byte 6], contentTopic = ContentTopic("2"), ts = ts(6) + ), + fakeWakuMessage( + payload = @[byte 7], contentTopic = ContentTopic("1"), ts = ts(7) + ), + fakeWakuMessage( + payload = @[byte 8], contentTopic = ContentTopic("2"), ts = ts(8) + ), + fakeWakuMessage( + payload = @[byte 9], contentTopic = ContentTopic("1"), ts = ts(9) + ), + ] for msg in msgList: require store - .put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - .isOk() + .put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + .isOk() store let storeB = block: let store = newTestMessageStore() - let msgList2 = - @[ - fakeWakuMessage( - payload = @[byte 0], contentTopic = ContentTopic("2"), ts = ts(0) - ), - fakeWakuMessage( - payload = @[byte 11], contentTopic = ContentTopic("1"), ts = ts(1) - ), - fakeWakuMessage( - payload = @[byte 12], contentTopic = ContentTopic("2"), ts = ts(2) - ), - fakeWakuMessage( - payload = @[byte 3], contentTopic = ContentTopic("1"), ts = ts(3) - ), - fakeWakuMessage( - payload = @[byte 4], contentTopic = ContentTopic("2"), ts = ts(4) - ), - fakeWakuMessage( - payload = @[byte 5], contentTopic = ContentTopic("1"), ts = ts(5) - ), - fakeWakuMessage( - payload = @[byte 13], contentTopic = ContentTopic("2"), ts = ts(6) - ), - fakeWakuMessage( - payload = @[byte 14], contentTopic = ContentTopic("1"), ts = ts(7) - ), - ] + let msgList2 = @[ + fakeWakuMessage( + payload = @[byte 0], contentTopic = ContentTopic("2"), ts = ts(0) + ), + fakeWakuMessage( + payload = @[byte 11], contentTopic = ContentTopic("1"), ts = ts(1) + ), + fakeWakuMessage( + payload = @[byte 12], contentTopic = ContentTopic("2"), ts = ts(2) + ), + fakeWakuMessage( + payload = @[byte 3], contentTopic = ContentTopic("1"), ts = ts(3) + ), + fakeWakuMessage( + payload = @[byte 4], contentTopic = ContentTopic("2"), ts = ts(4) + ), + fakeWakuMessage( + payload = @[byte 5], contentTopic = ContentTopic("1"), ts = ts(5) + ), + fakeWakuMessage( + payload = @[byte 13], contentTopic = ContentTopic("2"), ts = ts(6) + ), + fakeWakuMessage( + payload = @[byte 14], contentTopic = ContentTopic("1"), ts = ts(7) + ), + ] for msg in msgList2: require store - .put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - .isOk() + .put( + DefaultPubsubTopic, + msg, + computeDigest(msg), + computeMessageHash(DefaultPubsubTopic, msg), + msg.timestamp, + ) + .isOk() store @@ -136,11 +134,10 @@ when defined(waku_exp_store_resume): client = newTestWakuStoreClient(clientSwitch) ## Given - let peers = - @[ - serverSwitchA.peerInfo.toRemotePeerInfo(), - serverSwitchB.peerInfo.toRemotePeerInfo(), - ] + let peers = @[ + serverSwitchA.peerInfo.toRemotePeerInfo(), + serverSwitchB.peerInfo.toRemotePeerInfo(), + ] let req = HistoryQuery(contentTopics: @[DefaultContentTopic], pageSize: 5) ## When @@ -226,12 +223,11 @@ when defined(waku_exp_store_resume): client = await newTestWakuStore(clientSwitch) ## Given - let peers = - @[ - offlineSwitch.peerInfo.toRemotePeerInfo(), - serverASwitch.peerInfo.toRemotePeerInfo(), - serverBSwitch.peerInfo.toRemotePeerInfo(), - ] + let peers = @[ + offlineSwitch.peerInfo.toRemotePeerInfo(), + serverASwitch.peerInfo.toRemotePeerInfo(), + serverBSwitch.peerInfo.toRemotePeerInfo(), + ] ## When let res = await client.resume(some(peers)) @@ -323,11 +319,11 @@ when defined(waku_exp_store_resume): receivedTime3 = now() + getNanosecondTime(10) digest3 = computeDigest(msg3) require server.wakuStore.store - .put(DefaultPubsubTopic, msg3, digest3, receivedTime3) - .isOk() + .put(DefaultPubsubTopic, msg3, digest3, receivedTime3) + .isOk() require client.wakuStore.store - .put(DefaultPubsubTopic, msg3, digest3, receivedTime3) - .isOk() + .put(DefaultPubsubTopic, msg3, digest3, receivedTime3) + .isOk() let serverPeer = server.peerInfo.toRemotePeerInfo() diff --git a/tests/waku_store_legacy/test_rpc_codec.nim b/tests/waku_store_legacy/test_rpc_codec.nim index 6897bab41..2801cc9a8 100644 --- a/tests/waku_store_legacy/test_rpc_codec.nim +++ b/tests/waku_store_legacy/test_rpc_codec.nim @@ -100,11 +100,10 @@ procSuite "Waku Store - RPC codec": direction: some(PagingDirection.BACKWARD), ) query = HistoryQueryRPC( - contentFilters: - @[ - HistoryContentFilterRPC(contentTopic: DefaultContentTopic), - HistoryContentFilterRPC(contentTopic: DefaultContentTopic), - ], + contentFilters: @[ + HistoryContentFilterRPC(contentTopic: DefaultContentTopic), + HistoryContentFilterRPC(contentTopic: DefaultContentTopic), + ], pagingInfo: some(pagingInfo), startTime: some(Timestamp(10)), endTime: some(Timestamp(11)), diff --git a/tests/waku_store_legacy/test_wakunode_store.nim b/tests/waku_store_legacy/test_wakunode_store.nim index 549033e98..58e3ca9e0 100644 --- a/tests/waku_store_legacy/test_wakunode_store.nim +++ b/tests/waku_store_legacy/test_wakunode_store.nim @@ -30,19 +30,18 @@ import procSuite "WakuNode - Store Legacy": ## Fixtures let timeOrigin = now() - let msgListA = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] + let msgListA = @[ + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + ] let archiveA = block: let driver = newSqliteArchiveDriver() diff --git a/tests/waku_store_sync/test_range_split.nim b/tests/waku_store_sync/test_range_split.nim index 546f2cfa5..fe5252416 100644 --- a/tests/waku_store_sync/test_range_split.nim +++ b/tests/waku_store_sync/test_range_split.nim @@ -119,12 +119,11 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(wholeRange, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - wholeRange, @[DefaultPubsubTopic], @[DefaultContentTopic] - ) - ], + fingerprints: @[ + remote.computeFingerprint( + wholeRange, @[DefaultPubsubTopic], @[DefaultContentTopic] + ) + ], itemSets: @[], ) @@ -180,12 +179,11 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(sliceWhole, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - sliceWhole, @[DefaultPubsubTopic], @[DefaultContentTopic] - ) - ], + fingerprints: @[ + remote.computeFingerprint( + sliceWhole, @[DefaultPubsubTopic], @[DefaultContentTopic] + ) + ], itemSets: @[], ) @@ -207,12 +205,11 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(subSlice, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - subSlice, @[DefaultPubsubTopic], @[DefaultContentTopic] - ) - ], + fingerprints: @[ + remote.computeFingerprint( + subSlice, @[DefaultPubsubTopic], @[DefaultContentTopic] + ) + ], itemSets: @[], ) @@ -272,12 +269,9 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(slice, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - slice, @[DefaultPubsubTopic], @[DefaultContentTopic] - ) - ], + fingerprints: @[ + remote.computeFingerprint(slice, @[DefaultPubsubTopic], @[DefaultContentTopic]) + ], itemSets: @[], ) diff --git a/tests/waku_store_sync/test_state_transition.nim b/tests/waku_store_sync/test_state_transition.nim index d94d6bed2..2e6bb30c3 100644 --- a/tests/waku_store_sync/test_state_transition.nim +++ b/tests/waku_store_sync/test_state_transition.nim @@ -44,12 +44,9 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(whole, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - whole, @[DefaultPubsubTopic], @[DefaultContentTopic] - ) - ], + fingerprints: @[ + remote.computeFingerprint(whole, @[DefaultPubsubTopic], @[DefaultContentTopic]) + ], itemSets: @[], ) let rep1 = local.processPayload(p1, s1, r1) @@ -131,15 +128,10 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(sliceA, RangeType.Fingerprint), (sliceB, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - sliceA, @[DefaultPubsubTopic], @[DefaultContentTopic] - ), - remote.computeFingerprint( - sliceB, @[DefaultPubsubTopic], @[DefaultContentTopic] - ), - ], + fingerprints: @[ + remote.computeFingerprint(sliceA, @[DefaultPubsubTopic], @[DefaultContentTopic]), + remote.computeFingerprint(sliceB, @[DefaultPubsubTopic], @[DefaultContentTopic]), + ], itemSets: @[], ) let reply = local.processPayload(payload, s, r) @@ -180,12 +172,9 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(slice, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - slice, @[DefaultPubsubTopic], @[DefaultContentTopic] - ) - ], + fingerprints: @[ + remote.computeFingerprint(slice, @[DefaultPubsubTopic], @[DefaultContentTopic]) + ], itemSets: @[], ) let reply = local.processPayload(p, toS, toR) @@ -236,12 +225,9 @@ suite "Waku Sync – reconciliation": pubsubTopics: @[DefaultPubsubTopic], contentTopics: @[DefaultContentTopic], ranges: @[(s, RangeType.Fingerprint)], - fingerprints: - @[ - remote.computeFingerprint( - s, @[DefaultPubsubTopic], @[DefaultContentTopic] - ) - ], + fingerprints: @[ + remote.computeFingerprint(s, @[DefaultPubsubTopic], @[DefaultContentTopic]) + ], itemSets: @[], ), sendQ, diff --git a/tests/wakunode_rest/test_rest_filter.nim b/tests/wakunode_rest/test_rest_filter.nim index 3d4d741a5..1a4731d6a 100644 --- a/tests/wakunode_rest/test_rest_filter.nim +++ b/tests/wakunode_rest/test_rest_filter.nim @@ -176,12 +176,11 @@ suite "Waku v2 Rest API - Filter V2": ) discard await restFilterTest.client.filterPostSubscriptions(requestBody) - let contentFilters = - @[ - ContentTopic("1"), - ContentTopic("2"), - ContentTopic("3"), # ,ContentTopic("4") # Keep this subscription for check - ] + let contentFilters = @[ + ContentTopic("1"), + ContentTopic("2"), + ContentTopic("3"), # ,ContentTopic("4") # Keep this subscription for check + ] let requestBodyUnsub = FilterUnsubscribeRequest( requestId: "4321", diff --git a/tests/wakunode_rest/test_rest_relay.nim b/tests/wakunode_rest/test_rest_relay.nim index efdd597ba..b791da29f 100644 --- a/tests/wakunode_rest/test_rest_relay.nim +++ b/tests/wakunode_rest/test_rest_relay.nim @@ -193,15 +193,14 @@ suite "Waku v2 Rest API - Relay": let pubSubTopic = "/waku/2/rs/0/0" - var messages = - @[ - fakeWakuMessage( - contentTopic = "content-topic-x", - payload = toBytes("TEST-1"), - meta = toBytes("test-meta"), - ephemeral = true, - ) - ] + var messages = @[ + fakeWakuMessage( + contentTopic = "content-topic-x", + payload = toBytes("TEST-1"), + meta = toBytes("test-meta"), + ephemeral = true, + ) + ] # Prevent duplicate messages for i in 0 ..< 2: @@ -345,12 +344,11 @@ suite "Waku v2 Rest API - Relay": installRelayApiHandlers(restServer.router, node, cache) restServer.start() - let contentTopics = - @[ - ContentTopic("/app-1/2/default-content/proto"), - ContentTopic("/app-2/2/default-content/proto"), - ContentTopic("/app-3/2/default-content/proto"), - ] + let contentTopics = @[ + ContentTopic("/app-1/2/default-content/proto"), + ContentTopic("/app-2/2/default-content/proto"), + ContentTopic("/app-3/2/default-content/proto"), + ] # When let client = newRestHttpClient(initTAddress(restAddress, restPort)) @@ -391,13 +389,12 @@ suite "Waku v2 Rest API - Relay": restPort = restServer.httpServer.address.port # update with bound port for client use - let contentTopics = - @[ - ContentTopic("/waku/2/default-content1/proto"), - ContentTopic("/waku/2/default-content2/proto"), - ContentTopic("/waku/2/default-content3/proto"), - ContentTopic("/waku/2/default-contentX/proto"), - ] + let contentTopics = @[ + ContentTopic("/waku/2/default-content1/proto"), + ContentTopic("/waku/2/default-content2/proto"), + ContentTopic("/waku/2/default-content3/proto"), + ContentTopic("/waku/2/default-contentX/proto"), + ] let cache = MessageCache.init() cache.contentSubscribe(contentTopics[0]) @@ -451,10 +448,9 @@ suite "Waku v2 Rest API - Relay": let contentTopic = DefaultContentTopic - var messages = - @[ - fakeWakuMessage(contentTopic = DefaultContentTopic, payload = toBytes("TEST-1")) - ] + var messages = @[ + fakeWakuMessage(contentTopic = DefaultContentTopic, payload = toBytes("TEST-1")) + ] # Prevent duplicate messages for i in 0 ..< 2: diff --git a/tests/wakunode_rest/test_rest_store.nim b/tests/wakunode_rest/test_rest_store.nim index 70a3c137a..01ccea9dd 100644 --- a/tests/wakunode_rest/test_rest_store.nim +++ b/tests/wakunode_rest/test_rest_store.nim @@ -115,17 +115,16 @@ procSuite "Waku Rest API - Store v3": await sleepAsync(1.seconds()) # Now prime it with some history before tests - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), - fakeWakuMessage(@[byte 1], ts = 1), - fakeWakuMessage(@[byte 1, byte 2], ts = 2), - fakeWakuMessage(@[byte 1], ts = 3), - fakeWakuMessage(@[byte 1], ts = 4), - fakeWakuMessage(@[byte 1], ts = 5), - fakeWakuMessage(@[byte 1], ts = 6), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("c2"), ts = 9), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), + fakeWakuMessage(@[byte 1], ts = 1), + fakeWakuMessage(@[byte 1, byte 2], ts = 2), + fakeWakuMessage(@[byte 1], ts = 3), + fakeWakuMessage(@[byte 1], ts = 4), + fakeWakuMessage(@[byte 1], ts = 5), + fakeWakuMessage(@[byte 1], ts = 6), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("c2"), ts = 9), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -191,17 +190,16 @@ procSuite "Waku Rest API - Store v3": peerSwitch.mount(node.wakuStore) # Now prime it with some history before tests - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), - fakeWakuMessage(@[byte 1], ts = 1), - fakeWakuMessage(@[byte 1, byte 2], ts = 2), - fakeWakuMessage(@[byte 1], ts = 3), - fakeWakuMessage(@[byte 1], ts = 4), - fakeWakuMessage(@[byte 1], ts = 5), - fakeWakuMessage(@[byte 1], ts = 6), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("c2"), ts = 9), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), + fakeWakuMessage(@[byte 1], ts = 1), + fakeWakuMessage(@[byte 1, byte 2], ts = 2), + fakeWakuMessage(@[byte 1], ts = 3), + fakeWakuMessage(@[byte 1], ts = 4), + fakeWakuMessage(@[byte 1], ts = 5), + fakeWakuMessage(@[byte 1], ts = 6), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("c2"), ts = 9), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -262,19 +260,18 @@ procSuite "Waku Rest API - Store v3": # Now prime it with some history before tests let timeOrigin = wakucore.now() - let msgList = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] + let msgList = @[ + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -357,12 +354,11 @@ procSuite "Waku Rest API - Store v3": peerSwitch.mount(node.wakuStore) # Now prime it with some history before tests - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2"), ts = 0), - fakeWakuMessage(@[byte 1], ts = 1), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2"), ts = 9), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2"), ts = 0), + fakeWakuMessage(@[byte 1], ts = 1), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2"), ts = 9), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -431,12 +427,11 @@ procSuite "Waku Rest API - Store v3": peerSwitch.mount(node.wakuStore) # Now prime it with some history before tests - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), - fakeWakuMessage(@[byte 1], ts = 1), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), + fakeWakuMessage(@[byte 1], ts = 1), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -521,12 +516,11 @@ procSuite "Waku Rest API - Store v3": peerSwitch.mount(node.wakuStore) # Now prime it with some history before tests - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), - fakeWakuMessage(@[byte 1], ts = 1), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), + fakeWakuMessage(@[byte 1], ts = 1), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -594,12 +588,11 @@ procSuite "Waku Rest API - Store v3": await node.mountStore() # Now prime it with some history before tests - let msgList = - @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), - fakeWakuMessage(@[byte 1], ts = 1), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), - ] + let msgList = @[ + fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("ct1"), ts = 0), + fakeWakuMessage(@[byte 1], ts = 1), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -640,14 +633,13 @@ procSuite "Waku Rest API - Store v3": await node.mountStore() # Now prime it with some history before tests - let msgList = - @[ - fakeWakuMessage( - @[byte 0], contentTopic = ContentTopic("ct1"), ts = 0, meta = (@[byte 8]) - ), - fakeWakuMessage(@[byte 1], ts = 1), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), - ] + let msgList = @[ + fakeWakuMessage( + @[byte 0], contentTopic = ContentTopic("ct1"), ts = 0, meta = (@[byte 8]) + ), + fakeWakuMessage(@[byte 1], ts = 1), + fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("ct2"), ts = 9), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() @@ -757,19 +749,18 @@ procSuite "Waku Rest API - Store v3": # Now prime it with some history before tests let timeOrigin = wakucore.now() - let msgList = - @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] + let msgList = @[ + fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), + fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), + fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), + fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), + fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), + fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), + fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), + fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), + fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), + fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), + ] for msg in msgList: require (await driver.put(DefaultPubsubTopic, msg)).isOk() diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim index 4a6e8c618..9854828ff 100644 --- a/tools/confutils/cli_args.nim +++ b/tools/confutils/cli_args.nim @@ -263,8 +263,7 @@ type WakuNodeConf* = object ## Circuit-relay config isRelayClient* {. - desc: - """Set the node as a relay-client. + desc: """Set the node as a relay-client. Set it to true for nodes that run behind a NAT or firewall and hence would have reachability issues.""", defaultValue: false, diff --git a/vendor/nph b/vendor/nph index c6e03162d..2cacf6cc2 160000 --- a/vendor/nph +++ b/vendor/nph @@ -1 +1 @@ -Subproject commit c6e03162dc2820d3088660f644818d7040e95791 +Subproject commit 2cacf6cc28116e4046e0b67a13545af5c4e756bd diff --git a/waku/api/api_conf.nim b/waku/api/api_conf.nim index 70bb02af3..30dfd1b2c 100644 --- a/waku/api/api_conf.nim +++ b/waku/api/api_conf.nim @@ -62,10 +62,9 @@ proc init*( ) const TheWakuNetworkPreset* = ProtocolsConfig( - entryNodes: - @[ - "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im" - ], + entryNodes: @[ + "enrtree://AIRVQ5DDA4FFWLRBCHJWUWOO6X6S4ZTZ5B667LQ6AJU6PEYDLRD5O@sandbox.waku.nodes.status.im" + ], staticStoreNodes: @[], clusterId: 1, autoShardingConfig: AutoShardingConfig(numShardsInCluster: 8), diff --git a/waku/common/broker/event_broker.nim b/waku/common/broker/event_broker.nim index 779689f88..3fd10cea2 100644 --- a/waku/common/broker/event_broker.nim +++ b/waku/common/broker/event_broker.nim @@ -151,14 +151,13 @@ macro EventBroker*(body: untyped): untyped = proc `accessProcIdent`(): `brokerTypeIdent` = if `globalVarIdent`.isNil(): new(`globalVarIdent`) - `globalVarIdent`.buckets = - @[ - `bucketTypeIdent`( - brokerCtx: DefaultBrokerContext, - listeners: initTable[uint64, `handlerProcIdent`](), - nextId: 1'u64, - ) - ] + `globalVarIdent`.buckets = @[ + `bucketTypeIdent`( + brokerCtx: DefaultBrokerContext, + listeners: initTable[uint64, `handlerProcIdent`](), + nextId: 1'u64, + ) + ] `globalVarIdent` ) diff --git a/waku/common/databases/db_postgres/dbconn.nim b/waku/common/databases/db_postgres/dbconn.nim index 7ccf32099..f24f3d4dd 100644 --- a/waku/common/databases/db_postgres/dbconn.nim +++ b/waku/common/databases/db_postgres/dbconn.nim @@ -211,11 +211,10 @@ proc waitQueryToFinish( pqclear(pqResult) proc containsRiskyPatterns(input: string): bool = - let riskyPatterns = - @[ - " OR ", " AND ", " UNION ", " SELECT ", "INSERT ", "DELETE ", "UPDATE ", "DROP ", - "EXEC ", "--", "/*", "*/", - ] + let riskyPatterns = @[ + " OR ", " AND ", " UNION ", " SELECT ", "INSERT ", "DELETE ", "UPDATE ", "DROP ", + "EXEC ", "--", "/*", "*/", + ] for pattern in riskyPatterns: if pattern.toLowerAscii() in input.toLowerAscii(): diff --git a/waku/common/rate_limit/timed_map.nim b/waku/common/rate_limit/timed_map.nim index b05dfb0fb..b9a5c4cbf 100644 --- a/waku/common/rate_limit/timed_map.nim +++ b/waku/common/rate_limit/timed_map.nim @@ -106,16 +106,8 @@ proc mgetOrPut*[K, V](t: var TimedMap[K, V], k: K, v: V, now = Moment.now()): va let previous = t.del(k) # Refresh existing item - addedAt = - if previous.isSome(): - previous[].addedAt - else: - now - value = - if previous.isSome(): - previous[].value - else: - v + addedAt = if previous.isSome(): previous[].addedAt else: now + value = if previous.isSome(): previous[].value else: v let node = TimedEntry[K, V](key: k, value: value, addedAt: addedAt, expiresAt: now + t.timeout) diff --git a/waku/factory/builder.nim b/waku/factory/builder.nim index e0b643fc0..87b0db492 100644 --- a/waku/factory/builder.nim +++ b/waku/factory/builder.nim @@ -84,20 +84,19 @@ proc withNetworkConfigurationDetails*( ): WakuNodeBuilderResult {. deprecated: "use 'builder.withNetworkConfiguration()' instead" .} = - let netConfig = - ?NetConfig.init( - bindIp = bindIp, - bindPort = bindPort, - extIp = extIp, - extPort = extPort, - extMultiAddrs = extMultiAddrs, - wsBindPort = some(wsBindPort), - wsEnabled = wsEnabled, - wssEnabled = wssEnabled, - wakuFlags = wakuFlags, - dns4DomainName = dns4DomainName, - dnsNameServers = dnsNameServers, - ) + let netConfig = ?NetConfig.init( + bindIp = bindIp, + bindPort = bindPort, + extIp = extIp, + extPort = extPort, + extMultiAddrs = extMultiAddrs, + wsBindPort = some(wsBindPort), + wsEnabled = wsEnabled, + wssEnabled = wssEnabled, + wakuFlags = wakuFlags, + dns4DomainName = dns4DomainName, + dnsNameServers = dnsNameServers, + ) builder.withNetworkConfiguration(netConfig) ok() diff --git a/waku/factory/networks_config.nim b/waku/factory/networks_config.nim index 94856fb21..d9c0cf879 100644 --- a/waku/factory/networks_config.nim +++ b/waku/factory/networks_config.nim @@ -56,12 +56,11 @@ proc TheWakuNetworkConf*(T: type NetworkConf): NetworkConf = mix: false, p2pReliability: false, discv5Discovery: true, - discv5BootstrapNodes: - @[ - "enr:-QESuED0qW1BCmF-oH_ARGPr97Nv767bl_43uoy70vrbah3EaCAdK3Q0iRQ6wkSTTpdrg_dU_NC2ydO8leSlRpBX4pxiAYJpZIJ2NIJpcIRA4VDAim11bHRpYWRkcnO4XAArNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwAtNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQOTd-h5owwj-cx7xrmbvQKU8CV3Fomfdvcv1MBc-67T5oN0Y3CCdl-DdWRwgiMohXdha3UyDw", - "enr:-QEkuED9X80QF_jcN9gA2ZRhhmwVEeJnsg_Hyg7IFCTYnZD0BDI7a8HArE61NhJZFwygpHCWkgwSt2vqiABXkBxzIqZBAYJpZIJ2NIJpcIQiQlleim11bHRpYWRkcnO4bgA0Ni9ub2RlLTAxLmdjLXVzLWNlbnRyYWwxLWEud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwA2Ni9ub2RlLTAxLmdjLXVzLWNlbnRyYWwxLWEud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQPFAS8zz2cg1QQhxMaK8CzkGQ5wdHvPJcrgLzJGOiHpwYN0Y3CCdl-DdWRwgiMohXdha3UyDw", - "enr:-QEkuEBfEzJm_kigJ2HoSS_RBFJYhKHocGdkhhBr6jSUAWjLdFPp6Pj1l4yiTQp7TGHyu1kC6FyaU573VN8klLsEm-XuAYJpZIJ2NIJpcIQI2SVcim11bHRpYWRkcnO4bgA0Ni9ub2RlLTAxLmFjLWNuLWhvbmdrb25nLWMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwA2Ni9ub2RlLTAxLmFjLWNuLWhvbmdrb25nLWMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQOwsS69tgD7u1K50r5-qG5hweuTwa0W26aYPnvivpNlrYN0Y3CCdl-DdWRwgiMohXdha3UyDw", - ], + discv5BootstrapNodes: @[ + "enr:-QESuED0qW1BCmF-oH_ARGPr97Nv767bl_43uoy70vrbah3EaCAdK3Q0iRQ6wkSTTpdrg_dU_NC2ydO8leSlRpBX4pxiAYJpZIJ2NIJpcIRA4VDAim11bHRpYWRkcnO4XAArNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwAtNiZub2RlLTAxLmRvLWFtczMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQOTd-h5owwj-cx7xrmbvQKU8CV3Fomfdvcv1MBc-67T5oN0Y3CCdl-DdWRwgiMohXdha3UyDw", + "enr:-QEkuED9X80QF_jcN9gA2ZRhhmwVEeJnsg_Hyg7IFCTYnZD0BDI7a8HArE61NhJZFwygpHCWkgwSt2vqiABXkBxzIqZBAYJpZIJ2NIJpcIQiQlleim11bHRpYWRkcnO4bgA0Ni9ub2RlLTAxLmdjLXVzLWNlbnRyYWwxLWEud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwA2Ni9ub2RlLTAxLmdjLXVzLWNlbnRyYWwxLWEud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQPFAS8zz2cg1QQhxMaK8CzkGQ5wdHvPJcrgLzJGOiHpwYN0Y3CCdl-DdWRwgiMohXdha3UyDw", + "enr:-QEkuEBfEzJm_kigJ2HoSS_RBFJYhKHocGdkhhBr6jSUAWjLdFPp6Pj1l4yiTQp7TGHyu1kC6FyaU573VN8klLsEm-XuAYJpZIJ2NIJpcIQI2SVcim11bHRpYWRkcnO4bgA0Ni9ub2RlLTAxLmFjLWNuLWhvbmdrb25nLWMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQZ2XwA2Ni9ub2RlLTAxLmFjLWNuLWhvbmdrb25nLWMud2FrdS5zYW5kYm94LnN0YXR1cy5pbQYfQN4DgnJzkwABCAAAAAEAAgADAAQABQAGAAeJc2VjcDI1NmsxoQOwsS69tgD7u1K50r5-qG5hweuTwa0W26aYPnvivpNlrYN0Y3CCdl-DdWRwgiMohXdha3UyDw", + ], ) # cluster-id=2 (Logos Dev Network) @@ -83,15 +82,14 @@ proc LogosDevConf*(T: type NetworkConf): NetworkConf = p2pReliability: true, discv5Discovery: true, discv5BootstrapNodes: @[], - entryNodes: - @[ - "/dns4/delivery-01.do-ams3.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAmTUbnxLGT9JvV6mu9oPyDjqHK4Phs1VDJNUgESgNSkuby", - "/dns4/delivery-02.do-ams3.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAmMK7PYygBtKUQ8EHp7EfaD3bCEsJrkFooK8RQ2PVpJprH", - "/dns4/delivery-01.gc-us-central1-a.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAm4S1JYkuzDKLKQvwgAhZKs9otxXqt8SCGtB4hoJP1S397", - "/dns4/delivery-02.gc-us-central1-a.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAm8Y9kgBNtjxvCnf1X6gnZJW5EGE4UwwCL3CCm55TwqBiH", - "/dns4/delivery-01.ac-cn-hongkong-c.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAm8YokiNun9BkeA1ZRmhLbtNUvcwRr64F69tYj9fkGyuEP", - "/dns4/delivery-02.ac-cn-hongkong-c.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAkvwhGHKNry6LACrB8TmEFoCJKEX29XR5dDUzk3UT3UNSE", - ], + entryNodes: @[ + "/dns4/delivery-01.do-ams3.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAmTUbnxLGT9JvV6mu9oPyDjqHK4Phs1VDJNUgESgNSkuby", + "/dns4/delivery-02.do-ams3.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAmMK7PYygBtKUQ8EHp7EfaD3bCEsJrkFooK8RQ2PVpJprH", + "/dns4/delivery-01.gc-us-central1-a.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAm4S1JYkuzDKLKQvwgAhZKs9otxXqt8SCGtB4hoJP1S397", + "/dns4/delivery-02.gc-us-central1-a.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAm8Y9kgBNtjxvCnf1X6gnZJW5EGE4UwwCL3CCm55TwqBiH", + "/dns4/delivery-01.ac-cn-hongkong-c.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAm8YokiNun9BkeA1ZRmhLbtNUvcwRr64F69tYj9fkGyuEP", + "/dns4/delivery-02.ac-cn-hongkong-c.logos.dev.status.im/tcp/30303/p2p/16Uiu2HAkvwhGHKNry6LACrB8TmEFoCJKEX29XR5dDUzk3UT3UNSE", + ], ) proc validateShards*( diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index 2f82440f6..57e18e3c0 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -126,11 +126,10 @@ proc initNode( builder.withRateLimit(conf.rateLimit) builder.withCircuitRelay(relay) - let node = - ?builder.build().mapErr( - proc(err: string): string = - "failed to create waku node instance: " & err - ) + let node = ?builder.build().mapErr( + proc(err: string): string = + "failed to create waku node instance: " & err + ) ok(node) diff --git a/waku/node/peer_manager/peer_manager.nim b/waku/node/peer_manager/peer_manager.nim index 0c435468f..dc0da9624 100644 --- a/waku/node/peer_manager/peer_manager.nim +++ b/waku/node/peer_manager/peer_manager.nim @@ -1,11 +1,9 @@ {.push raises: [].} import - std/ - [ - options, sets, sequtils, times, strformat, strutils, math, random, tables, - algorithm, - ], + std/[ + options, sets, sequtils, times, strformat, strutils, math, random, tables, algorithm + ], chronos, chronicles, metrics, diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 0cef4cc5d..0c6cb7ac4 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -356,12 +356,11 @@ proc mountStoreSync*( let pubsubTopics = shards.mapIt($RelayShard(clusterId: cluster, shardId: it)) - let recon = - ?await SyncReconciliation.new( - pubsubTopics, contentTopics, node.peerManager, node.wakuArchive, - storeSyncRange.seconds, storeSyncInterval.seconds, storeSyncRelayJitter.seconds, - idsChannel, wantsChannel, needsChannel, - ) + let recon = ?await SyncReconciliation.new( + pubsubTopics, contentTopics, node.peerManager, node.wakuArchive, + storeSyncRange.seconds, storeSyncInterval.seconds, storeSyncRelayJitter.seconds, + idsChannel, wantsChannel, needsChannel, + ) node.wakuStoreReconciliation = recon diff --git a/waku/rest_api/endpoint/builder.nim b/waku/rest_api/endpoint/builder.nim index 41ab7e06b..cb23bc284 100644 --- a/waku/rest_api/endpoint/builder.nim +++ b/waku/rest_api/endpoint/builder.nim @@ -86,13 +86,12 @@ proc startRestServerEssentials*( let address = conf.listenAddress let port = Port(conf.port.uint16 + portsShift) - let server = - ?newRestHttpServer( - address, - port, - allowedOrigin = allowedOrigin, - requestErrorHandler = requestErrorHandler, - ) + let server = ?newRestHttpServer( + address, + port, + allowedOrigin = allowedOrigin, + requestErrorHandler = requestErrorHandler, + ) ## Health REST API installHealthApiHandler(server.router, nodeHealthMonitor) diff --git a/waku/rest_api/endpoint/server.nim b/waku/rest_api/endpoint/server.nim index 1b61425c8..44a02ccb2 100644 --- a/waku/rest_api/endpoint/server.nim +++ b/waku/rest_api/endpoint/server.nim @@ -91,23 +91,22 @@ proc new*( ): Future[HttpResponseRef] {.async: (raises: [CancelledError]).} = discard - server.httpServer = - ?HttpServerRef.new( - address, - defaultProcessCallback, - serverFlags, - socketFlags, - serverUri, - serverIdent, - maxConnections, - bufferSize, - backlogSize, - httpHeadersTimeout, - maxHeadersSize, - maxRequestBodySize, - dualstack = dualstack, - middlewares = middlewares, - ) + server.httpServer = ?HttpServerRef.new( + address, + defaultProcessCallback, + serverFlags, + socketFlags, + serverUri, + serverIdent, + maxConnections, + bufferSize, + backlogSize, + httpHeadersTimeout, + maxHeadersSize, + maxRequestBodySize, + dualstack = dualstack, + middlewares = middlewares, + ) return ok(server) proc getRouter(): RestRouter = diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index 2f495ba5d..6a322cf77 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -45,8 +45,7 @@ const SelectClause = const SelectNoCursorAscStmtName = "SelectWithoutCursorAsc" const SelectNoCursorAscStmtDef = - SelectClause & - """WHERE contentTopic IN ($1) AND + SelectClause & """WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND timestamp >= $4 AND @@ -54,8 +53,7 @@ const SelectNoCursorAscStmtDef = ORDER BY timestamp ASC, messageHash ASC LIMIT $6;""" const SelectNoCursorNoDataAscStmtName = "SelectWithoutCursorAndDataAsc" -const SelectNoCursorNoDataAscStmtDef = - """SELECT messageHash FROM messages +const SelectNoCursorNoDataAscStmtDef = """SELECT messageHash FROM messages WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND @@ -65,8 +63,7 @@ const SelectNoCursorNoDataAscStmtDef = const SelectNoCursorDescStmtName = "SelectWithoutCursorDesc" const SelectNoCursorDescStmtDef = - SelectClause & - """WHERE contentTopic IN ($1) AND + SelectClause & """WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND timestamp >= $4 AND @@ -74,8 +71,7 @@ const SelectNoCursorDescStmtDef = ORDER BY timestamp DESC, messageHash DESC LIMIT $6;""" const SelectNoCursorNoDataDescStmtName = "SelectWithoutCursorAndDataDesc" -const SelectNoCursorNoDataDescStmtDef = - """SELECT messageHash FROM messages +const SelectNoCursorNoDataDescStmtDef = """SELECT messageHash FROM messages WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND @@ -85,8 +81,7 @@ const SelectNoCursorNoDataDescStmtDef = const SelectWithCursorDescStmtName = "SelectWithCursorDesc" const SelectWithCursorDescStmtDef = - SelectClause & - """WHERE contentTopic IN ($1) AND + SelectClause & """WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND (timestamp, messageHash) < ($4,$5) AND @@ -95,8 +90,7 @@ const SelectWithCursorDescStmtDef = ORDER BY timestamp DESC, messageHash DESC LIMIT $8;""" const SelectWithCursorNoDataDescStmtName = "SelectWithCursorNoDataDesc" -const SelectWithCursorNoDataDescStmtDef = - """SELECT messageHash FROM messages +const SelectWithCursorNoDataDescStmtDef = """SELECT messageHash FROM messages WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND @@ -107,8 +101,7 @@ const SelectWithCursorNoDataDescStmtDef = const SelectWithCursorAscStmtName = "SelectWithCursorAsc" const SelectWithCursorAscStmtDef = - SelectClause & - """WHERE contentTopic IN ($1) AND + SelectClause & """WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND (timestamp, messageHash) > ($4,$5) AND @@ -117,8 +110,7 @@ const SelectWithCursorAscStmtDef = ORDER BY timestamp ASC, messageHash ASC LIMIT $8;""" const SelectWithCursorNoDataAscStmtName = "SelectWithCursorNoDataAsc" -const SelectWithCursorNoDataAscStmtDef = - """SELECT messageHash FROM messages +const SelectWithCursorNoDataAscStmtDef = """SELECT messageHash FROM messages WHERE contentTopic IN ($1) AND messageHash IN ($2) AND pubsubTopic = $3 AND @@ -128,8 +120,7 @@ const SelectWithCursorNoDataAscStmtDef = ORDER BY timestamp ASC, messageHash ASC LIMIT $8;""" const SelectCursorByHashName = "SelectMessageByHashInMessagesLookup" -const SelectCursorByHashDef = - """SELECT timestamp FROM messages_lookup +const SelectCursorByHashDef = """SELECT timestamp FROM messages_lookup WHERE messageHash = $1""" const @@ -896,11 +887,10 @@ method getMessages*( let splittedHashes = hashes[i ..< stop] - let subRows = - ?await s.getMessagesWithinLimits( - includeData, contentTopics, pubsubTopic, cursor, startTime, endTime, - splittedHashes, maxPageSize, ascendingOrder, requestId, - ) + let subRows = ?await s.getMessagesWithinLimits( + includeData, contentTopics, pubsubTopic, cursor, startTime, endTime, + splittedHashes, maxPageSize, ascendingOrder, requestId, + ) for row in subRows: row diff --git a/waku/waku_archive/driver/sqlite_driver/queries.nim b/waku/waku_archive/driver/sqlite_driver/queries.nim index e7e31dbe0..9ef6591c2 100644 --- a/waku/waku_archive/driver/sqlite_driver/queries.nim +++ b/waku/waku_archive/driver/sqlite_driver/queries.nim @@ -78,12 +78,11 @@ proc createTableQuery(table: string): SqlQueryStr = proc createTable*(db: SqliteDatabase): DatabaseResult[void] = let query = createTableQuery(DbTable) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Create indices @@ -93,12 +92,11 @@ proc createOldestMessageTimestampIndexQuery(table: string): SqlQueryStr = proc createOldestMessageTimestampIndex*(db: SqliteDatabase): DatabaseResult[void] = let query = createOldestMessageTimestampIndexQuery(DbTable) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Insert message @@ -175,12 +173,11 @@ proc deleteMessagesOlderThanTimestamp*( db: SqliteDatabase, ts: int64 ): DatabaseResult[void] = let query = deleteMessagesOlderThanTimestampQuery(DbTable, ts) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Delete oldest messages not within limit @@ -196,12 +193,11 @@ proc deleteOldestMessagesNotWithinLimit*( ): DatabaseResult[void] = # NOTE: The word `limit` here refers the store capacity/maximum number-of-messages allowed limit let query = deleteOldestMessagesNotWithinLimitQuery(DbTable, limit = limit) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Select all messages diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim b/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim index 0cb2bf64d..4590a8df1 100644 --- a/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim +++ b/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim @@ -92,12 +92,11 @@ proc createTableQuery(table: string): SqlQueryStr = proc createTable*(db: SqliteDatabase): DatabaseResult[void] = let query = createTableQuery(DbTable) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Create indices @@ -107,12 +106,11 @@ proc createOldestMessageTimestampIndexQuery(table: string): SqlQueryStr = proc createOldestMessageTimestampIndex*(db: SqliteDatabase): DatabaseResult[void] = let query = createOldestMessageTimestampIndexQuery(DbTable) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() proc createHistoryQueryIndexQuery(table: string): SqlQueryStr = @@ -121,12 +119,11 @@ proc createHistoryQueryIndexQuery(table: string): SqlQueryStr = proc createHistoryQueryIndex*(db: SqliteDatabase): DatabaseResult[void] = let query = createHistoryQueryIndexQuery(DbTable) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Insert message @@ -216,12 +213,11 @@ proc deleteMessagesOlderThanTimestamp*( db: SqliteDatabase, ts: int64 ): DatabaseResult[void] = let query = deleteMessagesOlderThanTimestampQuery(DbTable, ts) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Delete oldest messages not within limit @@ -237,12 +233,11 @@ proc deleteOldestMessagesNotWithinLimit*( ): DatabaseResult[void] = # NOTE: The word `limit` here refers the store capacity/maximum number-of-messages allowed limit let query = deleteOldestMessagesNotWithinLimitQuery(DbTable, limit = limit) - discard - ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) + discard ?db.query( + query, + proc(s: ptr sqlite3_stmt) = + discard, + ) return ok() ## Select all messages diff --git a/waku/waku_core/peers.nim b/waku/waku_core/peers.nim index 51a8e1157..c4b8b593e 100644 --- a/waku/waku_core/peers.nim +++ b/waku/waku_core/peers.nim @@ -176,17 +176,15 @@ proc parsePeerInfoFromRegularAddr(peer: MultiAddress): Result[RemotePeerInfo, st case addrPart[].protoName()[] # All protocols listed here: https://github.com/multiformats/multiaddr/blob/b746a7d014e825221cc3aea6e57a92d78419990f/protocols.csv of "p2p": - p2pPart = - ?addrPart.mapErr( - proc(err: string): string = - "Error getting p2pPart [" & err & "]" - ) + p2pPart = ?addrPart.mapErr( + proc(err: string): string = + "Error getting p2pPart [" & err & "]" + ) of "ip4", "ip6", "dns", "dnsaddr", "dns4", "dns6", "tcp", "ws", "wss": - let val = - ?addrPart.mapErr( - proc(err: string): string = - "Error getting addrPart [" & err & "]" - ) + let val = ?addrPart.mapErr( + proc(err: string): string = + "Error getting addrPart [" & err & "]" + ) ?wireAddr.append(val).mapErr( proc(err: string): string = "Error appending addrPart [" & err & "]" @@ -199,11 +197,10 @@ proc parsePeerInfoFromRegularAddr(peer: MultiAddress): Result[RemotePeerInfo, st "] [peer:" & $peer & "]" return err(msg) - let peerId = - ?PeerID.init(p2pPartStr.split("/")[^1]).mapErr( - proc(e: cstring): string = - $e - ) + let peerId = ?PeerID.init(p2pPartStr.split("/")[^1]).mapErr( + proc(e: cstring): string = + $e + ) if not wireAddr.validWireAddr(): return err("invalid multiaddress: no supported transport found") @@ -233,11 +230,10 @@ proc parsePeerInfo*(maddrs: varargs[string]): Result[RemotePeerInfo, string] = ## format `(ip4|ip6)/tcp/p2p`, into dialable PeerInfo var multiAddresses = newSeq[MultiAddress]() for maddr in maddrs: - let multiAddr = - ?MultiAddress.init(maddr).mapErr( - proc(err: string): string = - "MultiAddress.init [" & err & "]" - ) + let multiAddr = ?MultiAddress.init(maddr).mapErr( + proc(err: string): string = + "MultiAddress.init [" & err & "]" + ) multiAddresses.add(multiAddr) parsePeerInfo(multiAddresses) diff --git a/waku/waku_core/topics/pubsub_topic.nim b/waku/waku_core/topics/pubsub_topic.nim index 27ea27180..1e921d06d 100644 --- a/waku/waku_core/topics/pubsub_topic.nim +++ b/waku/waku_core/topics/pubsub_topic.nim @@ -54,20 +54,18 @@ proc parseStaticSharding*( let clusterPart = parts[0] if clusterPart.len == 0: return err(ParsingError.missingPart("cluster_id")) - let clusterId = - ?Base10.decode(uint16, clusterPart).mapErr( - proc(err: auto): auto = - ParsingError.invalidFormat($err) - ) + let clusterId = ?Base10.decode(uint16, clusterPart).mapErr( + proc(err: auto): auto = + ParsingError.invalidFormat($err) + ) let shardPart = parts[1] if shardPart.len == 0: return err(ParsingError.missingPart("shard_number")) - let shardId = - ?Base10.decode(uint16, shardPart).mapErr( - proc(err: auto): auto = - ParsingError.invalidFormat($err) - ) + let shardId = ?Base10.decode(uint16, shardPart).mapErr( + proc(err: auto): auto = + ParsingError.invalidFormat($err) + ) ok(RelayShard(clusterId: clusterId, shardId: shardId)) diff --git a/waku/waku_enr/sharding.nim b/waku/waku_enr/sharding.nim index 392900cdb..2aeb96a9d 100644 --- a/waku/waku_enr/sharding.nim +++ b/waku/waku_enr/sharding.nim @@ -70,10 +70,9 @@ func topicsToRelayShards*(topics: seq[string]): Result[Option[RelayShards], stri if parsedTopicsRes.anyIt(it.get().clusterId != parsedTopicsRes[0].get().clusterId): return err("use shards with the same cluster Id.") - let relayShard = - ?RelayShards.init( - parsedTopicsRes[0].get().clusterId, parsedTopicsRes.mapIt(it.get().shardId) - ) + let relayShard = ?RelayShards.init( + parsedTopicsRes[0].get().clusterId, parsedTopicsRes.mapIt(it.get().shardId) + ) return ok(some(relayShard)) diff --git a/waku/waku_keystore/protocol_types.nim b/waku/waku_keystore/protocol_types.nim index 6cfc2f183..0f50c66ee 100644 --- a/waku/waku_keystore/protocol_types.nim +++ b/waku/waku_keystore/protocol_types.nim @@ -119,10 +119,9 @@ proc `==`*(x, y: KeystoreMembership): bool = proc hash*(m: KeystoreMembership): string = # hash together the chainId, address and treeIndex - return - $sha256.digest( - m.membershipContract.chainId & m.membershipContract.address & $m.treeIndex - ) + return $sha256.digest( + m.membershipContract.chainId & m.membershipContract.address & $m.treeIndex + ) type MembershipTable* = Table[string, KeystoreMembership] diff --git a/waku/waku_noise/noise_handshake_processing.nim b/waku/waku_noise/noise_handshake_processing.nim index 7688f0a80..8b84bf958 100644 --- a/waku/waku_noise/noise_handshake_processing.nim +++ b/waku/waku_noise/noise_handshake_processing.nim @@ -59,15 +59,14 @@ proc isValid(msg: seq[PreMessagePattern]): bool = var isValid: bool = true # Non-empty pre-messages can only have patterns "e", "s", "e,s" in each direction - let allowedPatterns: seq[PreMessagePattern] = - @[ - PreMessagePattern(direction: D_r, tokens: @[T_s]), - PreMessagePattern(direction: D_r, tokens: @[T_e]), - PreMessagePattern(direction: D_r, tokens: @[T_e, T_s]), - PreMessagePattern(direction: D_l, tokens: @[T_s]), - PreMessagePattern(direction: D_l, tokens: @[T_e]), - PreMessagePattern(direction: D_l, tokens: @[T_e, T_s]), - ] + let allowedPatterns: seq[PreMessagePattern] = @[ + PreMessagePattern(direction: D_r, tokens: @[T_s]), + PreMessagePattern(direction: D_r, tokens: @[T_e]), + PreMessagePattern(direction: D_r, tokens: @[T_e, T_s]), + PreMessagePattern(direction: D_l, tokens: @[T_s]), + PreMessagePattern(direction: D_l, tokens: @[T_e]), + PreMessagePattern(direction: D_l, tokens: @[T_e, T_s]), + ] # We check if pre message patterns are allowed for pattern in msg: diff --git a/waku/waku_noise/noise_types.nim b/waku/waku_noise/noise_types.nim index 3b88c43e8..543bd4329 100644 --- a/waku/waku_noise/noise_types.nim +++ b/waku/waku_noise/noise_types.nim @@ -223,57 +223,51 @@ const NoiseHandshakePatterns* = { "K1K1": HandshakePattern( name: "Noise_K1K1_25519_ChaChaPoly_SHA256", - preMessagePatterns: - @[ - PreMessagePattern(direction: D_r, tokens: @[T_s]), - PreMessagePattern(direction: D_l, tokens: @[T_s]), - ], - messagePatterns: - @[ - MessagePattern(direction: D_r, tokens: @[T_e]), - MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_es]), - MessagePattern(direction: D_r, tokens: @[T_se]), - ], + preMessagePatterns: @[ + PreMessagePattern(direction: D_r, tokens: @[T_s]), + PreMessagePattern(direction: D_l, tokens: @[T_s]), + ], + messagePatterns: @[ + MessagePattern(direction: D_r, tokens: @[T_e]), + MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_es]), + MessagePattern(direction: D_r, tokens: @[T_se]), + ], ), "XK1": HandshakePattern( name: "Noise_XK1_25519_ChaChaPoly_SHA256", preMessagePatterns: @[PreMessagePattern(direction: D_l, tokens: @[T_s])], - messagePatterns: - @[ - MessagePattern(direction: D_r, tokens: @[T_e]), - MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_es]), - MessagePattern(direction: D_r, tokens: @[T_s, T_se]), - ], + messagePatterns: @[ + MessagePattern(direction: D_r, tokens: @[T_e]), + MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_es]), + MessagePattern(direction: D_r, tokens: @[T_s, T_se]), + ], ), "XX": HandshakePattern( name: "Noise_XX_25519_ChaChaPoly_SHA256", preMessagePatterns: EmptyPreMessage, - messagePatterns: - @[ - MessagePattern(direction: D_r, tokens: @[T_e]), - MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_s, T_es]), - MessagePattern(direction: D_r, tokens: @[T_s, T_se]), - ], + messagePatterns: @[ + MessagePattern(direction: D_r, tokens: @[T_e]), + MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_s, T_es]), + MessagePattern(direction: D_r, tokens: @[T_s, T_se]), + ], ), "XXpsk0": HandshakePattern( name: "Noise_XXpsk0_25519_ChaChaPoly_SHA256", preMessagePatterns: EmptyPreMessage, - messagePatterns: - @[ - MessagePattern(direction: D_r, tokens: @[T_psk, T_e]), - MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_s, T_es]), - MessagePattern(direction: D_r, tokens: @[T_s, T_se]), - ], + messagePatterns: @[ + MessagePattern(direction: D_r, tokens: @[T_psk, T_e]), + MessagePattern(direction: D_l, tokens: @[T_e, T_ee, T_s, T_es]), + MessagePattern(direction: D_r, tokens: @[T_s, T_se]), + ], ), "WakuPairing": HandshakePattern( name: "Noise_WakuPairing_25519_ChaChaPoly_SHA256", preMessagePatterns: @[PreMessagePattern(direction: D_l, tokens: @[T_e])], - messagePatterns: - @[ - MessagePattern(direction: D_r, tokens: @[T_e, T_ee]), - MessagePattern(direction: D_l, tokens: @[T_s, T_es]), - MessagePattern(direction: D_r, tokens: @[T_s, T_se, T_ss]), - ], + messagePatterns: @[ + MessagePattern(direction: D_r, tokens: @[T_e, T_ee]), + MessagePattern(direction: D_l, tokens: @[T_s, T_es]), + MessagePattern(direction: D_r, tokens: @[T_s, T_se, T_ss]), + ], ), }.toTable() diff --git a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim index 2e4882891..38c533029 100644 --- a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim +++ b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim @@ -235,8 +235,8 @@ method register*( "Failed to register the member", proc(): Future[TxHash] {.async.} = return await wakuRlnContract - .register(idCommitment, userMessageLimit.stuint(32), idCommitmentsToErase) - .send(gasPrice = gasPrice), + .register(idCommitment, userMessageLimit.stuint(32), idCommitmentsToErase) + .send(gasPrice = gasPrice), ) ).valueOr: return err("Failed to register member: " & error) diff --git a/waku/waku_rln_relay/rln/wrappers.nim b/waku/waku_rln_relay/rln/wrappers.nim index 1b2b0270f..f6f001d70 100644 --- a/waku/waku_rln_relay/rln/wrappers.nim +++ b/waku/waku_rln_relay/rln/wrappers.nim @@ -72,13 +72,12 @@ type RlnConfig = ref object of RootObj proc `%`(c: RlnConfig): JsonNode = ## wrapper around the generic JObject constructor. ## We don't need to have a separate proc for the tree_config field - let tree_config = - %{ - "cache_capacity": %c.tree_config.cache_capacity, - "mode": %c.tree_config.mode, - "compression": %c.tree_config.compression, - "flush_every_ms": %c.tree_config.flush_every_ms, - } + let tree_config = %{ + "cache_capacity": %c.tree_config.cache_capacity, + "mode": %c.tree_config.mode, + "compression": %c.tree_config.compression, + "flush_every_ms": %c.tree_config.flush_every_ms, + } return %[("resources_folder", %c.resources_folder), ("tree_config", %tree_config)] proc createRLNInstanceLocal(): RLNResult = From 85a7bf3322bb9087a3983288117eabbb5e322da2 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:39:00 +0100 Subject: [PATCH 03/19] chore: add .nph.toml to exclude vendor and nimbledeps from nph formatting (#3762) Co-authored-by: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> --- .nph.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .nph.toml diff --git a/.nph.toml b/.nph.toml new file mode 100644 index 000000000..f0398059c --- /dev/null +++ b/.nph.toml @@ -0,0 +1,4 @@ +extend-exclude = [ + "vendor", + "nimbledeps", +] From 11461aed44e777ffde5655405c8fbce9c0c2e9c3 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Thu, 19 Mar 2026 21:37:04 +0100 Subject: [PATCH 04/19] allow union of several retention policies (#3766) * refactor retention policy to allow union of several retention policies * bug fix time retention policy * add removal of orphan partitions if any * use nim-http-utils 0.4.1 --- tests/api/test_node_conf.nim | 65 ++++++++++++- tests/wakunode2/test_cli_args.nim | 81 +++++++++++++++- tools/confutils/cli_args.nim | 4 +- vendor/nim-http-utils | 2 +- .../store_service_conf_builder.nim | 50 +++++++++- waku/factory/node_factory.nim | 4 +- waku/factory/waku_conf.nim | 2 +- waku/node/kernel_api/store.nim | 4 +- waku/waku_archive/archive.nim | 33 ++++--- .../postgres_driver/postgres_driver.nim | 95 ++++++++++++++++--- waku/waku_archive/retention_policy.nim | 3 + .../waku_archive/retention_policy/builder.nim | 13 ++- .../retention_policy_capacity.nim | 3 + .../retention_policy_size.nim | 3 + .../retention_policy_time.nim | 16 +--- 15 files changed, 323 insertions(+), 55 deletions(-) diff --git a/tests/api/test_node_conf.nim b/tests/api/test_node_conf.nim index 232ffc7d2..255dec8b2 100644 --- a/tests/api/test_node_conf.nim +++ b/tests/api/test_node_conf.nim @@ -1,7 +1,11 @@ {.used.} -import std/options, results, stint, testutils/unittests -import waku/api/api_conf, waku/factory/waku_conf, waku/factory/networks_config +import std/[options, strutils], results, stint, testutils/unittests, chronos +import + waku/api/api_conf, + waku/factory/waku_conf, + waku/factory/networks_config, + waku/factory/conf_builder/conf_builder suite "LibWaku Conf - toWakuConf": test "Minimal configuration": @@ -277,3 +281,60 @@ suite "LibWaku Conf - toWakuConf": check: wakuConf.staticNodes.len == 1 wakuConf.staticNodes[0] == entryNodes[1] + +suite "WakuConfBuilder - store retention policies": + test "Multiple retention policies": + ## Given + var b = WakuConfBuilder.init() + b.storeServiceConf.withEnabled(true) + b.storeServiceConf.withDbUrl("sqlite://test.db") + b.storeServiceConf.withRetentionPolicies(@["time:86400", "capacity:10000"]) + + ## When + let wakuConf = b.build().valueOr: + raiseAssert error + + ## Then + require wakuConf.storeServiceConf.isSome() + let storeConf = wakuConf.storeServiceConf.get() + check storeConf.retentionPolicies == @["time:86400", "capacity:10000"] + + test "Duplicated retention policies returns error": + ## Given + var b = WakuConfBuilder.init() + b.storeServiceConf.withEnabled(true) + b.storeServiceConf.withDbUrl("sqlite://test.db") + b.storeServiceConf.withRetentionPolicies(@["time:86400", "time:800", "capacity:10000"]) + + ## When + let wakuConfRes = b.build() + + ## Then + check wakuConfRes.isErr() + check wakuConfRes.error.contains("duplicated retention policy type") + + test "Incorrect retention policy type returns error": + ## Given + var b = WakuConfBuilder.init() + b.storeServiceConf.withEnabled(true) + b.storeServiceConf.withDbUrl("sqlite://test.db") + b.storeServiceConf.withRetentionPolicies(@["capaity:10000"]) + + ## When + let wakuConfRes = b.build() + + ## Then + check wakuConfRes.isErr() + check wakuConfRes.error.contains("unknown retention policy type") + + test "Store disabled - no retention policy applied": + ## Given + var b = WakuConfBuilder.init() + # storeServiceConf not enabled + + ## When + let wakuConf = b.build().valueOr: + raiseAssert error + + ## Then + check wakuConf.storeServiceConf.isNone() diff --git a/tests/wakunode2/test_cli_args.nim b/tests/wakunode2/test_cli_args.nim index dabc78083..5108b4a9d 100644 --- a/tests/wakunode2/test_cli_args.nim +++ b/tests/wakunode2/test_cli_args.nim @@ -1,7 +1,7 @@ {.used.} import - std/options, + std/[options, strutils], testutils/unittests, chronos, libp2p/crypto/[crypto, secp], @@ -261,6 +261,85 @@ suite "Waku external config - Shards": ## Then assert res.isErr(), "Invalid shard was accepted" +suite "Waku external config - store retention policy": + test "Default retention policy": + ## Given + var conf = defaultWakuNodeConf().get() + conf.store = true + conf.storeMessageDbUrl = "sqlite://test.db" + # storeMessageRetentionPolicy keeps its default: "time:<2 days in seconds>" + + ## When + let res = conf.toWakuConf() + + ## Then + assert res.isOk(), $res.error + let wakuConf = res.get() + require wakuConf.storeServiceConf.isSome() + check wakuConf.storeServiceConf.get().retentionPolicies == + @["time:" & $2.days.seconds] + + test "Single custom retention policy": + ## Given + var conf = defaultWakuNodeConf().get() + conf.store = true + conf.storeMessageDbUrl = "sqlite://test.db" + conf.storeMessageRetentionPolicy = "capacity:50000" + + ## When + let res = conf.toWakuConf() + + ## Then + assert res.isOk(), $res.error + let wakuConf = res.get() + require wakuConf.storeServiceConf.isSome() + check wakuConf.storeServiceConf.get().retentionPolicies == @["capacity:50000"] + + test "Retention policies with whitespace around semicolons and colons": + ## Given + var conf = defaultWakuNodeConf().get() + conf.store = true + conf.storeMessageDbUrl = "sqlite://test.db" + conf.storeMessageRetentionPolicy = "time:3600 ; capacity:10000 ; size : 30GB" + + ## When + let res = conf.toWakuConf() + + ## Then + assert res.isOk(), $res.error + let wakuConf = res.get() + require wakuConf.storeServiceConf.isSome() + check wakuConf.storeServiceConf.get().retentionPolicies == + @["time:3600", "capacity:10000", "size:30GB"] + + test "Invalid retention policy type returns error": + ## Given + var conf = defaultWakuNodeConf().get() + conf.store = true + conf.storeMessageDbUrl = "sqlite://test.db" + conf.storeMessageRetentionPolicy = "foo:1234" + + ## When + let res = conf.toWakuConf() + + ## Then + check res.isErr() + check res.error.contains("unknown retention policy type") + + test "Duplicated retention policy type returns error": + ## Given + var conf = defaultWakuNodeConf().get() + conf.store = true + conf.storeMessageDbUrl = "sqlite://test.db" + conf.storeMessageRetentionPolicy = "time:3600;time:7200;capacity:10000" + + ## When + let res = conf.toWakuConf() + + ## Then + check res.isErr() + check res.error.contains("duplicated retention policy type") + suite "Waku external config - http url parsing": test "Basic HTTP URLs without authentication": check string(parseCmdArg(EthRpcUrl, "https://example.com/path")) == diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim index e6b3fc97d..994bc6a2d 100644 --- a/tools/confutils/cli_args.nim +++ b/tools/confutils/cli_args.nim @@ -347,7 +347,7 @@ hence would have reachability issues.""", storeMessageRetentionPolicy* {. desc: - "Message store retention policy. Time retention policy: 'time:'. Capacity retention policy: 'capacity:'. Size retention policy: 'size:'. Set to 'none' to disable.", + "Message store retention policy. Multiple policies may be provided as a semicolon-separated string and are applied as a union. Time retention policy: 'time:'. Capacity retention policy: 'capacity:'. Size retention policy: 'size:'. Set to 'none' to disable. Example: 'time:3600;size:1GB;capacity:100'.", defaultValue: "time:" & $2.days.seconds, name: "store-message-retention-policy" .}: string @@ -991,7 +991,7 @@ proc toWakuConf*(n: WakuNodeConf): ConfResult[WakuConf] = b.storeServiceConf.withEnabled(n.store) b.storeServiceConf.withSupportV2(n.legacyStore) - b.storeServiceConf.withRetentionPolicy(n.storeMessageRetentionPolicy) + b.storeServiceConf.withRetentionPolicies(n.storeMessageRetentionPolicy) b.storeServiceConf.withDbUrl(n.storeMessageDbUrl) b.storeServiceConf.withDbVacuum(n.storeMessageDbVacuum) b.storeServiceConf.withDbMigration(n.storeMessageDbMigration) diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils index 08db60946..f142cb2e8 160000 --- a/vendor/nim-http-utils +++ b/vendor/nim-http-utils @@ -1 +1 @@ -Subproject commit 08db609467a0e2b5a6e8ce118bb83dba7a8a9375 +Subproject commit f142cb2e8bd812dd002a6493b6082827bb248592 diff --git a/waku/factory/conf_builder/store_service_conf_builder.nim b/waku/factory/conf_builder/store_service_conf_builder.nim index d5d48c34d..30c743e01 100644 --- a/waku/factory/conf_builder/store_service_conf_builder.nim +++ b/waku/factory/conf_builder/store_service_conf_builder.nim @@ -1,4 +1,5 @@ -import chronicles, std/options, results, chronos +import std/[options, strutils, sequtils] +import chronicles, results, chronos import ../waku_conf, ./store_sync_conf_builder logScope: @@ -15,7 +16,7 @@ type StoreServiceConfBuilder* = object dbVacuum*: Option[bool] supportV2*: Option[bool] maxNumDbConnections*: Option[int] - retentionPolicy*: Option[string] + retentionPolicies*: seq[string] resume*: Option[bool] storeSyncConf*: StoreSyncConfBuilder @@ -42,12 +43,43 @@ proc withMaxNumDbConnections*( ) = b.maxNumDbConnections = some(maxNumDbConnections) -proc withRetentionPolicy*(b: var StoreServiceConfBuilder, retentionPolicy: string) = - b.retentionPolicy = some(retentionPolicy) +proc withRetentionPolicies*(b: var StoreServiceConfBuilder, retentionPolicies: string) = + b.retentionPolicies = retentionPolicies + .multiReplace((" ", ""), ("\t", "")) + .split(";") + .mapIt(it.strip()) + .filterIt(it.len > 0) proc withResume*(b: var StoreServiceConfBuilder, resume: bool) = b.resume = some(resume) +const ValidRetentionPolicyTypes = ["time", "capacity", "size"] + +proc validateRetentionPolicies(policies: seq[string]): Result[void, string] = + var seen: seq[string] + + for p in policies: + let policy = p.multiReplace((" ", ""), ("\t", "")) + let parts = policy.split(":", 1) + if parts.len != 2 or parts[1] == "": + return err( + "invalid retention policy format: '" & policy & "', expected ':'" + ) + + let policyType = parts[0].toLowerAscii() + if policyType notin ValidRetentionPolicyTypes: + return err( + "unknown retention policy type: '" & policyType & + "', valid types are: time, capacity, size" + ) + + if policyType in seen: + return err("duplicated retention policy type: '" & policyType & "'") + + seen.add(policyType) + + return ok() + proc build*(b: StoreServiceConfBuilder): Result[Option[StoreServiceConf], string] = if not b.enabled.get(false): return ok(none(StoreServiceConf)) @@ -58,6 +90,14 @@ proc build*(b: StoreServiceConfBuilder): Result[Option[StoreServiceConf], string let storeSyncConf = b.storeSyncConf.build().valueOr: return err("Store Sync Conf failed to build") + let retentionPolicies = + if b.retentionPolicies.len == 0: + @["time:" & $2.days.seconds] + else: + validateRetentionPolicies(b.retentionPolicies).isOkOr: + return err("invalid retention policies: " & error) + b.retentionPolicies + return ok( some( StoreServiceConf( @@ -66,7 +106,7 @@ proc build*(b: StoreServiceConfBuilder): Result[Option[StoreServiceConf], string dbVacuum: b.dbVacuum.get(false), supportV2: b.supportV2.get(false), maxNumDbConnections: b.maxNumDbConnections.get(50), - retentionPolicy: b.retentionPolicy.get("time:" & $2.days.seconds), + retentionPolicies: retentionPolicies, resume: b.resume.get(false), storeSyncConf: storeSyncConf, ) diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index 488d07c06..86ab75d77 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -206,10 +206,10 @@ proc setupProtocols( ).valueOr: return err("failed to setup archive driver: " & error) - let retPolicy = policy.RetentionPolicy.new(storeServiceConf.retentionPolicy).valueOr: + let retPolicies = policy.RetentionPolicy.new(storeServiceConf.retentionPolicies).valueOr: return err("failed to create retention policy: " & error) - node.mountArchive(archiveDriver, retPolicy).isOkOr: + node.mountArchive(archiveDriver, retPolicies).isOkOr: return err("failed to mount waku archive protocol: " & error) if storeServiceConf.supportV2: diff --git a/waku/factory/waku_conf.nim b/waku/factory/waku_conf.nim index 89ffb366c..3b99d1070 100644 --- a/waku/factory/waku_conf.nim +++ b/waku/factory/waku_conf.nim @@ -57,7 +57,7 @@ type StoreServiceConf* {.requiresInit.} = object dbVacuum*: bool supportV2*: bool maxNumDbConnections*: int - retentionPolicy*: string + retentionPolicies*: seq[string] resume*: bool storeSyncConf*: Option[StoreSyncConf] diff --git a/waku/node/kernel_api/store.nim b/waku/node/kernel_api/store.nim index 7edae7966..ca9917163 100644 --- a/waku/node/kernel_api/store.nim +++ b/waku/node/kernel_api/store.nim @@ -39,10 +39,10 @@ logScope: proc mountArchive*( node: WakuNode, driver: waku_archive.ArchiveDriver, - retentionPolicy = none(waku_archive.RetentionPolicy), + retentionPolicies = newSeq[waku_archive.RetentionPolicy](), ): Result[void, string] = node.wakuArchive = waku_archive.WakuArchive.new( - driver = driver, retentionPolicy = retentionPolicy + driver = driver, retentionPolicies = retentionPolicies ).valueOr: return err("error in mountArchive: " & error) diff --git a/waku/waku_archive/archive.nim b/waku/waku_archive/archive.nim index 707c757a3..4c4ac4bba 100644 --- a/waku/waku_archive/archive.nim +++ b/waku/waku_archive/archive.nim @@ -45,7 +45,7 @@ type WakuArchive* = ref object validator: MessageValidator - retentionPolicy: Option[RetentionPolicy] + retentionPolicies: seq[RetentionPolicy] retentionPolicyHandle: Future[void] metricsHandle: Future[void] @@ -72,13 +72,17 @@ proc new*( T: type WakuArchive, driver: ArchiveDriver, validator: MessageValidator = validate, - retentionPolicy = none(RetentionPolicy), + retentionPolicies = newSeq[RetentionPolicy](0), ): Result[T, string] = if driver.isNil(): return err("archive driver is Nil") - let archive = - WakuArchive(driver: driver, validator: validator, retentionPolicy: retentionPolicy) + if retentionPolicies.len == 0: + return err("at least one retention policy must be provided") + + let archive = WakuArchive( + driver: driver, validator: validator, retentionPolicies: retentionPolicies + ) return ok(archive) @@ -253,16 +257,15 @@ proc findMessages*( ) proc periodicRetentionPolicy(self: WakuArchive) {.async.} = - let policy = self.retentionPolicy.get() - while true: - info "executing message retention policy" - (await policy.execute(self.driver)).isOkOr: - waku_archive_errors.inc(labelValues = [retPolicyFailure]) - error "failed execution of retention policy", error = error - await sleepAsync(WakuArchiveDefaultRetentionPolicyIntervalWhenError) - ## in case of error, let's try again faster - continue + for policy in self.retentionPolicies: + info "executing message retention policy", policy = $policy + (await policy.execute(self.driver)).isOkOr: + waku_archive_errors.inc(labelValues = [retPolicyFailure]) + error "failed execution of retention policy", policy = $policy, error = error + await sleepAsync(WakuArchiveDefaultRetentionPolicyIntervalWhenError) + ## in case of error, let's try again faster + continue await sleepAsync(WakuArchiveDefaultRetentionPolicyInterval) @@ -279,7 +282,7 @@ proc periodicMetricReport(self: WakuArchive) {.async.} = await sleepAsync(WakuArchiveDefaultMetricsReportInterval) proc start*(self: WakuArchive) = - if self.retentionPolicy.isSome(): + if self.retentionPolicies.len > 0: self.retentionPolicyHandle = self.periodicRetentionPolicy() self.metricsHandle = self.periodicMetricReport() @@ -287,7 +290,7 @@ proc start*(self: WakuArchive) = proc stopWait*(self: WakuArchive) {.async.} = var futures: seq[Future[void]] - if self.retentionPolicy.isSome() and not self.retentionPolicyHandle.isNil(): + if not self.retentionPolicyHandle.isNil(): futures.add(self.retentionPolicyHandle.cancelAndWait()) if not self.metricsHandle.isNil: diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index 842d7cbc2..c6e50d0dd 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -16,6 +16,9 @@ import ./postgres_healthcheck, ./partitions_manager +logScope: + topics = "postgres driver" + type PostgresDriver* = ref object of ArchiveDriver ## Establish a separate pools for read/write operations writeConnPool: PgAsyncPool @@ -367,6 +370,7 @@ proc getPartitionsList( ): Future[ArchiveDriverResult[seq[string]]] {.async.} = ## Retrieves the seq of partition table names. ## e.g: @["messages_1708534333_1708534393", "messages_1708534273_1708534333"] + ## This returns the partitions that are attached to the main messages table. var partitions: seq[string] proc rowCallback(pqResult: ptr PGresult) = for iRow in 0 ..< pqResult.pqNtuples(): @@ -393,6 +397,49 @@ proc getPartitionsList( return ok(partitions) +## fwd declaration. The implementation is below. +proc dropPartition( + self: PostgresDriver, partitionName: string +): Future[ArchiveDriverResult[void]] {.async.} + +proc dropOrphanPartitions( + s: PostgresDriver +): Future[ArchiveDriverResult[void]] {.async.} = + ## Tries to remove partitions that weren't correctly removed during retention policy execution. + ## Orphan partition is a partition that is not attached to the main messages table. + ## Therefore, it is not used for queries and can be safely removed. + var partitions: seq[string] + proc rowCallback(pqResult: ptr PGresult) = + for iRow in 0 ..< pqResult.pqNtuples(): + let partitionName = $(pqgetvalue(pqResult, iRow, 0)) + partitions.add(partitionName) + + ( + await s.readConnPool.pgQuery( + """ + SELECT c.relname AS partition_name + FROM pg_class c + LEFT JOIN pg_inherits i ON i.inhrelid = c.oid + WHERE c.relname LIKE 'messages_%' + AND c.relname != 'messages_lookup' + AND c.relkind = 'r' -- only regular tables + AND i.inhrelid IS NULL -- detached partition + ORDER BY partition_name + """, + newSeq[string](0), + rowCallback, + ) + ).isOkOr: + return err("dropOrphanPartitions failed in query: " & $error) + + for partition in partitions: + info "orphan partition found", partitionName = partition + (await s.dropPartition(partition)).isOkOr: + error "failed to drop orphan partition", partitionName = partition, error = $error + continue + + return ok() + proc getTimeCursor( s: PostgresDriver, hashHex: string ): Future[ArchiveDriverResult[Option[Timestamp]]] {.async.} = @@ -1259,11 +1306,18 @@ proc loopPartitionFactory( self: PostgresDriver, onFatalError: OnFatalErrorHandler ) {.async.} = ## Loop proc that continuously checks whether we need to create a new partition. - ## Notice that the deletion of partitions is handled by the retention policy modules. + ## Notice that the deletion of partitions is mostly handled by the retention policy modules. + ## This loop only removes orphan partitions which were detached but not properly removed by the + ## retention policy module due to some error. However, the main task of this loop is to create + ## new partitions when needed. info "starting loopPartitionFactory" while true: + trace "loopPartitionFactory iteration started" + (await self.dropOrphanPartitions()).isOkOr: + onFatalError("error when dropping orphan partitions: " & $error) + trace "Check if a new partition is needed" ## Let's make the 'partition_manager' aware of the current partitions @@ -1321,14 +1375,24 @@ proc getTableSize*( return ok(tableSize) -proc removePartition( +proc dropPartition( + self: PostgresDriver, partitionName: string +): Future[ArchiveDriverResult[void]] {.async.} = + let dropPartitionQuery = "DROP TABLE " & partitionName + info "drop partition", query = dropPartitionQuery + (await self.performWriteQuery(dropPartitionQuery)).isOkOr: + return err(fmt"error in dropPartition: {dropPartitionQuery}: " & $error) + + return ok() + +proc detachAndDropPartition( self: PostgresDriver, partition: Partition ): Future[ArchiveDriverResult[void]] {.async.} = - ## Removes the desired partition and also removes the rows from messages_lookup table + ## Detaches and drops the desired partition and also removes the rows from messages_lookup table ## whose rows belong to the partition time range let partitionName = partition.getName() - info "beginning of removePartition", partitionName + info "beginning of detachAndDropPartition", partitionName let partSize = (await self.getTableSize(partitionName)).valueOr("") @@ -1351,11 +1415,8 @@ proc removePartition( else: return err(fmt"error in {detachPartitionQuery}: " & $error) - ## Drop the partition - let dropPartitionQuery = "DROP TABLE " & partitionName - info "removeOldestPartition drop partition", query = dropPartitionQuery - (await self.performWriteQuery(dropPartitionQuery)).isOkOr: - return err(fmt"error in {dropPartitionQuery}: " & $error) + ## Drop partition + ?(await self.dropPartition(partitionName)) info "removed partition", partition_name = partitionName, partition_size = partSize self.partitionMngr.removeOldestPartitionName() @@ -1380,8 +1441,18 @@ proc removePartitionsOlderThan( var oldestPartition = self.partitionMngr.getOldestPartition().valueOr: return err("could not get oldest partition in removePartitionOlderThan: " & $error) - while not oldestPartition.containsMoment(tsInSec): - (await self.removePartition(oldestPartition)).isOkOr: + debug "oldest partition info", + partitionName = oldestPartition.getName(), + partitionLastMoment = oldestPartition.getLastMoment(), + tsInSec + + while oldestPartition.getLastMoment() < tsInSec: + info "start removing partition whose first record is older than the specified timestamp", + partitionName = oldestPartition.getName(), + partitionFirstMoment = oldestPartition.getLastMoment(), + tsInSec + + (await self.detachAndDropPartition(oldestPartition)).isOkOr: return err("issue in removePartitionsOlderThan: " & $error) oldestPartition = self.partitionMngr.getOldestPartition().valueOr: @@ -1409,7 +1480,7 @@ proc removeOldestPartition( info "Skipping to remove the current partition" return ok() - return await self.removePartition(oldestPartition) + return await self.detachAndDropPartition(oldestPartition) proc containsAnyPartition*(self: PostgresDriver): bool = return not self.partitionMngr.isEmpty() diff --git a/waku/waku_archive/retention_policy.nim b/waku/waku_archive/retention_policy.nim index d4b75ee1f..c2663fb66 100644 --- a/waku/waku_archive/retention_policy.nim +++ b/waku/waku_archive/retention_policy.nim @@ -11,3 +11,6 @@ method execute*( p: RetentionPolicy, store: ArchiveDriver ): Future[RetentionPolicyResult[void]] {.base, async.} = discard + +method `$`*(p: RetentionPolicy): string {.base, gcsafe.} = + "unknown retention policy" diff --git a/waku/waku_archive/retention_policy/builder.nim b/waku/waku_archive/retention_policy/builder.nim index 6cb131bbc..7e777f4a0 100644 --- a/waku/waku_archive/retention_policy/builder.nim +++ b/waku/waku_archive/retention_policy/builder.nim @@ -7,7 +7,7 @@ import ./retention_policy_capacity, ./retention_policy_size -proc new*( +proc new( T: type RetentionPolicy, retPolicy: string ): RetentionPolicyResult[Option[RetentionPolicy]] = let retPolicy = retPolicy.toLower @@ -83,3 +83,14 @@ proc new*( return ok(some(retPolicy)) else: return err("unknown retention policy") + +proc new*( + T: typedesc[RetentionPolicy], retPolicies: seq[string] +): RetentionPolicyResult[seq[RetentionPolicy]] = + var policies: seq[RetentionPolicy] + for retPolicy in retPolicies: + let policy = RetentionPolicy.new(retPolicy).valueOr: + return err(error) + if policy.isSome(): + policies.add(policy.get()) + return ok(policies) diff --git a/waku/waku_archive/retention_policy/retention_policy_capacity.nim b/waku/waku_archive/retention_policy/retention_policy_capacity.nim index ed4dd2339..ff4da6861 100644 --- a/waku/waku_archive/retention_policy/retention_policy_capacity.nim +++ b/waku/waku_archive/retention_policy/retention_policy_capacity.nim @@ -50,6 +50,9 @@ proc new*(T: type CapacityRetentionPolicy, capacity = DefaultCapacity): T = capacity: capacity, totalCapacity: totalCapacity, deleteWindow: deleteWindow ) +method `$`*(p: CapacityRetentionPolicy): string = + "capacity:" & $p.capacity + method execute*( p: CapacityRetentionPolicy, driver: ArchiveDriver ): Future[RetentionPolicyResult[void]] {.async.} = diff --git a/waku/waku_archive/retention_policy/retention_policy_size.nim b/waku/waku_archive/retention_policy/retention_policy_size.nim index e60aba303..416d95ec0 100644 --- a/waku/waku_archive/retention_policy/retention_policy_size.nim +++ b/waku/waku_archive/retention_policy/retention_policy_size.nim @@ -15,6 +15,9 @@ type SizeRetentionPolicy* = ref object of RetentionPolicy proc new*(T: type SizeRetentionPolicy, size = DefaultRetentionSize): T = SizeRetentionPolicy(sizeLimit: size) +method `$`*(p: SizeRetentionPolicy): string = + "size:" & $p.sizeLimit & "b" + method execute*( p: SizeRetentionPolicy, driver: ArchiveDriver ): Future[RetentionPolicyResult[void]] {.async.} = diff --git a/waku/waku_archive/retention_policy/retention_policy_time.nim b/waku/waku_archive/retention_policy/retention_policy_time.nim index 6d4c0815a..12f056c7b 100644 --- a/waku/waku_archive/retention_policy/retention_policy_time.nim +++ b/waku/waku_archive/retention_policy/retention_policy_time.nim @@ -6,29 +6,23 @@ import ../../waku_core, ../driver, ../retention_policy logScope: topics = "waku archive retention_policy" -const DefaultRetentionTime*: int64 = 30.days.seconds - type TimeRetentionPolicy* = ref object of RetentionPolicy retentionTime: chronos.Duration -proc new*(T: type TimeRetentionPolicy, retentionTime = DefaultRetentionTime): T = +proc new*(T: type TimeRetentionPolicy, retentionTime: int64): T = TimeRetentionPolicy(retentionTime: retentionTime.seconds) +method `$`*(p: TimeRetentionPolicy): string = + "time:" & $p.retentionTime.seconds + method execute*( p: TimeRetentionPolicy, driver: ArchiveDriver ): Future[RetentionPolicyResult[void]] {.async.} = - ## Delete messages that exceed the retention time by 10% and more (batch delete for efficiency) + ## Delete messages that exceed the retention time info "beginning of executing message retention policy - time" - let omt = (await driver.getOldestMessageTimestamp()).valueOr: - return err("failed to get oldest message timestamp: " & error) - let now = getNanosecondTime(getTime().toUnixFloat()) let retentionTimestamp = now - p.retentionTime.nanoseconds - let thresholdTimestamp = retentionTimestamp - p.retentionTime.nanoseconds div 10 - - if thresholdTimestamp <= omt: - return ok() (await driver.deleteMessagesOlderThanTimestamp(ts = retentionTimestamp)).isOkOr: return err("failed to delete oldest messages: " & error) From 84e0af8dc0ff9bb505889c24c93cba131eb4fb41 Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Thu, 19 Mar 2026 23:09:22 +0100 Subject: [PATCH 05/19] update changelog for v0.37.2 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index edc4a705c..a5fefbc3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.37.2 (2026-03-19) + +### Features + +- Allow union of several retention policies ([#3766](https://github.com/logos-messaging/logos-delivery/pull/3766)) + +### Bug Fixes + +- Bump nim-http-utils to v0.4.1 to allow accepting <:><(> as a valid header and tests to validate html rfc7230 ([#43](https://github.com/status-im/nim-http-utils/pull/43)) + ## v0.37.1 (2026-03-12) ### Bug Fixes From 3e7aa18a42067a7bb3aa3f9724f02295712c132f Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Tue, 17 Feb 2026 00:18:46 +0100 Subject: [PATCH 06/19] force FINALIZE partition detach after detecting shorter error (#3728) --- .../waku_archive/driver/postgres_driver/postgres_driver.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index c6e50d0dd..64ede6cf5 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -1403,8 +1403,10 @@ proc detachAndDropPartition( (await self.performWriteQuery(detachPartitionQuery)).isOkOr: info "detected error when trying to detach partition", error - if ($error).contains("FINALIZE") or - ($error).contains("already pending detach in part"): + if ($error).contains("FINALIZE") or ($error).contains("already pending"): + ## We assume "already pending detach in partitioned table ..." as possible error + debug "enforce detach with FINALIZE because of detected error", error + ## We assume the database is suggesting to use FINALIZE when detaching a partition let detachPartitionFinalizeQuery = "ALTER TABLE messages DETACH PARTITION " & partitionName & " FINALIZE;" From 4e17776f71caf12917549988cfe5fe2bdf448250 Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Fri, 20 Mar 2026 00:18:26 +0100 Subject: [PATCH 07/19] update change log for v0.37.2 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5fefbc3d..45f9e2a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Bug Fixes - Bump nim-http-utils to v0.4.1 to allow accepting <:><(> as a valid header and tests to validate html rfc7230 ([#43](https://github.com/status-im/nim-http-utils/pull/43)) +- Force FINALIZE partition detach after detecting shorter error ([#3728](https://github.com/logos-messaging/logos-delivery/pull/3766)) ## v0.37.1 (2026-03-12) From d9aa46e22f958def8df1a538331552bdd5911bd7 Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Fri, 20 Mar 2026 16:54:42 +0100 Subject: [PATCH 08/19] fix compilation issue in test_node_conf.nim --- tests/api/test_node_conf.nim | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/api/test_node_conf.nim b/tests/api/test_node_conf.nim index 342fe809d..b19739393 100644 --- a/tests/api/test_node_conf.nim +++ b/tests/api/test_node_conf.nim @@ -1,17 +1,14 @@ {.used.} -import std/[options, strutils], results, stint, testutils/unittests, chronos +import std/[options, json, strutils], results, stint, testutils/unittests +import json_serialization, confutils, confutils/std/net import + tools/confutils/cli_args, waku/api/api_conf, waku/factory/waku_conf, waku/factory/networks_config, - waku/factory/conf_builder/conf_builder -import std/[options, json, strutils], results, stint, testutils/unittests -import json_serialization -import confutils, confutils/std/net -import tools/confutils/cli_args -import waku/factory/waku_conf, waku/factory/networks_config -import waku/common/logging + waku/factory/conf_builder/conf_builder, + waku/common/logging # Helper: parse JSON into WakuNodeConf using fieldPairs (same as liblogosdelivery) proc parseWakuNodeConfFromJson(jsonStr: string): Result[WakuNodeConf, string] = @@ -81,7 +78,7 @@ suite "WakuNodeConf - mode-driven toWakuConf": ## Given var conf = defaultWakuNodeConf().valueOr: raiseAssert error - conf.mode = WakuMode.noMode + conf.mode = cli_args.WakuMode.noMode conf.relay = true conf.lightpush = false conf.clusterId = 5 @@ -124,7 +121,7 @@ suite "WakuNodeConf - JSON parsing with fieldPairs": require confRes.isOk() let conf = confRes.get() check: - conf.mode == WakuMode.noMode + conf.mode == cli_args.WakuMode.noMode conf.clusterId == 0 conf.logLevel == logging.LogLevel.INFO @@ -381,7 +378,9 @@ suite "WakuConfBuilder - store retention policies": var b = WakuConfBuilder.init() b.storeServiceConf.withEnabled(true) b.storeServiceConf.withDbUrl("sqlite://test.db") - b.storeServiceConf.withRetentionPolicies(@["time:86400", "capacity:10000"]) + b.storeServiceConf.withRetentionPolicies( + "time:86400 ; capacity:10000; size : 50GB" + ) ## When let wakuConf = b.build().valueOr: @@ -390,14 +389,14 @@ suite "WakuConfBuilder - store retention policies": ## Then require wakuConf.storeServiceConf.isSome() let storeConf = wakuConf.storeServiceConf.get() - check storeConf.retentionPolicies == @["time:86400", "capacity:10000"] + check storeConf.retentionPolicies == @["time:86400", "capacity:10000", "size:50GB"] test "Duplicated retention policies returns error": ## Given var b = WakuConfBuilder.init() b.storeServiceConf.withEnabled(true) b.storeServiceConf.withDbUrl("sqlite://test.db") - b.storeServiceConf.withRetentionPolicies(@["time:86400", "time:800", "capacity:10000"]) + b.storeServiceConf.withRetentionPolicies("time:86400;time:800;capacity:10000") ## When let wakuConfRes = b.build() @@ -409,7 +408,7 @@ suite "WakuConfBuilder - store retention policies": var b = WakuConfBuilder.init() b.storeServiceConf.withEnabled(true) b.storeServiceConf.withDbUrl("sqlite://test.db") - b.storeServiceConf.withRetentionPolicies(@["capaity:10000"]) + b.storeServiceConf.withRetentionPolicies("capaity:10000") ## When let wakuConfRes = b.build() From de3143e3515a093eaaf04614e1d2eff0db87032d Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Fri, 20 Mar 2026 21:05:42 +0100 Subject: [PATCH 09/19] set default retention policy in archive.nim --- waku/waku_archive/archive.nim | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/waku/waku_archive/archive.nim b/waku/waku_archive/archive.nim index 95c1a905d..976d7d035 100644 --- a/waku/waku_archive/archive.nim +++ b/waku/waku_archive/archive.nim @@ -14,7 +14,8 @@ import ../waku_core, ../waku_core/message/digest, ./common, - ./archive_metrics + ./archive_metrics, + waku/waku_archive/retention_policy/retention_policy_time logScope: topics = "waku archive" @@ -82,14 +83,11 @@ proc new*( T: type WakuArchive, driver: ArchiveDriver, validator: MessageValidator = validate, - retentionPolicies = newSeq[RetentionPolicy](0), + retentionPolicies = @[RetentionPolicy(TimeRetentionPolicy.new(2.days.seconds))], ): Result[T, string] = if driver.isNil(): return err("archive driver is Nil") - if retentionPolicies.len == 0: - return err("at least one retention policy must be provided") - let archive = WakuArchive( driver: driver, validator: validator, retentionPolicies: retentionPolicies ) From 0b86093247da92060c503544d39e5d0a23922c15 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:23:20 +0100 Subject: [PATCH 10/19] allow override user-message-rate-limit (#3778) --- tests/factory/test_waku_conf.nim | 10 +++-- tests/wakunode2/test_cli_args.nim | 37 ------------------- tools/confutils/cli_args.nim | 12 +++--- .../conf_builder/waku_conf_builder.nim | 7 +++- 4 files changed, 18 insertions(+), 48 deletions(-) diff --git a/tests/factory/test_waku_conf.nim b/tests/factory/test_waku_conf.nim index 3d3fec20e..9d05f7fb5 100644 --- a/tests/factory/test_waku_conf.nim +++ b/tests/factory/test_waku_conf.nim @@ -4,7 +4,7 @@ import libp2p/crypto/[crypto, secp], libp2p/multiaddress, nimcrypto/utils, - std/[options, sequtils], + std/[options, random, sequtils], results, testutils/unittests import @@ -22,11 +22,13 @@ suite "Waku Conf - build with cluster conf": builder.withRelayServiceRatio("50:50") # Mount all shards in network let expectedShards = toSeq[0.uint16 .. 7.uint16] + let userMessageLimit = rand(1 .. 1000).uint64 ## Given builder.rlnRelayConf.withEthClientUrls(@["https://my_eth_rpc_url/"]) builder.withNetworkConf(networkConf) builder.withRelay(true) + builder.rlnRelayConf.withUserMessageLimit(userMessageLimit) ## When let resConf = builder.build() @@ -54,7 +56,7 @@ suite "Waku Conf - build with cluster conf": check rlnRelayConf.dynamic == networkConf.rlnRelayDynamic check rlnRelayConf.chainId == networkConf.rlnRelayChainId check rlnRelayConf.epochSizeSec == networkConf.rlnEpochSizeSec - check rlnRelayConf.userMessageLimit == networkConf.rlnRelayUserMessageLimit + check rlnRelayConf.userMessageLimit == userMessageLimit.uint test "Cluster Conf is passed, but relay is disabled": ## Setup @@ -174,11 +176,13 @@ suite "Waku Conf - build with cluster conf": # Mount all shards in network let expectedShards = toSeq[0.uint16 .. 7.uint16] let contractAddress = "0x0123456789ABCDEF" + let userMessageLimit = rand(1 .. 1000).uint64 ## Given builder.rlnRelayConf.withEthContractAddress(contractAddress) builder.withNetworkConf(networkConf) builder.withRelay(true) + builder.rlnRelayConf.withUserMessageLimit(userMessageLimit) ## When let resConf = builder.build() @@ -207,7 +211,7 @@ suite "Waku Conf - build with cluster conf": check rlnRelayConf.dynamic == networkConf.rlnRelayDynamic check rlnRelayConf.chainId == networkConf.rlnRelayChainId check rlnRelayConf.epochSizeSec == networkConf.rlnEpochSizeSec - check rlnRelayConf.userMessageLimit == networkConf.rlnRelayUserMessageLimit + check rlnRelayConf.userMessageLimit == userMessageLimit.uint suite "Waku Conf - node key": test "Node key is generated": diff --git a/tests/wakunode2/test_cli_args.nim b/tests/wakunode2/test_cli_args.nim index 5108b4a9d..9197afe02 100644 --- a/tests/wakunode2/test_cli_args.nim +++ b/tests/wakunode2/test_cli_args.nim @@ -141,43 +141,6 @@ suite "Waku external config - apply preset": ## Then assert res.isErr(), "Invalid shard was accepted" - test "Apply TWN preset when cluster id = 1": - ## Setup - let expectedConf = NetworkConf.TheWakuNetworkConf() - - ## Given - let preConfig = WakuNodeConf( - cmd: noCommand, - clusterId: 1.uint16, - relay: true, - ethClientUrls: @["http://someaddress".EthRpcUrl], - ) - - ## When - let res = preConfig.toWakuConf() - assert res.isOk(), $res.error - - ## Then - let conf = res.get() - check conf.maxMessageSizeBytes == - uint64(parseCorrectMsgSize(expectedConf.maxMessageSize)) - check conf.clusterId == expectedConf.clusterId - check conf.rlnRelayConf.isSome() == expectedConf.rlnRelay - if conf.rlnRelayConf.isSome(): - let rlnRelayConf = conf.rlnRelayConf.get() - check rlnRelayConf.ethContractAddress == expectedConf.rlnRelayEthContractAddress - check rlnRelayConf.dynamic == expectedConf.rlnRelayDynamic - check rlnRelayConf.chainId == expectedConf.rlnRelayChainId - check rlnRelayConf.epochSizeSec == expectedConf.rlnEpochSizeSec - check rlnRelayConf.userMessageLimit == expectedConf.rlnRelayUserMessageLimit - check conf.shardingConf.kind == expectedConf.shardingConf.kind - check conf.shardingConf.numShardsInCluster == - expectedConf.shardingConf.numShardsInCluster - check conf.discv5Conf.isSome() == expectedConf.discv5Discovery - if conf.discv5Conf.isSome(): - let discv5Conf = conf.discv5Conf.get() - check discv5Conf.bootstrapNodes == expectedConf.discv5BootstrapNodes - suite "Waku external config - node key": test "Passed node key is used": ## Setup diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim index 17b77816e..541215f76 100644 --- a/tools/confutils/cli_args.nim +++ b/tools/confutils/cli_args.nim @@ -948,6 +948,12 @@ proc toNetworkConf( proc toWakuConf*(n: WakuNodeConf): ConfResult[WakuConf] = var b = WakuConfBuilder.init() + let networkConf = toNetworkConf(n.preset, some(n.clusterId)).valueOr: + return err("Error determining cluster from preset: " & $error) + + if networkConf.isSome(): + b.withNetworkConf(networkConf.get()) + b.withLogLevel(n.logLevel) b.withLogFormat(n.logFormat) @@ -976,12 +982,6 @@ proc toWakuConf*(n: WakuNodeConf): ConfResult[WakuConf] = b.withProtectedShards(n.protectedShards) b.withClusterId(n.clusterId) - let networkConf = toNetworkConf(n.preset, some(n.clusterId)).valueOr: - return err("Error determining cluster from preset: " & $error) - - if networkConf.isSome(): - b.withNetworkConf(networkConf.get()) - b.withAgentString(n.agentString) if n.nodeKey.isSome(): diff --git a/waku/factory/conf_builder/waku_conf_builder.nim b/waku/factory/conf_builder/waku_conf_builder.nim index 2c427918d..956d733d3 100644 --- a/waku/factory/conf_builder/waku_conf_builder.nim +++ b/waku/factory/conf_builder/waku_conf_builder.nim @@ -353,10 +353,13 @@ proc applyNetworkConf(builder: var WakuConfBuilder) = builder.rlnRelayConf.withEpochSizeSec(networkConf.rlnEpochSizeSec) if builder.rlnRelayConf.userMessageLimit.isSome(): - warn "RLN Relay Dynamic was provided alongside a network conf", + warn "RLN Relay User Message Limit was provided alongside a network conf", used = networkConf.rlnRelayUserMessageLimit, discarded = builder.rlnRelayConf.userMessageLimit - builder.rlnRelayConf.withUserMessageLimit(networkConf.rlnRelayUserMessageLimit) + if builder.rlnRelayConf.userMessageLimit.get(0) == 0: + ## only override with the "preset" value if there was not explicit set value + builder.rlnRelayConf.withUserMessageLimit(networkConf.rlnRelayUserMessageLimit) + # End Apply relay parameters case builder.maxMessageSize.kind From b1e1c875343df05352a2c97ce6901dacd00162b8 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Fri, 27 Mar 2026 13:54:06 +0100 Subject: [PATCH 11/19] update changelog for v0.37.3 (#3783) Co-authored-by: darshankabariya --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f9e2a1e..be2b795f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.37.3 (2026-03-25) + +### Features + +- Allow override user-message-rate-limit ([#3778](https://github.com/logos-messaging/logos-delivery/pull/3778)) + ## v0.37.2 (2026-03-19) ### Features From 5c335c20022704e9230f487727adb8f3c51d998c Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Fri, 27 Mar 2026 13:55:27 +0100 Subject: [PATCH 12/19] address leftover comments (#3782) --- .github/ISSUE_TEMPLATE/deploy_release.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/deploy_release.md b/.github/ISSUE_TEMPLATE/deploy_release.md index 68557bf46..9b9a4f32c 100644 --- a/.github/ISSUE_TEMPLATE/deploy_release.md +++ b/.github/ISSUE_TEMPLATE/deploy_release.md @@ -8,7 +8,7 @@ assignees: '' --- ### Link to the Release PR @@ -20,17 +20,22 @@ Kindly add a link to the release PR where we have a sign-off from QA. At this ti ### Items to complete, in order - [ ] Receive sign-off from DST. - [ ] Inform DST team about what are the expectations for this release. For example, if we expect higher, same or lower bandwidth consumption. Or a new protocol appears, etc. - [ ] Ask DST to add a comment approving this deployment and add a link to the analysis report. -- [ ] Update waku.sandbox with [this deployment job](https://ci.infra.status.im/job/nim-waku/job/deploy-waku-sandbox/). +- [ ] Deploy to waku.sandbox + - [ ] Coordinate with Infra Team about possible changes in CI behavior + - [ ] Update waku.sandbox with [this deployment job](https://ci.infra.status.im/job/nim-waku/job/deploy-waku-sandbox/). - [ ] Deploy to status.prod + - [ ] Coordinate with Infra Team about possible changes in CI behavior - [ ] Ask Status admin to add a comment approving that this deployment to happen now. - [ ] Update status.prod with [this deployment job](https://ci.infra.status.im/job/nim-waku/job/deploy-status-prod/). @@ -42,7 +47,7 @@ For status.prod it is crucial to coordinate such deployment with status friends. - [Release process](https://github.com/logos-messaging/logos-delivery/blob/master/docs/contributors/release-process.md) - [Release notes](https://github.com/logos-messaging/logos-delivery/blob/master/CHANGELOG.md) - [Infra-role-nim-waku](https://github.com/status-im/infra-role-nim-waku) -- [Infra-nim-waku](https://github.com/status-im/infra-nim-waku) +- [Infra-waku](https://github.com/status-im/infra-waku) - [Infra-Status](https://github.com/status-im/infra-status) - [Jenkins](https://ci.infra.status.im/job/nim-waku/) - [Fleets](https://fleets.waku.org/) From 0623c10635e5f00cd73db50e28aafc535b4b59ee Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Mon, 30 Mar 2026 00:08:08 +0200 Subject: [PATCH 13/19] completely remove storev2 (#3781) --- apps/chat2/chat2.nim | 1 - tests/all_tests_waku.nim | 22 +- tests/common/test_ratelimit_setting.nim | 29 +- tests/node/test_all.nim | 1 - tests/node/test_wakunode_legacy_store.nim | 1064 ---------- tests/node/test_wakunode_sharding.nim | 71 +- tests/testlib/futures.nim | 5 +- tests/testlib/postgres_legacy.nim | 27 - tests/testlib/wakunode.nim | 1 - tests/waku_archive_legacy/archive_utils.nim | 55 - tests/waku_archive_legacy/test_all.nim | 13 - .../test_driver_postgres.nim | 220 -- .../test_driver_postgres_query.nim | 1854 ----------------- .../waku_archive_legacy/test_driver_queue.nim | 182 -- .../test_driver_queue_index.nim | 219 -- .../test_driver_queue_pagination.nim | 405 ---- .../test_driver_queue_query.nim | 1668 --------------- .../test_driver_sqlite.nim | 58 - .../test_driver_sqlite_query.nim | 1745 ---------------- .../waku_archive_legacy/test_waku_archive.nim | 532 ----- tests/waku_store_legacy/store_utils.nim | 33 - tests/waku_store_legacy/test_all.nim | 8 - tests/waku_store_legacy/test_client.nim | 214 -- tests/waku_store_legacy/test_resume.nim | 338 --- tests/waku_store_legacy/test_rpc_codec.nim | 184 -- tests/waku_store_legacy/test_waku_store.nim | 113 - .../waku_store_legacy/test_wakunode_store.nim | 315 --- tests/wakunode_rest/test_rest_health.nim | 4 +- tools/confutils/cli_args.nim | 9 +- waku/common/rate_limit/setting.nim | 6 +- waku/common/waku_protocol.nim | 4 +- .../store_service_conf_builder.nim | 5 - waku/factory/node_factory.nim | 55 +- waku/factory/waku_conf.nim | 1 - .../health_monitor/node_health_monitor.nim | 34 - waku/node/kernel_api/relay.nim | 6 - waku/node/kernel_api/store.nim | 157 +- waku/node/waku_node.nim | 7 - waku/rest_api/endpoint/admin/handlers.nim | 5 +- waku/rest_api/endpoint/builder.nim | 2 - .../rest_api/endpoint/legacy_store/client.nim | 75 - .../endpoint/legacy_store/handlers.nim | 246 --- waku/rest_api/endpoint/legacy_store/types.nim | 375 ---- waku/waku_archive_legacy.nim | 6 - waku/waku_archive_legacy/archive.nim | 285 --- waku/waku_archive_legacy/archive_metrics.nim | 22 - waku/waku_archive_legacy/common.nim | 88 - waku/waku_archive_legacy/driver.nim | 121 -- waku/waku_archive_legacy/driver/builder.nim | 89 - .../driver/postgres_driver.nim | 8 - .../postgres_driver/postgres_driver.nim | 976 --------- .../postgres_driver/postgres_healthcheck.nim | 37 - .../driver/queue_driver.nim | 8 - .../driver/queue_driver/index.nim | 91 - .../driver/queue_driver/queue_driver.nim | 363 ---- .../driver/sqlite_driver.nim | 8 - .../driver/sqlite_driver/cursor.nim | 11 - .../driver/sqlite_driver/migrations.nim | 71 - .../driver/sqlite_driver/queries.nim | 729 ------- .../driver/sqlite_driver/sqlite_driver.nim | 220 -- waku/waku_core/codecs.nim | 1 - waku/waku_store_legacy.nim | 3 - waku/waku_store_legacy/README.md | 3 - waku/waku_store_legacy/client.nim | 241 --- waku/waku_store_legacy/common.nim | 108 - waku/waku_store_legacy/protocol.nim | 188 -- waku/waku_store_legacy/protocol_metrics.nim | 21 - waku/waku_store_legacy/rpc.nim | 218 -- waku/waku_store_legacy/rpc_codec.nim | 255 --- waku/waku_store_legacy/self_req_handler.nim | 31 - 70 files changed, 56 insertions(+), 14514 deletions(-) delete mode 100644 tests/node/test_wakunode_legacy_store.nim delete mode 100644 tests/testlib/postgres_legacy.nim delete mode 100644 tests/waku_archive_legacy/archive_utils.nim delete mode 100644 tests/waku_archive_legacy/test_all.nim delete mode 100644 tests/waku_archive_legacy/test_driver_postgres.nim delete mode 100644 tests/waku_archive_legacy/test_driver_postgres_query.nim delete mode 100644 tests/waku_archive_legacy/test_driver_queue.nim delete mode 100644 tests/waku_archive_legacy/test_driver_queue_index.nim delete mode 100644 tests/waku_archive_legacy/test_driver_queue_pagination.nim delete mode 100644 tests/waku_archive_legacy/test_driver_queue_query.nim delete mode 100644 tests/waku_archive_legacy/test_driver_sqlite.nim delete mode 100644 tests/waku_archive_legacy/test_driver_sqlite_query.nim delete mode 100644 tests/waku_archive_legacy/test_waku_archive.nim delete mode 100644 tests/waku_store_legacy/store_utils.nim delete mode 100644 tests/waku_store_legacy/test_all.nim delete mode 100644 tests/waku_store_legacy/test_client.nim delete mode 100644 tests/waku_store_legacy/test_resume.nim delete mode 100644 tests/waku_store_legacy/test_rpc_codec.nim delete mode 100644 tests/waku_store_legacy/test_waku_store.nim delete mode 100644 tests/waku_store_legacy/test_wakunode_store.nim delete mode 100644 waku/rest_api/endpoint/legacy_store/client.nim delete mode 100644 waku/rest_api/endpoint/legacy_store/handlers.nim delete mode 100644 waku/rest_api/endpoint/legacy_store/types.nim delete mode 100644 waku/waku_archive_legacy.nim delete mode 100644 waku/waku_archive_legacy/archive.nim delete mode 100644 waku/waku_archive_legacy/archive_metrics.nim delete mode 100644 waku/waku_archive_legacy/common.nim delete mode 100644 waku/waku_archive_legacy/driver.nim delete mode 100644 waku/waku_archive_legacy/driver/builder.nim delete mode 100644 waku/waku_archive_legacy/driver/postgres_driver.nim delete mode 100644 waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim delete mode 100644 waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim delete mode 100644 waku/waku_archive_legacy/driver/queue_driver.nim delete mode 100644 waku/waku_archive_legacy/driver/queue_driver/index.nim delete mode 100644 waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim delete mode 100644 waku/waku_archive_legacy/driver/sqlite_driver.nim delete mode 100644 waku/waku_archive_legacy/driver/sqlite_driver/cursor.nim delete mode 100644 waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim delete mode 100644 waku/waku_archive_legacy/driver/sqlite_driver/queries.nim delete mode 100644 waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim delete mode 100644 waku/waku_store_legacy.nim delete mode 100644 waku/waku_store_legacy/README.md delete mode 100644 waku/waku_store_legacy/client.nim delete mode 100644 waku/waku_store_legacy/common.nim delete mode 100644 waku/waku_store_legacy/protocol.nim delete mode 100644 waku/waku_store_legacy/protocol_metrics.nim delete mode 100644 waku/waku_store_legacy/rpc.nim delete mode 100644 waku/waku_store_legacy/rpc_codec.nim delete mode 100644 waku/waku_store_legacy/self_req_handler.nim diff --git a/apps/chat2/chat2.nim b/apps/chat2/chat2.nim index 4102cf074..e76c7be17 100644 --- a/apps/chat2/chat2.nim +++ b/apps/chat2/chat2.nim @@ -36,7 +36,6 @@ import waku_lightpush_legacy/rpc, waku_enr, discovery/waku_dnsdisc, - waku_store_legacy, waku_node, node/waku_metrics, node/peer_manager, diff --git a/tests/all_tests_waku.nim b/tests/all_tests_waku.nim index 4d4225f9f..879b1a55a 100644 --- a/tests/all_tests_waku.nim +++ b/tests/all_tests_waku.nim @@ -20,14 +20,7 @@ import ./waku_archive/test_driver_sqlite, ./waku_archive/test_retention_policy, ./waku_archive/test_waku_archive, - ./waku_archive/test_partition_manager, - ./waku_archive_legacy/test_driver_queue_index, - ./waku_archive_legacy/test_driver_queue_pagination, - ./waku_archive_legacy/test_driver_queue_query, - ./waku_archive_legacy/test_driver_queue, - ./waku_archive_legacy/test_driver_sqlite_query, - ./waku_archive_legacy/test_driver_sqlite, - ./waku_archive_legacy/test_waku_archive + ./waku_archive/test_partition_manager const os* {.strdefine.} = "" when os == "Linux" and @@ -37,8 +30,6 @@ when os == "Linux" and import ./waku_archive/test_driver_postgres_query, ./waku_archive/test_driver_postgres, - #./waku_archive_legacy/test_driver_postgres_query, - #./waku_archive_legacy/test_driver_postgres, ./factory/test_node_factory, ./wakunode_rest/test_rest_store, ./wakunode_rest/test_all @@ -50,20 +41,9 @@ import ./waku_store/test_waku_store, ./waku_store/test_wakunode_store -# Waku legacy store test suite -import - ./waku_store_legacy/test_client, - ./waku_store_legacy/test_rpc_codec, - ./waku_store_legacy/test_waku_store, - ./waku_store_legacy/test_wakunode_store - # Waku store sync suite import ./waku_store_sync/test_all -when defined(waku_exp_store_resume): - # TODO: Review store resume test cases (#1282) - import ./waku_store_legacy/test_resume - import ./node/test_all, ./waku_filter_v2/test_all, diff --git a/tests/common/test_ratelimit_setting.nim b/tests/common/test_ratelimit_setting.nim index 97d69e06a..2bc95fbfb 100644 --- a/tests/common/test_ratelimit_setting.nim +++ b/tests/common/test_ratelimit_setting.nim @@ -25,7 +25,6 @@ suite "RateLimitSetting": test "Parse rate limit setting - ok": let test1 = "10/2m" let test2 = " store : 10 /1h" - let test2a = "storev2 : 10 /1h" let test2b = "storeV3: 12 /1s" let test3 = "LIGHTPUSH: 10/ 1m" let test4 = "px:10/2 s " @@ -34,7 +33,6 @@ suite "RateLimitSetting": let expU = UnlimitedRateLimit let exp1: RateLimitSetting = (10, 2.minutes) let exp2: RateLimitSetting = (10, 1.hours) - let exp2a: RateLimitSetting = (10, 1.hours) let exp2b: RateLimitSetting = (12, 1.seconds) let exp3: RateLimitSetting = (10, 1.minutes) let exp4: RateLimitSetting = (10, 2.seconds) @@ -42,7 +40,6 @@ suite "RateLimitSetting": let res1 = ProtocolRateLimitSettings.parse(@[test1]) let res2 = ProtocolRateLimitSettings.parse(@[test2]) - let res2a = ProtocolRateLimitSettings.parse(@[test2a]) let res2b = ProtocolRateLimitSettings.parse(@[test2b]) let res3 = ProtocolRateLimitSettings.parse(@[test3]) let res4 = ProtocolRateLimitSettings.parse(@[test4]) @@ -53,15 +50,7 @@ suite "RateLimitSetting": res1.get() == {GLOBAL: exp1, FILTER: FilterDefaultPerPeerRateLimit}.toTable() res2.isOk() res2.get() == - { - GLOBAL: expU, - FILTER: FilterDefaultPerPeerRateLimit, - STOREV2: exp2, - STOREV3: exp2, - }.toTable() - res2a.isOk() - res2a.get() == - {GLOBAL: expU, FILTER: FilterDefaultPerPeerRateLimit, STOREV2: exp2a}.toTable() + {GLOBAL: expU, FILTER: FilterDefaultPerPeerRateLimit, STOREV3: exp2}.toTable() res2b.isOk() res2b.get() == {GLOBAL: expU, FILTER: FilterDefaultPerPeerRateLimit, STOREV3: exp2b}.toTable() @@ -77,7 +66,6 @@ suite "RateLimitSetting": test "Parse rate limit setting - err": let test1 = "10/2d" let test2 = " stre : 10 /1h" - let test2a = "storev2 10 /1h" let test2b = "storev3: 12 1s" let test3 = "somethingelse: 10/ 1m" let test4 = ":px:10/2 s " @@ -85,7 +73,6 @@ suite "RateLimitSetting": let res1 = ProtocolRateLimitSettings.parse(@[test1]) let res2 = ProtocolRateLimitSettings.parse(@[test2]) - let res2a = ProtocolRateLimitSettings.parse(@[test2a]) let res2b = ProtocolRateLimitSettings.parse(@[test2b]) let res3 = ProtocolRateLimitSettings.parse(@[test3]) let res4 = ProtocolRateLimitSettings.parse(@[test4]) @@ -94,7 +81,6 @@ suite "RateLimitSetting": check: res1.isErr() res2.isErr() - res2a.isErr() res2b.isErr() res3.isErr() res4.isErr() @@ -103,13 +89,12 @@ suite "RateLimitSetting": test "Parse rate limit setting - complex": let expU = UnlimitedRateLimit - let test1 = @["lightpush:2/2ms", "10/2m", " store: 3/3s", " storev2:12/12s"] + let test1 = @["lightpush:2/2ms", "10/2m", " store: 3/3s"] let exp1 = { GLOBAL: (10, 2.minutes), FILTER: FilterDefaultPerPeerRateLimit, LIGHTPUSH: (2, 2.milliseconds), STOREV3: (3, 3.seconds), - STOREV2: (12, 12.seconds), }.toTable() let res1 = ProtocolRateLimitSettings.parse(test1) @@ -118,7 +103,6 @@ suite "RateLimitSetting": res1.isOk() res1.get() == exp1 res1.get().getSetting(PEEREXCHG) == (10, 2.minutes) - res1.get().getSetting(STOREV2) == (12, 12.seconds) res1.get().getSetting(STOREV3) == (3, 3.seconds) res1.get().getSetting(LIGHTPUSH) == (2, 2.milliseconds) @@ -127,7 +111,6 @@ suite "RateLimitSetting": GLOBAL: expU, LIGHTPUSH: (2, 2.milliseconds), STOREV3: (3, 3.seconds), - STOREV2: (3, 3.seconds), FILTER: (4, 42.milliseconds), PEEREXCHG: (10, 10.hours), }.toTable() @@ -138,13 +121,9 @@ suite "RateLimitSetting": res2.isOk() res2.get() == exp2 - let test3 = - @["storev2:1/1s", "store:3/3s", "storev3:4/42ms", "storev3:5/5s", "storev3:6/6s"] + let test3 = @["store:3/3s", "storev3:4/42ms", "storev3:5/5s", "storev3:6/6s"] let exp3 = { - GLOBAL: expU, - FILTER: FilterDefaultPerPeerRateLimit, - STOREV3: (6, 6.seconds), - STOREV2: (1, 1.seconds), + GLOBAL: expU, FILTER: FilterDefaultPerPeerRateLimit, STOREV3: (6, 6.seconds) }.toTable() let res3 = ProtocolRateLimitSettings.parse(test3) diff --git a/tests/node/test_all.nim b/tests/node/test_all.nim index fe785dee2..2846fdb7f 100644 --- a/tests/node/test_all.nim +++ b/tests/node/test_all.nim @@ -6,6 +6,5 @@ import ./test_wakunode_lightpush, ./test_wakunode_peer_exchange, ./test_wakunode_store, - ./test_wakunode_legacy_store, ./test_wakunode_peer_manager, ./test_wakunode_health_monitor diff --git a/tests/node/test_wakunode_legacy_store.nim b/tests/node/test_wakunode_legacy_store.nim deleted file mode 100644 index e9b0c9170..000000000 --- a/tests/node/test_wakunode_legacy_store.nim +++ /dev/null @@ -1,1064 +0,0 @@ -{.used.} - -import std/options, testutils/unittests, chronos, libp2p/crypto/crypto - -import - waku/[ - common/paging, - node/waku_node, - node/kernel_api, - node/peer_manager, - waku_core, - waku_store_legacy, - waku_archive_legacy, - ], - ../waku_store_legacy/store_utils, - ../waku_archive_legacy/archive_utils, - ../testlib/[wakucore, wakunode, testasync, testutils] - -suite "Waku Store - End to End - Sorted Archive": - var pubsubTopic {.threadvar.}: PubsubTopic - var contentTopic {.threadvar.}: ContentTopic - var contentTopicSeq {.threadvar.}: seq[ContentTopic] - - var archiveMessages {.threadvar.}: seq[WakuMessage] - var historyQuery {.threadvar.}: HistoryQuery - - var server {.threadvar.}: WakuNode - var client {.threadvar.}: WakuNode - - var archiveDriver {.threadvar.}: ArchiveDriver - var serverRemotePeerInfo {.threadvar.}: RemotePeerInfo - var clientPeerId {.threadvar.}: PeerId - - asyncSetup: - pubsubTopic = DefaultPubsubTopic - contentTopic = DefaultContentTopic - contentTopicSeq = @[contentTopic] - - let timeOrigin = now() - archiveMessages = @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] - - historyQuery = HistoryQuery( - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.Forward, - pageSize: 5, - ) - - let - serverKey = generateSecp256k1Key() - clientKey = generateSecp256k1Key() - - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - archiveDriver = newArchiveDriverWithMessages(pubsubTopic, archiveMessages) - let mountArchiveResult = server.mountLegacyArchive(archiveDriver) - assert mountArchiveResult.isOk() - - await server.mountLegacyStore() - client.mountLegacyStoreClient() - - await allFutures(server.start(), client.start()) - - serverRemotePeerInfo = server.peerInfo.toRemotePeerInfo() - clientPeerId = client.peerInfo.toRemotePeerInfo().peerId - - asyncTeardown: - await allFutures(client.stop(), server.stop()) - - suite "Message Pagination": - asyncTest "Forward Pagination": - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == archiveMessages[0 ..< 5] - - # Given the next query - var otherHistoryQuery = HistoryQuery( - cursor: queryResponse.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 5, - ) - - # When making the next history query - let otherQueryResponse = - await client.query(otherHistoryQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - otherQueryResponse.get().messages == archiveMessages[5 ..< 10] - - asyncTest "Backward Pagination": - # Given the history query is backward - historyQuery.direction = PagingDirection.BACKWARD - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == archiveMessages[5 ..< 10] - - # Given the next query - var nextHistoryQuery = HistoryQuery( - cursor: queryResponse.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.BACKWARD, - pageSize: 5, - ) - - # When making the next history query - let otherQueryResponse = - await client.query(nextHistoryQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - otherQueryResponse.get().messages == archiveMessages[0 ..< 5] - - suite "Pagination with Differente Page Sizes": - asyncTest "Pagination with Small Page Size": - # Given the first query (1/5) - historyQuery.pageSize = 2 - - # When making a history query - let queryResponse1 = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse1.get().messages == archiveMessages[0 ..< 2] - - # Given the next query (2/5) - let historyQuery2 = HistoryQuery( - cursor: queryResponse1.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 2, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse2.get().messages == archiveMessages[2 ..< 4] - - # Given the next query (3/5) - let historyQuery3 = HistoryQuery( - cursor: queryResponse2.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 2, - ) - - # When making the next history query - let queryResponse3 = await client.query(historyQuery3, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse3.get().messages == archiveMessages[4 ..< 6] - - # Given the next query (4/5) - let historyQuery4 = HistoryQuery( - cursor: queryResponse3.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 2, - ) - - # When making the next history query - let queryResponse4 = await client.query(historyQuery4, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse4.get().messages == archiveMessages[6 ..< 8] - - # Given the next query (5/5) - let historyQuery5 = HistoryQuery( - cursor: queryResponse4.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 2, - ) - - # When making the next history query - let queryResponse5 = await client.query(historyQuery5, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse5.get().messages == archiveMessages[8 ..< 10] - - asyncTest "Pagination with Large Page Size": - # Given the first query (1/2) - historyQuery.pageSize = 8 - - # When making a history query - let queryResponse1 = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse1.get().messages == archiveMessages[0 ..< 8] - - # Given the next query (2/2) - let historyQuery2 = HistoryQuery( - cursor: queryResponse1.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 8, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse2.get().messages == archiveMessages[8 ..< 10] - - asyncTest "Pagination with Excessive Page Size": - # Given the first query (1/1) - historyQuery.pageSize = 100 - - # When making a history query - let queryResponse1 = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse1.get().messages == archiveMessages[0 ..< 10] - - asyncTest "Pagination with Mixed Page Size": - # Given the first query (1/3) - historyQuery.pageSize = 2 - - # When making a history query - let queryResponse1 = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse1.get().messages == archiveMessages[0 ..< 2] - - # Given the next query (2/3) - let historyQuery2 = HistoryQuery( - cursor: queryResponse1.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 4, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse2.get().messages == archiveMessages[2 ..< 6] - - # Given the next query (3/3) - let historyQuery3 = HistoryQuery( - cursor: queryResponse2.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 6, - ) - - # When making the next history query - let queryResponse3 = await client.query(historyQuery3, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse3.get().messages == archiveMessages[6 ..< 10] - - asyncTest "Pagination with Zero Page Size (Behaves as DefaultPageSize)": - # Given a message list of size higher than the default page size - let currentStoreLen = uint((await archiveDriver.getMessagesCount()).get()) - assert archive.DefaultPageSize > currentStoreLen, - "This test requires a store with more than (DefaultPageSize) messages" - let missingMessagesAmount = archive.DefaultPageSize - currentStoreLen + 5 - - let lastMessageTimestamp = archiveMessages[archiveMessages.len - 1].timestamp - var extraMessages: seq[WakuMessage] = @[] - for i in 0 ..< missingMessagesAmount: - let - timestampOffset = 10 * int(i + 1) - # + 1 to avoid collision with existing messages - message: WakuMessage = - fakeWakuMessage(@[byte i], ts = ts(timestampOffset, lastMessageTimestamp)) - extraMessages.add(message) - discard archiveDriver.put(pubsubTopic, extraMessages) - - let totalMessages = archiveMessages & extraMessages - - # Given the a query with zero page size (1/2) - historyQuery.pageSize = 0 - - # When making a history query - let queryResponse1 = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the archive.DefaultPageSize messages - check: - queryResponse1.get().messages == totalMessages[0 ..< archive.DefaultPageSize] - - # Given the next query (2/2) - let historyQuery2 = HistoryQuery( - cursor: queryResponse1.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 0, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the remaining messages - check: - queryResponse2.get().messages == - totalMessages[archive.DefaultPageSize ..< archive.DefaultPageSize + 5] - - asyncTest "Pagination with Default Page Size": - # Given a message list of size higher than the default page size - let currentStoreLen = uint((await archiveDriver.getMessagesCount()).get()) - assert archive.DefaultPageSize > currentStoreLen, - "This test requires a store with more than (DefaultPageSize) messages" - let missingMessagesAmount = archive.DefaultPageSize - currentStoreLen + 5 - - let lastMessageTimestamp = archiveMessages[archiveMessages.len - 1].timestamp - var extraMessages: seq[WakuMessage] = @[] - for i in 0 ..< missingMessagesAmount: - let - timestampOffset = 10 * int(i + 1) - # + 1 to avoid collision with existing messages - message: WakuMessage = - fakeWakuMessage(@[byte i], ts = ts(timestampOffset, lastMessageTimestamp)) - extraMessages.add(message) - discard archiveDriver.put(pubsubTopic, extraMessages) - - let totalMessages = archiveMessages & extraMessages - - # Given a query with default page size (1/2) - historyQuery = HistoryQuery( - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - ) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == totalMessages[0 ..< archive.DefaultPageSize] - - # Given the next query (2/2) - let historyQuery2 = HistoryQuery( - cursor: queryResponse.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse2.get().messages == - totalMessages[archive.DefaultPageSize ..< archive.DefaultPageSize + 5] - - suite "Pagination with Different Cursors": - asyncTest "Starting Cursor": - # Given a cursor pointing to the first message - let cursor = computeHistoryCursor(pubsubTopic, archiveMessages[0]) - historyQuery.cursor = some(cursor) - historyQuery.pageSize = 1 - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the message - check: - queryResponse.get().messages == archiveMessages[1 ..< 2] - - asyncTest "Middle Cursor": - # Given a cursor pointing to the middle message1 - let cursor = computeHistoryCursor(pubsubTopic, archiveMessages[5]) - historyQuery.cursor = some(cursor) - historyQuery.pageSize = 1 - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the message - check: - queryResponse.get().messages == archiveMessages[6 ..< 7] - - asyncTest "Ending Cursor": - # Given a cursor pointing to the last message - let cursor = computeHistoryCursor(pubsubTopic, archiveMessages[9]) - historyQuery.cursor = some(cursor) - historyQuery.pageSize = 1 - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages.len == 0 - - suite "Message Sorting": - asyncTest "Cursor Reusability Across Nodes": - # Given a different server node with the same archive - let - otherArchiveDriverWithMessages = - newArchiveDriverWithMessages(pubsubTopic, archiveMessages) - otherServerKey = generateSecp256k1Key() - otherServer = - newTestWakuNode(otherServerKey, parseIpAddress("0.0.0.0"), Port(0)) - mountOtherArchiveResult = - otherServer.mountLegacyArchive(otherArchiveDriverWithMessages) - assert mountOtherArchiveResult.isOk() - - await otherServer.mountLegacyStore() - - await otherServer.start() - let otherServerRemotePeerInfo = otherServer.peerInfo.toRemotePeerInfo() - - # When making a history query to the first server node - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == archiveMessages[0 ..< 5] - - # Given the cursor from the first query - let cursor = queryResponse.get().cursor - - # When making a history query to the second server node - let otherHistoryQuery = HistoryQuery( - cursor: cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 5, - ) - let otherQueryResponse = - await client.query(otherHistoryQuery, otherServerRemotePeerInfo) - - # Then the response contains the remaining messages - check: - otherQueryResponse.get().messages == archiveMessages[5 ..< 10] - - # Cleanup - await otherServer.stop() - -suite "Waku Store - End to End - Unsorted Archive": - var pubsubTopic {.threadvar.}: PubsubTopic - var contentTopic {.threadvar.}: ContentTopic - var contentTopicSeq {.threadvar.}: seq[ContentTopic] - - var historyQuery {.threadvar.}: HistoryQuery - var unsortedArchiveMessages {.threadvar.}: seq[WakuMessage] - - var server {.threadvar.}: WakuNode - var client {.threadvar.}: WakuNode - - var serverRemotePeerInfo {.threadvar.}: RemotePeerInfo - - asyncSetup: - pubsubTopic = DefaultPubsubTopic - contentTopic = DefaultContentTopic - contentTopicSeq = @[contentTopic] - - historyQuery = HistoryQuery( - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 5, - ) - - let timeOrigin = now() - unsortedArchiveMessages = @[ # SortIndex (by timestamp and digest) - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), # 1 - fakeWakuMessage(@[byte 03], ts = ts(00, timeOrigin)), # 2 - fakeWakuMessage(@[byte 08], ts = ts(00, timeOrigin)), # 0 - fakeWakuMessage(@[byte 07], ts = ts(10, timeOrigin)), # 4 - fakeWakuMessage(@[byte 02], ts = ts(10, timeOrigin)), # 3 - fakeWakuMessage(@[byte 09], ts = ts(10, timeOrigin)), # 5 - fakeWakuMessage(@[byte 06], ts = ts(20, timeOrigin)), # 6 - fakeWakuMessage(@[byte 01], ts = ts(20, timeOrigin)), # 9 - fakeWakuMessage(@[byte 04], ts = ts(20, timeOrigin)), # 7 - fakeWakuMessage(@[byte 05], ts = ts(20, timeOrigin)), # 8 - ] - - let - serverKey = generateSecp256k1Key() - clientKey = generateSecp256k1Key() - - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - let - unsortedArchiveDriverWithMessages = - newArchiveDriverWithMessages(pubsubTopic, unsortedArchiveMessages) - mountUnsortedArchiveResult = - server.mountLegacyArchive(unsortedArchiveDriverWithMessages) - - assert mountUnsortedArchiveResult.isOk() - - await server.mountLegacyStore() - client.mountLegacyStoreClient() - - await allFutures(server.start(), client.start()) - - serverRemotePeerInfo = server.peerInfo.toRemotePeerInfo() - - asyncTeardown: - await allFutures(client.stop(), server.stop()) - - asyncTest "Basic (Timestamp and Digest) Sorting Validation": - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - unsortedArchiveMessages[2], - unsortedArchiveMessages[0], - unsortedArchiveMessages[1], - unsortedArchiveMessages[4], - unsortedArchiveMessages[3], - ] - - # Given the next query - var historyQuery2 = HistoryQuery( - cursor: queryResponse.get().cursor, - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 5, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse2.get().messages == - @[ - unsortedArchiveMessages[5], - unsortedArchiveMessages[6], - unsortedArchiveMessages[8], - unsortedArchiveMessages[9], - unsortedArchiveMessages[7], - ] - - asyncTest "Backward pagination with Ascending Sorting": - # Given a history query with backward pagination - let cursor = computeHistoryCursor(pubsubTopic, unsortedArchiveMessages[4]) - historyQuery.direction = PagingDirection.BACKWARD - historyQuery.cursor = some(cursor) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - unsortedArchiveMessages[2], - unsortedArchiveMessages[0], - unsortedArchiveMessages[1], - ] - - asyncTest "Forward Pagination with Ascending Sorting": - # Given a history query with forward pagination - let cursor = computeHistoryCursor(pubsubTopic, unsortedArchiveMessages[4]) - historyQuery.direction = PagingDirection.FORWARD - historyQuery.cursor = some(cursor) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - unsortedArchiveMessages[3], - unsortedArchiveMessages[5], - unsortedArchiveMessages[6], - unsortedArchiveMessages[8], - unsortedArchiveMessages[9], - ] - -suite "Waku Store - End to End - Archive with Multiple Topics": - var pubsubTopic {.threadvar.}: PubsubTopic - var pubsubTopicB {.threadvar.}: PubsubTopic - var contentTopic {.threadvar.}: ContentTopic - var contentTopicB {.threadvar.}: ContentTopic - var contentTopicC {.threadvar.}: ContentTopic - var contentTopicSpecials {.threadvar.}: ContentTopic - var contentTopicSeq {.threadvar.}: seq[ContentTopic] - - var historyQuery {.threadvar.}: HistoryQuery - var originTs {.threadvar.}: proc(offset: int): Timestamp {.gcsafe, raises: [].} - var archiveMessages {.threadvar.}: seq[WakuMessage] - - var server {.threadvar.}: WakuNode - var client {.threadvar.}: WakuNode - - var serverRemotePeerInfo {.threadvar.}: RemotePeerInfo - - asyncSetup: - pubsubTopic = DefaultPubsubTopic - pubsubTopicB = "topicB" - contentTopic = DefaultContentTopic - contentTopicB = "topicB" - contentTopicC = "topicC" - contentTopicSpecials = "!@#$%^&*()_+" - contentTopicSeq = - @[contentTopic, contentTopicB, contentTopicC, contentTopicSpecials] - - historyQuery = HistoryQuery( - pubsubTopic: some(pubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 5, - ) - - let timeOrigin = now() - originTs = proc(offset = 0): Timestamp {.gcsafe, raises: [].} = - ts(offset, timeOrigin) - - archiveMessages = @[ - fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic), - fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic), - fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC), - fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic), - fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB), - fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC), - fakeWakuMessage( - @[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials - ), - ] - - let - serverKey = generateSecp256k1Key() - clientKey = generateSecp256k1Key() - - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - let archiveDriver = newSqliteArchiveDriver() - .put(pubsubTopic, archiveMessages[0 ..< 6]) - .put(pubsubTopicB, archiveMessages[6 ..< 10]) - let mountSortedArchiveResult = server.mountLegacyArchive(archiveDriver) - - assert mountSortedArchiveResult.isOk() - - await server.mountLegacyStore() - client.mountLegacyStoreClient() - - await allFutures(server.start(), client.start()) - - serverRemotePeerInfo = server.peerInfo.toRemotePeerInfo() - - asyncTeardown: - await allFutures(client.stop(), server.stop()) - - suite "Validation of Content Filtering": - asyncTest "Basic Content Filtering": - # Given a history query with content filtering - historyQuery.contentTopics = @[contentTopic] - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == @[archiveMessages[0], archiveMessages[3]] - - asyncTest "Multiple Content Filters": - # Given a history query with multiple content filtering - historyQuery.contentTopics = @[contentTopic, contentTopicB] - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - archiveMessages[0], - archiveMessages[1], - archiveMessages[3], - archiveMessages[4], - ] - - asyncTest "Empty Content Filtering": - # Given a history query with empty content filtering - historyQuery.contentTopics = @[] - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == archiveMessages[0 ..< 5] - - # Given the next query - let historyQuery2 = HistoryQuery( - cursor: queryResponse.get().cursor, - pubsubTopic: none(PubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 5, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse2.get().messages == archiveMessages[5 ..< 10] - - asyncTest "Non-Existent Content Topic": - # Given a history query with non-existent content filtering - historyQuery.contentTopics = @["non-existent-topic"] - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages.len == 0 - - asyncTest "Special Characters in Content Filtering": - # Given a history query with special characters in content filtering - historyQuery.pubsubTopic = some(pubsubTopicB) - historyQuery.contentTopics = @["!@#$%^&*()_+"] - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages == @[archiveMessages[9]] - - asyncTest "PubsubTopic Specified": - # Given a history query with pubsub topic specified - historyQuery.pubsubTopic = some(pubsubTopicB) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - archiveMessages[6], - archiveMessages[7], - archiveMessages[8], - archiveMessages[9], - ] - - asyncTest "PubsubTopic Left Empty": - # Given a history query with pubsub topic left empty - historyQuery.pubsubTopic = none(PubsubTopic) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == archiveMessages[0 ..< 5] - - # Given the next query - let historyQuery2 = HistoryQuery( - cursor: queryResponse.get().cursor, - pubsubTopic: none(PubsubTopic), - contentTopics: contentTopicSeq, - direction: PagingDirection.FORWARD, - pageSize: 5, - ) - - # When making the next history query - let queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse2.get().messages == archiveMessages[5 ..< 10] - - suite "Validation of Time-based Filtering": - asyncTest "Basic Time Filtering": - # Given a history query with start and end time - historyQuery.startTime = some(originTs(20)) - historyQuery.endTime = some(originTs(40)) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[archiveMessages[2], archiveMessages[3], archiveMessages[4]] - - asyncTest "Only Start Time Specified": - # Given a history query with only start time - historyQuery.startTime = some(originTs(20)) - historyQuery.endTime = none(Timestamp) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - archiveMessages[2], - archiveMessages[3], - archiveMessages[4], - archiveMessages[5], - ] - - asyncTest "Only End Time Specified": - # Given a history query with only end time - historyQuery.startTime = none(Timestamp) - historyQuery.endTime = some(originTs(40)) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages == - @[ - archiveMessages[0], - archiveMessages[1], - archiveMessages[2], - archiveMessages[3], - archiveMessages[4], - ] - - asyncTest "Invalid Time Range": - # Given a history query with invalid time range - historyQuery.startTime = some(originTs(60)) - historyQuery.endTime = some(originTs(40)) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages.len == 0 - - asyncTest "Time Filtering with Content Filtering": - # Given a history query with time and content filtering - historyQuery.startTime = some(originTs(20)) - historyQuery.endTime = some(originTs(60)) - historyQuery.contentTopics = @[contentTopicC] - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == @[archiveMessages[2], archiveMessages[5]] - - asyncTest "Messages Outside of Time Range": - # Given a history query with a valid time range which does not contain any messages - historyQuery.startTime = some(originTs(100)) - historyQuery.endTime = some(originTs(200)) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages.len == 0 - - suite "Ephemeral": - # TODO: Ephemeral value is not properly set for Sqlite - xasyncTest "Only ephemeral Messages:": - # Given an archive with only ephemeral messages - let - ephemeralMessages = @[ - fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), - fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), - fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), - ] - ephemeralArchiveDriver = - newSqliteArchiveDriver().put(pubsubTopic, ephemeralMessages) - - # And a server node with the ephemeral archive - let - ephemeralServerKey = generateSecp256k1Key() - ephemeralServer = - newTestWakuNode(ephemeralServerKey, parseIpAddress("0.0.0.0"), Port(0)) - mountEphemeralArchiveResult = - ephemeralServer.mountLegacyArchive(ephemeralArchiveDriver) - assert mountEphemeralArchiveResult.isOk() - - await ephemeralServer.mountLegacyStore() - await ephemeralServer.start() - let ephemeralServerRemotePeerInfo = ephemeralServer.peerInfo.toRemotePeerInfo() - - # When making a history query to the server with only ephemeral messages - let queryResponse = - await client.query(historyQuery, ephemeralServerRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages.len == 0 - - # Cleanup - await ephemeralServer.stop() - - xasyncTest "Mixed messages": - # Given an archive with both ephemeral and non-ephemeral messages - let - ephemeralMessages = @[ - fakeWakuMessage(@[byte 00], ts = ts(00), ephemeral = true), - fakeWakuMessage(@[byte 01], ts = ts(10), ephemeral = true), - fakeWakuMessage(@[byte 02], ts = ts(20), ephemeral = true), - ] - nonEphemeralMessages = @[ - fakeWakuMessage(@[byte 03], ts = ts(30), ephemeral = false), - fakeWakuMessage(@[byte 04], ts = ts(40), ephemeral = false), - fakeWakuMessage(@[byte 05], ts = ts(50), ephemeral = false), - ] - mixedArchiveDriver = newSqliteArchiveDriver() - .put(pubsubTopic, ephemeralMessages) - .put(pubsubTopic, nonEphemeralMessages) - - # And a server node with the mixed archive - let - mixedServerKey = generateSecp256k1Key() - mixedServer = - newTestWakuNode(mixedServerKey, parseIpAddress("0.0.0.0"), Port(0)) - mountMixedArchiveResult = mixedServer.mountLegacyArchive(mixedArchiveDriver) - assert mountMixedArchiveResult.isOk() - - await mixedServer.mountLegacyStore() - await mixedServer.start() - let mixedServerRemotePeerInfo = mixedServer.peerInfo.toRemotePeerInfo() - - # When making a history query to the server with mixed messages - let queryResponse = await client.query(historyQuery, mixedServerRemotePeerInfo) - - # Then the response contains the non-ephemeral messages - check: - queryResponse.get().messages == nonEphemeralMessages - - # Cleanup - await mixedServer.stop() - - suite "Edge Case Scenarios": - asyncTest "Empty Message Store": - # Given an empty archive - let emptyArchiveDriver = newSqliteArchiveDriver() - - # And a server node with the empty archive - let - emptyServerKey = generateSecp256k1Key() - emptyServer = - newTestWakuNode(emptyServerKey, parseIpAddress("0.0.0.0"), Port(0)) - mountEmptyArchiveResult = emptyServer.mountLegacyArchive(emptyArchiveDriver) - assert mountEmptyArchiveResult.isOk() - - await emptyServer.mountLegacyStore() - await emptyServer.start() - let emptyServerRemotePeerInfo = emptyServer.peerInfo.toRemotePeerInfo() - - # When making a history query to the server with an empty archive - let queryResponse = await client.query(historyQuery, emptyServerRemotePeerInfo) - - # Then the response contains no messages - check: - queryResponse.get().messages.len == 0 - - # Cleanup - await emptyServer.stop() - - asyncTest "Voluminous Message Store": - # Given a voluminous archive (1M+ messages) - var voluminousArchiveMessages: seq[WakuMessage] = @[] - for i in 0 ..< 100000: - let topic = "topic" & $i - voluminousArchiveMessages.add(fakeWakuMessage(@[byte i], contentTopic = topic)) - let voluminousArchiveDriverWithMessages = - newArchiveDriverWithMessages(pubsubTopic, voluminousArchiveMessages) - - # And a server node with the voluminous archive - let - voluminousServerKey = generateSecp256k1Key() - voluminousServer = - newTestWakuNode(voluminousServerKey, parseIpAddress("0.0.0.0"), Port(0)) - mountVoluminousArchiveResult = - voluminousServer.mountLegacyArchive(voluminousArchiveDriverWithMessages) - assert mountVoluminousArchiveResult.isOk() - - await voluminousServer.mountLegacyStore() - await voluminousServer.start() - let voluminousServerRemotePeerInfo = voluminousServer.peerInfo.toRemotePeerInfo() - - # Given the following history query - historyQuery.contentTopics = - @["topic10000", "topic30000", "topic50000", "topic70000", "topic90000"] - - # When making a history query to the server with a voluminous archive - let queryResponse = - await client.query(historyQuery, voluminousServerRemotePeerInfo) - - # Then the response contains the messages - check: - queryResponse.get().messages == - @[ - voluminousArchiveMessages[10000], - voluminousArchiveMessages[30000], - voluminousArchiveMessages[50000], - voluminousArchiveMessages[70000], - voluminousArchiveMessages[90000], - ] - - # Cleanup - await voluminousServer.stop() - - asyncTest "Large contentFilters Array": - # Given a history query with the max contentFilters len, 10 - historyQuery.contentTopics = @[contentTopic] - for i in 0 ..< 9: - let topic = "topic" & $i - historyQuery.contentTopics.add(topic) - - # When making a history query - let queryResponse = await client.query(historyQuery, serverRemotePeerInfo) - - # Then the response should trigger no errors - check: - queryResponse.get().messages == @[archiveMessages[0], archiveMessages[3]] diff --git a/tests/node/test_wakunode_sharding.nim b/tests/node/test_wakunode_sharding.nim index 236510fe1..f482b6abc 100644 --- a/tests/node/test_wakunode_sharding.nim +++ b/tests/node/test_wakunode_sharding.nim @@ -14,7 +14,6 @@ import waku/[ waku_core/topics/pubsub_topic, waku_core/topics/sharding, - waku_store_legacy/common, node/waku_node, node/kernel_api, common/paging, @@ -454,29 +453,33 @@ suite "Sharding": # Given one query for each content topic format let - historyQuery1 = HistoryQuery( + storeQuery1 = StoreQueryRequest( contentTopics: @[contentTopicShort], - direction: PagingDirection.Forward, - pageSize: 3, + paginationForward: PagingDirection.Forward, + paginationLimit: some(3'u64), + includeData: true, ) - historyQuery2 = HistoryQuery( + storeQuery2 = StoreQueryRequest( contentTopics: @[contentTopicFull], - direction: PagingDirection.Forward, - pageSize: 3, + paginationForward: PagingDirection.Forward, + paginationLimit: some(3'u64), + includeData: true, ) # When the client queries the server for the messages let serverRemotePeerInfo = server.switch.peerInfo.toRemotePeerInfo() - queryResponse1 = await client.query(historyQuery1, serverRemotePeerInfo) - queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) + queryResponse1 = await client.query(storeQuery1, serverRemotePeerInfo) + queryResponse2 = await client.query(storeQuery2, serverRemotePeerInfo) assertResultOk(queryResponse1) assertResultOk(queryResponse2) # Then the responses of both queries should contain all the messages check: - queryResponse1.get().messages == archiveMessages1 & archiveMessages2 - queryResponse2.get().messages == archiveMessages1 & archiveMessages2 + queryResponse1.get().messages.mapIt(it.message.get()) == + archiveMessages1 & archiveMessages2 + queryResponse2.get().messages.mapIt(it.message.get()) == + archiveMessages1 & archiveMessages2 asyncTest "relay - exclusion (automatic sharding filtering)": # Given a connected server and client subscribed to different content topics @@ -615,29 +618,31 @@ suite "Sharding": # Given one query for each content topic let - historyQuery1 = HistoryQuery( + storeQuery1 = StoreQueryRequest( contentTopics: @[contentTopic1], - direction: PagingDirection.Forward, - pageSize: 2, + paginationForward: PagingDirection.Forward, + paginationLimit: some(2'u64), + includeData: true, ) - historyQuery2 = HistoryQuery( + storeQuery2 = StoreQueryRequest( contentTopics: @[contentTopic2], - direction: PagingDirection.Forward, - pageSize: 2, + paginationForward: PagingDirection.Forward, + paginationLimit: some(2'u64), + includeData: true, ) # When the client queries the server for the messages let serverRemotePeerInfo = server.switch.peerInfo.toRemotePeerInfo() - queryResponse1 = await client.query(historyQuery1, serverRemotePeerInfo) - queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) + queryResponse1 = await client.query(storeQuery1, serverRemotePeerInfo) + queryResponse2 = await client.query(storeQuery2, serverRemotePeerInfo) assertResultOk(queryResponse1) assertResultOk(queryResponse2) # Then each response should contain only the messages of the corresponding content topic check: - queryResponse1.get().messages == archiveMessages1 - queryResponse2.get().messages == archiveMessages2 + queryResponse1.get().messages.mapIt(it.message.get()) == archiveMessages1 + queryResponse2.get().messages.mapIt(it.message.get()) == archiveMessages2 suite "Specific Tests": asyncTest "Configure Node with Multiple PubSub Topics": @@ -1003,22 +1008,30 @@ suite "Sharding": # Given one query for each pubsub topic let - historyQuery1 = HistoryQuery( - pubsubTopic: some(topic1), direction: PagingDirection.Forward, pageSize: 2 + storeQuery1 = StoreQueryRequest( + pubsubTopic: some(topic1), + paginationForward: PagingDirection.Forward, + paginationLimit: some(2'u64), + includeData: true, ) - historyQuery2 = HistoryQuery( - pubsubTopic: some(topic2), direction: PagingDirection.Forward, pageSize: 2 + storeQuery2 = StoreQueryRequest( + pubsubTopic: some(topic2), + paginationForward: PagingDirection.Forward, + paginationLimit: some(2'u64), + includeData: true, ) # When the client queries the server for the messages let serverRemotePeerInfo = server.switch.peerInfo.toRemotePeerInfo() - queryResponse1 = await client.query(historyQuery1, serverRemotePeerInfo) - queryResponse2 = await client.query(historyQuery2, serverRemotePeerInfo) + queryResponse1 = await client.query(storeQuery1, serverRemotePeerInfo) + queryResponse2 = await client.query(storeQuery2, serverRemotePeerInfo) assertResultOk(queryResponse1) assertResultOk(queryResponse2) # Then each response should contain only the messages of the corresponding pubsub topic check: - queryResponse1.get().messages == archiveMessages1[0 ..< 1] - queryResponse2.get().messages == archiveMessages2[0 ..< 1] + queryResponse1.get().messages.mapIt(it.message.get()) == + archiveMessages1[0 ..< 1] + queryResponse2.get().messages.mapIt(it.message.get()) == + archiveMessages2[0 ..< 1] diff --git a/tests/testlib/futures.nim b/tests/testlib/futures.nim index e9a793388..dad1baec8 100644 --- a/tests/testlib/futures.nim +++ b/tests/testlib/futures.nim @@ -1,6 +1,6 @@ import chronos -import waku/[waku_core/message, waku_store, waku_store_legacy] +import waku/[waku_core/message, waku_store] const FUTURE_TIMEOUT* = 1.seconds @@ -18,9 +18,6 @@ proc newBoolFuture*(): Future[bool] = proc newHistoryFuture*(): Future[StoreQueryRequest] = newFuture[StoreQueryRequest]() -proc newLegacyHistoryFuture*(): Future[waku_store_legacy.HistoryQuery] = - newFuture[waku_store_legacy.HistoryQuery]() - proc toResult*[T](future: Future[T]): Result[T, string] = if future.cancelled(): return chronos.err("Future timeouted before completing.") diff --git a/tests/testlib/postgres_legacy.nim b/tests/testlib/postgres_legacy.nim deleted file mode 100644 index 50988c6c8..000000000 --- a/tests/testlib/postgres_legacy.nim +++ /dev/null @@ -1,27 +0,0 @@ -import chronicles, chronos -import - waku/waku_archive_legacy, - waku/waku_archive_legacy/driver as driver_module, - waku/waku_archive_legacy/driver/builder, - waku/waku_archive_legacy/driver/postgres_driver - -const storeMessageDbUrl = "postgres://postgres:test123@localhost:5432/postgres" - -proc newTestPostgresDriver*(): Future[Result[ArchiveDriver, string]] {. - async, deprecated -.} = - proc onErr(errMsg: string) {.gcsafe, closure.} = - error "error creating ArchiveDriver", error = errMsg - quit(QuitFailure) - - let - vacuum = false - migrate = true - maxNumConn = 50 - - let driverRes = - await ArchiveDriver.new(storeMessageDbUrl, vacuum, migrate, maxNumConn, onErr) - if driverRes.isErr(): - onErr("could not create archive driver: " & driverRes.error) - - return ok(driverRes.get()) diff --git a/tests/testlib/wakunode.nim b/tests/testlib/wakunode.nim index f59546ec8..e904604ab 100644 --- a/tests/testlib/wakunode.nim +++ b/tests/testlib/wakunode.nim @@ -42,7 +42,6 @@ proc defaultTestWakuConfBuilder*(): WakuConfBuilder = builder.withRelay(true) builder.withRendezvous(true) builder.storeServiceConf.withDbMigration(false) - builder.storeServiceConf.withSupportV2(false) return builder proc defaultTestWakuConf*(): WakuConf = diff --git a/tests/waku_archive_legacy/archive_utils.nim b/tests/waku_archive_legacy/archive_utils.nim deleted file mode 100644 index 8df0f5d7f..000000000 --- a/tests/waku_archive_legacy/archive_utils.nim +++ /dev/null @@ -1,55 +0,0 @@ -{.used.} - -import std/options, results, chronos, libp2p/crypto/crypto - -import - waku/[ - node/peer_manager, - waku_core, - waku_archive_legacy, - waku_archive_legacy/common, - waku_archive_legacy/driver/sqlite_driver, - waku_archive_legacy/driver/sqlite_driver/migrations, - common/databases/db_sqlite, - ], - ../testlib/[wakucore] - -proc newSqliteDatabase*(path: Option[string] = string.none()): SqliteDatabase = - SqliteDatabase.new(path.get(":memory:")).tryGet() - -proc newSqliteArchiveDriver*(): ArchiveDriver = - let database = newSqliteDatabase() - migrate(database).tryGet() - return SqliteDriver.new(database).tryGet() - -proc newWakuArchive*(driver: ArchiveDriver): WakuArchive = - WakuArchive.new(driver).get() - -proc computeArchiveCursor*( - pubsubTopic: PubsubTopic, message: WakuMessage -): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - -proc put*( - driver: ArchiveDriver, pubsubTopic: PubSubTopic, msgList: seq[WakuMessage] -): ArchiveDriver = - for msg in msgList: - let - msgDigest = computeDigest(msg) - msgHash = computeMessageHash(pubsubTopic, msg) - _ = waitFor driver.put(pubsubTopic, msg, msgDigest, msgHash, msg.timestamp) - # discard crashes - return driver - -proc newArchiveDriverWithMessages*( - pubsubTopic: PubSubTopic, msgList: seq[WakuMessage] -): ArchiveDriver = - var driver = newSqliteArchiveDriver() - driver = driver.put(pubsubTopic, msgList) - return driver diff --git a/tests/waku_archive_legacy/test_all.nim b/tests/waku_archive_legacy/test_all.nim deleted file mode 100644 index 9d45d99a1..000000000 --- a/tests/waku_archive_legacy/test_all.nim +++ /dev/null @@ -1,13 +0,0 @@ -{.used.} - -import - ./test_driver_postgres_query, - ./test_driver_postgres, - ./test_driver_queue_index, - ./test_driver_queue_pagination, - ./test_driver_queue_query, - ./test_driver_queue, - ./test_driver_sqlite_query, - ./test_driver_sqlite, - ./test_retention_policy, - ./test_waku_archive diff --git a/tests/waku_archive_legacy/test_driver_postgres.nim b/tests/waku_archive_legacy/test_driver_postgres.nim deleted file mode 100644 index 7657b6e1f..000000000 --- a/tests/waku_archive_legacy/test_driver_postgres.nim +++ /dev/null @@ -1,220 +0,0 @@ -{.used.} - -import std/[sequtils, options], testutils/unittests, chronos -import - waku/waku_archive_legacy, - waku/waku_archive_legacy/driver/postgres_driver, - waku/waku_archive/driver/postgres_driver as new_postgres_driver, - waku/waku_core, - waku/waku_core/message/digest, - ../testlib/wakucore, - ../testlib/testasync, - ../testlib/postgres_legacy, - ../testlib/postgres as new_postgres - -proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - -suite "Postgres driver": - ## Unique driver instance - var driver {.threadvar.}: postgres_driver.PostgresDriver - - ## We need to artificially create an instance of the "newDriver" - ## because this is the only one in charge of creating partitions - ## We will clean legacy store soon and this file will get removed. - var newDriver {.threadvar.}: new_postgres_driver.PostgresDriver - - asyncSetup: - let driverRes = await postgres_legacy.newTestPostgresDriver() - if driverRes.isErr(): - assert false, driverRes.error - - driver = postgres_driver.PostgresDriver(driverRes.get()) - - let newDriverRes = await new_postgres.newTestPostgresDriver() - if driverRes.isErr(): - assert false, driverRes.error - - newDriver = new_postgres_driver.PostgresDriver(newDriverRes.get()) - - asyncTeardown: - var resetRes = await driver.reset() - if resetRes.isErr(): - assert false, resetRes.error - - (await driver.close()).expect("driver to close") - - resetRes = await newDriver.reset() - if resetRes.isErr(): - assert false, resetRes.error - - (await newDriver.close()).expect("driver to close") - - asyncTest "Asynchronous queries": - var futures = newSeq[Future[ArchiveDriverResult[void]]](0) - - let beforeSleep = now() - for _ in 1 .. 100: - futures.add(driver.sleep(1)) - - await allFutures(futures) - - let diff = now() - beforeSleep - # Actually, the diff randomly goes between 1 and 2 seconds. - # although in theory it should spend 1s because we establish 100 - # connections and we spawn 100 tasks that spend ~1s each. - assert diff < 20_000_000_000 - - asyncTest "Insert a message": - const contentTopic = "test-content-topic" - const meta = "test meta" - - let msg = fakeWakuMessage(contentTopic = contentTopic, meta = meta) - - let computedDigest = computeDigest(msg) - let computedHash = computeMessageHash(DefaultPubsubTopic, msg) - - let putRes = await driver.put( - DefaultPubsubTopic, msg, computedDigest, computedHash, msg.timestamp - ) - assert putRes.isOk(), putRes.error - - let storedMsg = (await driver.getAllMessages()).tryGet() - - assert storedMsg.len == 1 - - let (pubsubTopic, actualMsg, digest, _, hash) = storedMsg[0] - assert actualMsg.contentTopic == contentTopic - assert pubsubTopic == DefaultPubsubTopic - assert toHex(computedDigest.data) == toHex(digest) - assert toHex(actualMsg.payload) == toHex(msg.payload) - assert toHex(computedHash) == toHex(hash) - assert toHex(actualMsg.meta) == toHex(msg.meta) - - asyncTest "Insert and query message": - const contentTopic1 = "test-content-topic-1" - const contentTopic2 = "test-content-topic-2" - const pubsubTopic1 = "pubsubtopic-1" - const pubsubTopic2 = "pubsubtopic-2" - - let msg1 = fakeWakuMessage(contentTopic = contentTopic1) - - var putRes = await driver.put( - pubsubTopic1, - msg1, - computeDigest(msg1), - computeMessageHash(pubsubTopic1, msg1), - msg1.timestamp, - ) - assert putRes.isOk(), putRes.error - - let msg2 = fakeWakuMessage(contentTopic = contentTopic2) - - putRes = await driver.put( - pubsubTopic2, - msg2, - computeDigest(msg2), - computeMessageHash(pubsubTopic2, msg2), - msg2.timestamp, - ) - assert putRes.isOk(), putRes.error - - let countMessagesRes = await driver.getMessagesCount() - - assert countMessagesRes.isOk(), $countMessagesRes.error - assert countMessagesRes.get() == 2 - - var messagesRes = await driver.getMessages(contentTopic = @[contentTopic1]) - - assert messagesRes.isOk(), $messagesRes.error - assert messagesRes.get().len == 1 - - # Get both content topics, check ordering - messagesRes = - await driver.getMessages(contentTopic = @[contentTopic1, contentTopic2]) - assert messagesRes.isOk(), messagesRes.error - - assert messagesRes.get().len == 2 - assert messagesRes.get()[0][1].contentTopic == contentTopic1 - - # Descending order - messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], ascendingOrder = false - ) - assert messagesRes.isOk(), messagesRes.error - - assert messagesRes.get().len == 2 - assert messagesRes.get()[0][1].contentTopic == contentTopic2 - - # cursor - # Get both content topics - messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], - cursor = some(computeTestCursor(pubsubTopic1, messagesRes.get()[1][1])), - ) - assert messagesRes.isOk() - assert messagesRes.get().len == 1 - - # Get both content topics but one pubsub topic - messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], pubsubTopic = some(pubsubTopic1) - ) - assert messagesRes.isOk(), messagesRes.error - - assert messagesRes.get().len == 1 - assert messagesRes.get()[0][1].contentTopic == contentTopic1 - - # Limit - messagesRes = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], maxPageSize = 1 - ) - assert messagesRes.isOk(), messagesRes.error - assert messagesRes.get().len == 1 - - asyncTest "Insert true duplicated messages": - # Validates that two completely equal messages can not be stored. - - let now = now() - - let msg1 = fakeWakuMessage(ts = now) - let msg2 = fakeWakuMessage(ts = now) - - let initialNumMsgs = (await driver.getMessagesCount()).valueOr: - raiseAssert "could not get num mgs correctly: " & $error - - var putRes = await driver.put( - DefaultPubsubTopic, - msg1, - computeDigest(msg1), - computeMessageHash(DefaultPubsubTopic, msg1), - msg1.timestamp, - ) - assert putRes.isOk(), putRes.error - - var newNumMsgs = (await driver.getMessagesCount()).valueOr: - raiseAssert "could not get num mgs correctly: " & $error - - assert newNumMsgs == (initialNumMsgs + 1.int64), - "wrong number of messages: " & $newNumMsgs - - putRes = await driver.put( - DefaultPubsubTopic, - msg2, - computeDigest(msg2), - computeMessageHash(DefaultPubsubTopic, msg2), - msg2.timestamp, - ) - - assert putRes.isOk() - - newNumMsgs = (await driver.getMessagesCount()).valueOr: - raiseAssert "could not get num mgs correctly: " & $error - - assert newNumMsgs == (initialNumMsgs + 1.int64), - "wrong number of messages: " & $newNumMsgs diff --git a/tests/waku_archive_legacy/test_driver_postgres_query.nim b/tests/waku_archive_legacy/test_driver_postgres_query.nim deleted file mode 100644 index ff513de76..000000000 --- a/tests/waku_archive_legacy/test_driver_postgres_query.nim +++ /dev/null @@ -1,1854 +0,0 @@ -{.used.} - -import - std/[options, sequtils, strformat, random, algorithm], - testutils/unittests, - chronos, - chronicles -import - waku/waku_archive_legacy, - waku/waku_archive_legacy/driver as driver_module, - waku/waku_archive_legacy/driver/postgres_driver, - waku/waku_archive/driver/postgres_driver as new_postgres_driver, - waku/waku_core, - waku/waku_core/message/digest, - ../testlib/common, - ../testlib/wakucore, - ../testlib/testasync, - ../testlib/postgres_legacy, - ../testlib/postgres as new_postgres, - ../testlib/testutils - -logScope: - topics = "test archive postgres driver" - -## This whole file is copied from the 'test_driver_sqlite_query.nim' file -## and it tests the same use cases but using the postgres driver. - -# Initialize the random number generator -common.randomize() - -proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - -suite "Postgres driver - queries": - ## Unique driver instance - var driver {.threadvar.}: postgres_driver.PostgresDriver - - ## We need to artificially create an instance of the "newDriver" - ## because this is the only one in charge of creating partitions - ## We will clean legacy store soon and this file will get removed. - var newDriver {.threadvar.}: new_postgres_driver.PostgresDriver - - asyncSetup: - let driverRes = await postgres_legacy.newTestPostgresDriver() - if driverRes.isErr(): - assert false, driverRes.error - - driver = postgres_driver.PostgresDriver(driverRes.get()) - - let newDriverRes = await new_postgres.newTestPostgresDriver() - if driverRes.isErr(): - assert false, driverRes.error - - newDriver = new_postgres_driver.PostgresDriver(newDriverRes.get()) - - asyncTeardown: - var resetRes = await driver.reset() - if resetRes.isErr(): - assert false, resetRes.error - - (await driver.close()).expect("driver to close") - - resetRes = await newDriver.reset() - if resetRes.isErr(): - assert false, resetRes.error - - (await newDriver.close()).expect("driver to close") - - asyncTest "no content topic": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages(maxPageSize = 5, ascendingOrder = true) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[0 .. 4] - - asyncTest "single content topic": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3] - - asyncTest "single content topic with meta field": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), - fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" - ), - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" - ), - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" - ), - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" - ), - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" - ), - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3] - - asyncTest "single content topic - descending order": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[6 .. 7].reversed() - - asyncTest "multiple content topic": - ## Given - const contentTopic1 = "test-content-topic-1" - const contentTopic2 = "test-content-topic-2" - const contentTopic3 = "test-content-topic-3" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - var res = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], - pubsubTopic = some(DefaultPubsubTopic), - maxPageSize = 2, - ascendingOrder = true, - startTime = some(ts(00)), - endTime = some(ts(40)), - ) - - ## Then - assert res.isOk(), res.error - var filteredMessages = res.tryGet().mapIt(it[1]) - check filteredMessages == expected[2 .. 3] - - ## When - ## This is very similar to the previous one but we enforce to use the prepared - ## statement by querying one single content topic - res = await driver.getMessages( - contentTopic = @[contentTopic1], - pubsubTopic = some(DefaultPubsubTopic), - maxPageSize = 2, - ascendingOrder = true, - startTime = some(ts(00)), - endTime = some(ts(40)), - ) - - ## Then - assert res.isOk(), res.error - filteredMessages = res.tryGet().mapIt(it[1]) - check filteredMessages == @[expected[2]] - - asyncTest "single content topic - no results": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - asyncTest "content topic and max page size - not enough messages stored": - ## Given - const pageSize: uint = 50 - - for t in 0 ..< 40: - let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[DefaultContentTopic], - maxPageSize = pageSize, - ascendingOrder = true, - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 40 - - asyncTest "pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages( - pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true - ) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - asyncTest "no pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages(maxPageSize = 2, ascendingOrder = true) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[0 .. 1] - - asyncTest "content topic and pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - maxPageSize = 2, - ascendingOrder = true, - ) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - asyncTest "only cursor": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = await driver.getMessages( - cursor = some(cursor), maxPageSize = 2, ascendingOrder = true - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[5 .. 6] - - asyncTest "only cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = await driver.getMessages( - cursor = some(cursor), maxPageSize = 2, ascendingOrder = false - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3].reversed() - - asyncTest "only cursor - invalid": - ## Given - const contentTopic = "test-content-topic" - - var messages = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) - let cursor = ArchiveCursor(hash: fakeCursor) - - ## When - let res = await driver.getMessages( - includeData = true, - contentTopicSeq = @[DefaultContentTopic], - pubsubTopic = none(PubsubTopic), - cursor = some(cursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hashes = @[], - maxPageSize = 5, - ascendingOrder = true, - ) - - ## Then - assert res.isOk(), res.error - - check: - res.value.len == 0 - - asyncTest "content topic and cursor": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[5 .. 6] - - asyncTest "content topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 5].reversed() - - asyncTest "pubsub topic and cursor": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeTestCursor(expected[5][0], expected[5][1]) - - ## When - let res = await driver.getMessages( - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[6 .. 7] - - asyncTest "pubsub topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeTestCursor(expected[6][0], expected[6][1]) - - ## When - let res = await driver.getMessages( - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5].reversed() - - asyncTest "only hashes - descending order": - ## Given - let timeOrigin = now() - var expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - let hashes = messages.mapIt(computeMessageHash(DefaultPubsubTopic, it)) - - for (msg, hash) in messages.zip(hashes): - require ( - await driver.put( - DefaultPubsubTopic, msg, computeDigest(msg), hash, msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages(hashes = hashes, ascendingOrder = false) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.reversed() - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages - - asyncTest "start time only": - ## Given - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6] - - asyncTest "end time only": - ## Given - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - endTime = some(ts(45, timeOrigin)), maxPageSize = 10, ascendingOrder = true - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[0 .. 4] - - asyncTest "start time and end time": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages( - startTime = some(ts(15, timeOrigin)), - endTime = some(ts(45, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[2 .. 4] - - asyncTest "invalid time range - no results": - ## Given - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(45, timeOrigin)), - endTime = some(ts(15, timeOrigin)), - maxPageSize = 2, - ascendingOrder = true, - ) - - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - asyncTest "time range start and content topic": - ## Given - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6] - - asyncTest "time range start and content topic - descending order": - ## Given - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6].reversed() - - asyncTest "time range start, single content topic and cursor": - ## Given - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[3]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[4 .. 9] - - asyncTest "time range start, single content topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[3 .. 4].reversed() - - asyncTest "time range, content topic, pubsub topic and cursor": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let timeOrigin = now() - let expected = @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[1][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(0, timeOrigin)), - endTime = some(ts(45, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[3 .. 4] - - asyncTest "time range, content topic, pubsub topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeTestCursor(expected[7][0], expected[7][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5].reversed() - - asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeTestCursor(expected[1][0], expected[1][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - assert res.isOk(), res.error - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range, descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeTestCursor(expected[1][0], expected[1][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - assert res.isOk(), res.error - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - xasyncTest "Get oldest and newest message timestamp": - ## This test no longer makes sense because that will always be controlled by the newDriver - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let oldestTime = ts(00, timeOrigin) - let newestTime = ts(100, timeOrigin) - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = oldestTime), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = newestTime), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## just keep the second resolution. - ## Notice that the oldest timestamps considers the minimum partition timestamp, which - ## is expressed in seconds. - let oldestPartitionTimestamp = - Timestamp(float(oldestTime) / 1_000_000_000) * 1_000_000_000 - - var res = await driver.getOldestMessageTimestamp() - assert res.isOk(), res.error - - ## We give certain margin of error. The oldest timestamp is obtained from - ## the oldest partition timestamp and there might be at most one second of difference - ## between the time created in the test and the oldest-partition-timestamp created within - ## the driver logic. - assert abs(res.get() - oldestPartitionTimestamp) < (2 * 1_000_000_000), - fmt"Failed to retrieve the latest timestamp {res.get()} != {oldestPartitionTimestamp}" - - res = await driver.getNewestMessageTimestamp() - assert res.isOk(), res.error - assert res.get() == newestTime, "Failed to retrieve the newest timestamp" - - xasyncTest "Delete messages older than certain timestamp": - ## This test no longer makes sense because that will always be controlled by the newDriver - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let targetTime = ts(40, timeOrigin) - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = targetTime), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - var res = await driver.getMessagesCount() - assert res.isOk(), res.error - assert res.get() == 7, "Failed to retrieve the initial number of messages" - - let deleteRes = await driver.deleteMessagesOlderThanTimestamp(targetTime) - assert deleteRes.isOk(), deleteRes.error - - res = await driver.getMessagesCount() - assert res.isOk(), res.error - assert res.get() == 3, "Failed to retrieve the # of messages after deletion" - - xasyncTest "Keep last n messages": - ## This test no longer makes sense because that will always be controlled by the newDriver - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - var res = await driver.getMessagesCount() - assert res.isOk(), res.error - assert res.get() == 7, "Failed to retrieve the initial number of messages" - - let deleteRes = await driver.deleteOldestMessagesNotWithinLimit(2) - assert deleteRes.isOk(), deleteRes.error - - res = await driver.getMessagesCount() - assert res.isOk(), res.error - assert res.get() == 2, "Failed to retrieve the # of messages after deletion" - - asyncTest "Exists table": - var existsRes = await driver.existsTable("version") - assert existsRes.isOk(), existsRes.error - check existsRes.get() == true - - asyncTest "Query by message hash only - legacy": - const contentTopic = "test-content-topic" - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ] - var messages = expected - - var hashes = newSeq[WakuMessageHash](0) - for msg in messages: - let hash = computeMessageHash(DefaultPubsubTopic, msg) - hashes.add(hash) - require ( - await driver.put( - DefaultPubsubTopic, msg, computeDigest(msg), hash, msg.timestamp - ) - ).isOk() - - let ret = (await driver.getMessages(hashes = hashes)).valueOr: - assert false, $error - return - - check: - ret.len == 3 - ## (PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash) - ret[2][4] == hashes[0] - ret[1][4] == hashes[1] - ret[0][4] == hashes[2] diff --git a/tests/waku_archive_legacy/test_driver_queue.nim b/tests/waku_archive_legacy/test_driver_queue.nim deleted file mode 100644 index aec9ad65d..000000000 --- a/tests/waku_archive_legacy/test_driver_queue.nim +++ /dev/null @@ -1,182 +0,0 @@ -{.used.} - -import std/options, results, testutils/unittests -import - waku/waku_archive_legacy, - waku/waku_archive_legacy/driver/queue_driver/queue_driver {.all.}, - waku/waku_archive_legacy/driver/queue_driver/index, - waku/waku_core - -# Helper functions - -proc genIndexedWakuMessage(i: int8): (Index, WakuMessage) = - ## Use i to generate an Index WakuMessage - var data {.noinit.}: array[32, byte] - for x in data.mitems: - x = i.byte - - let - message = WakuMessage(payload: @[byte i], timestamp: Timestamp(i)) - topic = "test-pubsub-topic" - cursor = Index( - receiverTime: Timestamp(i), - senderTime: Timestamp(i), - digest: MessageDigest(data: data), - pubsubTopic: topic, - hash: computeMessageHash(topic, message), - ) - - (cursor, message) - -proc getPrepopulatedTestQueue(unsortedSet: auto, capacity: int): QueueDriver = - let driver = QueueDriver.new(capacity) - - for i in unsortedSet: - let (index, message) = genIndexedWakuMessage(i.int8) - discard driver.add(index, message) - - driver - -procSuite "Sorted driver queue": - test "queue capacity - add a message over the limit": - ## Given - let capacity = 5 - let driver = QueueDriver.new(capacity) - - ## When - # Fill up the queue - for i in 1 .. capacity: - let (index, message) = genIndexedWakuMessage(i.int8) - require(driver.add(index, message).isOk()) - - # Add one more. Capacity should not be exceeded - let (index, message) = genIndexedWakuMessage(capacity.int8 + 1) - require(driver.add(index, message).isOk()) - - ## Then - check: - driver.len == capacity - - test "queue capacity - add message older than oldest in the queue": - ## Given - let capacity = 5 - let driver = QueueDriver.new(capacity) - - ## When - # Fill up the queue - for i in 1 .. capacity: - let (index, message) = genIndexedWakuMessage(i.int8) - require(driver.add(index, message).isOk()) - - # Attempt to add message with older value than oldest in queue should fail - let - oldestTimestamp = driver.first().get().senderTime - (index, message) = genIndexedWakuMessage(oldestTimestamp.int8 - 1) - addRes = driver.add(index, message) - - ## Then - check: - addRes.isErr() - addRes.error() == "too_old" - - check: - driver.len == capacity - - test "queue sort-on-insert": - ## Given - let - capacity = 5 - unsortedSet = [5, 1, 3, 2, 4] - let driver = getPrepopulatedTestQueue(unsortedSet, capacity) - - # Walk forward through the set and verify ascending order - var (prevSmaller, _) = genIndexedWakuMessage(min(unsortedSet).int8 - 1) - for i in driver.fwdIterator: - let (index, _) = i - check cmp(index, prevSmaller) > 0 - prevSmaller = index - - # Walk backward through the set and verify descending order - var (prevLarger, _) = genIndexedWakuMessage(max(unsortedSet).int8 + 1) - for i in driver.bwdIterator: - let (index, _) = i - check cmp(index, prevLarger) < 0 - prevLarger = index - - test "access first item from queue": - ## Given - let - capacity = 5 - unsortedSet = [5, 1, 3, 2, 4] - let driver = getPrepopulatedTestQueue(unsortedSet, capacity) - - ## When - let firstRes = driver.first() - - ## Then - check: - firstRes.isOk() - - let first = firstRes.tryGet() - check: - first.senderTime == Timestamp(1) - - test "get first item from empty queue should fail": - ## Given - let capacity = 5 - let driver = QueueDriver.new(capacity) - - ## When - let firstRes = driver.first() - - ## Then - check: - firstRes.isErr() - firstRes.error() == "Not found" - - test "access last item from queue": - ## Given - let - capacity = 5 - unsortedSet = [5, 1, 3, 2, 4] - let driver = getPrepopulatedTestQueue(unsortedSet, capacity) - - ## When - let lastRes = driver.last() - - ## Then - check: - lastRes.isOk() - - let last = lastRes.tryGet() - check: - last.senderTime == Timestamp(5) - - test "get last item from empty queue should fail": - ## Given - let capacity = 5 - let driver = QueueDriver.new(capacity) - - ## When - let lastRes = driver.last() - - ## Then - check: - lastRes.isErr() - lastRes.error() == "Not found" - - test "verify if queue contains an index": - ## Given - let - capacity = 5 - unsortedSet = [5, 1, 3, 2, 4] - let driver = getPrepopulatedTestQueue(unsortedSet, capacity) - - let - (existingIndex, _) = genIndexedWakuMessage(4) - (nonExistingIndex, _) = genIndexedWakuMessage(99) - - ## Then - check: - driver.contains(existingIndex) == true - driver.contains(nonExistingIndex) == false diff --git a/tests/waku_archive_legacy/test_driver_queue_index.nim b/tests/waku_archive_legacy/test_driver_queue_index.nim deleted file mode 100644 index 404dca8cb..000000000 --- a/tests/waku_archive_legacy/test_driver_queue_index.nim +++ /dev/null @@ -1,219 +0,0 @@ -{.used.} - -import std/[times, random], stew/byteutils, testutils/unittests, nimcrypto -import waku/waku_core, waku/waku_archive_legacy/driver/queue_driver/index - -var rng = initRand() - -## Helpers - -proc getTestTimestamp(offset = 0): Timestamp = - let now = getNanosecondTime(epochTime() + float(offset)) - Timestamp(now) - -proc hashFromStr(input: string): MDigest[256] = - var ctx: sha256 - - ctx.init() - ctx.update(input.toBytes()) - let hashed = ctx.finish() - ctx.clear() - - return hashed - -proc randomHash(): WakuMessageHash = - var hash: WakuMessageHash - - for i in 0 ..< hash.len: - let numb: byte = byte(rng.next()) - hash[i] = numb - - hash - -suite "Queue Driver - index": - ## Test vars - let - smallIndex1 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - hash: randomHash(), - ) - smallIndex2 = Index( - digest: hashFromStr("1234567"), # digest is less significant than senderTime - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - hash: randomHash(), - ) - largeIndex1 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(9000), - hash: randomHash(), - ) # only senderTime differ from smallIndex1 - largeIndex2 = Index( - digest: hashFromStr("12345"), # only digest differs from smallIndex1 - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - hash: randomHash(), - ) - eqIndex1 = Index( - digest: hashFromStr("0003"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(54321), - hash: randomHash(), - ) - eqIndex2 = Index( - digest: hashFromStr("0003"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(54321), - hash: randomHash(), - ) - eqIndex3 = Index( - digest: hashFromStr("0003"), - receiverTime: getNanosecondTime(9999), - # receiverTime difference should have no effect on comparisons - senderTime: getNanosecondTime(54321), - hash: randomHash(), - ) - diffPsTopic = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(0), - senderTime: getNanosecondTime(1000), - pubsubTopic: "zzzz", - hash: randomHash(), - ) - noSenderTime1 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(1100), - senderTime: getNanosecondTime(0), - pubsubTopic: "zzzz", - hash: randomHash(), - ) - noSenderTime2 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(10000), - senderTime: getNanosecondTime(0), - pubsubTopic: "zzzz", - hash: randomHash(), - ) - noSenderTime3 = Index( - digest: hashFromStr("1234"), - receiverTime: getNanosecondTime(1200), - senderTime: getNanosecondTime(0), - pubsubTopic: "aaaa", - hash: randomHash(), - ) - noSenderTime4 = Index( - digest: hashFromStr("0"), - receiverTime: getNanosecondTime(1200), - senderTime: getNanosecondTime(0), - pubsubTopic: "zzzz", - hash: randomHash(), - ) - - test "Index comparison": - # Index comparison with senderTime diff - check: - cmp(smallIndex1, largeIndex1) < 0 - cmp(smallIndex2, largeIndex1) < 0 - - # Index comparison with digest diff - check: - cmp(smallIndex1, smallIndex2) < 0 - cmp(smallIndex1, largeIndex2) < 0 - cmp(smallIndex2, largeIndex2) > 0 - cmp(largeIndex1, largeIndex2) > 0 - - # Index comparison when equal - check: - cmp(eqIndex1, eqIndex2) == 0 - - # pubsubTopic difference - check: - cmp(smallIndex1, diffPsTopic) < 0 - - # receiverTime diff plays no role when senderTime set - check: - cmp(eqIndex1, eqIndex3) == 0 - - # receiverTime diff plays no role when digest/pubsubTopic equal - check: - cmp(noSenderTime1, noSenderTime2) == 0 - - # sort on receiverTime with no senderTimestamp and unequal pubsubTopic - check: - cmp(noSenderTime1, noSenderTime3) < 0 - - # sort on receiverTime with no senderTimestamp and unequal digest - check: - cmp(noSenderTime1, noSenderTime4) < 0 - - # sort on receiverTime if no senderTimestamp on only one side - check: - cmp(smallIndex1, noSenderTime1) < 0 - cmp(noSenderTime1, smallIndex1) > 0 # Test symmetry - cmp(noSenderTime2, eqIndex3) < 0 - cmp(eqIndex3, noSenderTime2) > 0 # Test symmetry - - test "Index equality": - # Exactly equal - check: - eqIndex1 == eqIndex2 - - # Receiver time plays no role, even without sender time - check: - eqIndex1 == eqIndex3 - noSenderTime1 == noSenderTime2 # only receiver time differs, indices are equal - noSenderTime1 != noSenderTime3 # pubsubTopics differ - noSenderTime1 != noSenderTime4 # digests differ - - # Unequal sender time - check: - smallIndex1 != largeIndex1 - - # Unequal digest - check: - smallIndex1 != smallIndex2 - - # Unequal hash and digest - check: - smallIndex1 != eqIndex1 - - # Unequal pubsubTopic - check: - smallIndex1 != diffPsTopic - - test "Index computation should not be empty": - ## Given - let ts = getTestTimestamp() - let wm = WakuMessage(payload: @[byte 1, 2, 3], timestamp: ts) - - ## When - let ts2 = getTestTimestamp() + 10 - let index = Index.compute(wm, ts2, DefaultContentTopic) - - ## Then - check: - index.digest.data.len != 0 - index.digest.data.len == 32 # sha2 output length in bytes - index.receiverTime == ts2 # the receiver timestamp should be a non-zero value - index.senderTime == ts - index.pubsubTopic == DefaultContentTopic - - test "Index digest of two identical messsage should be the same": - ## Given - let topic = ContentTopic("test-content-topic") - let - wm1 = WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic) - wm2 = WakuMessage(payload: @[byte 1, 2, 3], contentTopic: topic) - - ## When - let ts = getTestTimestamp() - let - index1 = Index.compute(wm1, ts, DefaultPubsubTopic) - index2 = Index.compute(wm2, ts, DefaultPubsubTopic) - - ## Then - check: - index1.digest == index2.digest diff --git a/tests/waku_archive_legacy/test_driver_queue_pagination.nim b/tests/waku_archive_legacy/test_driver_queue_pagination.nim deleted file mode 100644 index 05d9759a2..000000000 --- a/tests/waku_archive_legacy/test_driver_queue_pagination.nim +++ /dev/null @@ -1,405 +0,0 @@ -{.used.} - -import - std/[options, sequtils, algorithm], testutils/unittests, libp2p/protobuf/minprotobuf -import - waku/waku_archive_legacy, - waku/waku_archive_legacy/driver/queue_driver/queue_driver {.all.}, - waku/waku_archive_legacy/driver/queue_driver/index, - waku/waku_core, - ../testlib/wakucore - -proc getTestQueueDriver(numMessages: int): QueueDriver = - let testQueueDriver = QueueDriver.new(numMessages) - - var data {.noinit.}: array[32, byte] - for x in data.mitems: - x = 1 - - for i in 0 ..< numMessages: - let msg = WakuMessage(payload: @[byte i], timestamp: Timestamp(i)) - - let index = Index( - receiverTime: Timestamp(i), - senderTime: Timestamp(i), - digest: MessageDigest(data: data), - hash: computeMessageHash(DefaultPubsubTopic, msg), - ) - - discard testQueueDriver.add(index, msg) - - return testQueueDriver - -procSuite "Queue driver - pagination": - let driver = getTestQueueDriver(10) - let - indexList: seq[Index] = toSeq(driver.fwdIterator()).mapIt(it[0]) - msgList: seq[WakuMessage] = toSeq(driver.fwdIterator()).mapIt(it[1]) - - test "Forward pagination - normal pagination": - ## Given - let - pageSize: uint = 2 - cursor: Option[Index] = some(indexList[3]) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 2 - data == msgList[4 .. 5] - - test "Forward pagination - initial pagination request with an empty cursor": - ## Given - let - pageSize: uint = 2 - cursor: Option[Index] = none(Index) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 2 - data == msgList[0 .. 1] - - test "Forward pagination - initial pagination request with an empty cursor to fetch the entire history": - ## Given - let - pageSize: uint = 13 - cursor: Option[Index] = none(Index) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 10 - data == msgList[0 .. 9] - - test "Forward pagination - empty msgList": - ## Given - let driver = getTestQueueDriver(0) - let - pageSize: uint = 2 - cursor: Option[Index] = none(Index) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 0 - - test "Forward pagination - page size larger than the remaining messages": - ## Given - let - pageSize: uint = 10 - cursor: Option[Index] = some(indexList[3]) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 6 - data == msgList[4 .. 9] - - test "Forward pagination - page size larger than the maximum allowed page size": - ## Given - let - pageSize: uint = MaxPageSize + 1 - cursor: Option[Index] = some(indexList[3]) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - uint(data.len) <= MaxPageSize - - test "Forward pagination - cursor pointing to the end of the message list": - ## Given - let - pageSize: uint = 10 - cursor: Option[Index] = some(indexList[9]) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 0 - - test "Forward pagination - invalid cursor": - ## Given - let msg = fakeWakuMessage(payload = @[byte 10]) - let index = ArchiveCursor( - pubsubTopic: DefaultPubsubTopic, - senderTime: msg.timestamp, - storeTime: msg.timestamp, - digest: computeDigest(msg), - ).toIndex() - - let - pageSize: uint = 10 - cursor: Option[Index] = some(index) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let error = page.tryError() - check: - error == QueueDriverErrorKind.INVALID_CURSOR - - test "Forward pagination - initial paging query over a message list with one message": - ## Given - let driver = getTestQueueDriver(1) - let - pageSize: uint = 10 - cursor: Option[Index] = none(Index) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 1 - - test "Forward pagination - pagination over a message list with one message": - ## Given - let driver = getTestQueueDriver(1) - let - pageSize: uint = 10 - cursor: Option[Index] = some(indexList[0]) - forward: bool = true - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 0 - - test "Forward pagination - with pradicate": - ## Given - let - pageSize: uint = 3 - cursor: Option[Index] = none(Index) - forward = true - - proc onlyEvenTimes(index: Index, msg: WakuMessage): bool = - msg.timestamp.int64 mod 2 == 0 - - ## When - let page = driver.getPage( - pageSize = pageSize, forward = forward, cursor = cursor, predicate = onlyEvenTimes - ) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.mapIt(it.timestamp.int) == @[0, 2, 4] - - test "Backward pagination - normal pagination": - ## Given - let - pageSize: uint = 2 - cursor: Option[Index] = some(indexList[3]) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data == msgList[1 .. 2].reversed - - test "Backward pagination - empty msgList": - ## Given - let driver = getTestQueueDriver(0) - let - pageSize: uint = 2 - cursor: Option[Index] = none(Index) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 0 - - test "Backward pagination - initial pagination request with an empty cursor": - ## Given - let - pageSize: uint = 2 - cursor: Option[Index] = none(Index) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 2 - data == msgList[8 .. 9].reversed - - test "Backward pagination - initial pagination request with an empty cursor to fetch the entire history": - ## Given - let - pageSize: uint = 13 - cursor: Option[Index] = none(Index) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 10 - data == msgList[0 .. 9].reversed - - test "Backward pagination - page size larger than the remaining messages": - ## Given - let - pageSize: uint = 5 - cursor: Option[Index] = some(indexList[3]) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data == msgList[0 .. 2].reversed - - test "Backward pagination - page size larger than the Maximum allowed page size": - ## Given - let - pageSize: uint = MaxPageSize + 1 - cursor: Option[Index] = some(indexList[3]) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - uint(data.len) <= MaxPageSize - - test "Backward pagination - cursor pointing to the begining of the message list": - ## Given - let - pageSize: uint = 5 - cursor: Option[Index] = some(indexList[0]) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 0 - - test "Backward pagination - invalid cursor": - ## Given - let msg = fakeWakuMessage(payload = @[byte 10]) - let index = ArchiveCursor( - pubsubTopic: DefaultPubsubTopic, - senderTime: msg.timestamp, - storeTime: msg.timestamp, - digest: computeDigest(msg), - ).toIndex() - - let - pageSize: uint = 2 - cursor: Option[Index] = some(index) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let error = page.tryError() - check: - error == QueueDriverErrorKind.INVALID_CURSOR - - test "Backward pagination - initial paging query over a message list with one message": - ## Given - let driver = getTestQueueDriver(1) - let - pageSize: uint = 10 - cursor: Option[Index] = none(Index) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 1 - - test "Backward pagination - paging query over a message list with one message": - ## Given - let driver = getTestQueueDriver(1) - let - pageSize: uint = 10 - cursor: Option[Index] = some(indexList[0]) - forward: bool = false - - ## When - let page = driver.getPage(pageSize = pageSize, forward = forward, cursor = cursor) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.len == 0 - - test "Backward pagination - with predicate": - ## Given - let - pageSize: uint = 3 - cursor: Option[Index] = none(Index) - forward = false - - proc onlyOddTimes(index: Index, msg: WakuMessage): bool = - msg.timestamp.int64 mod 2 != 0 - - ## When - let page = driver.getPage( - pageSize = pageSize, forward = forward, cursor = cursor, predicate = onlyOddTimes - ) - - ## Then - let data = page.tryGet().mapIt(it[1]) - check: - data.mapIt(it.timestamp.int) == @[5, 7, 9].reversed diff --git a/tests/waku_archive_legacy/test_driver_queue_query.nim b/tests/waku_archive_legacy/test_driver_queue_query.nim deleted file mode 100644 index 0726d1931..000000000 --- a/tests/waku_archive_legacy/test_driver_queue_query.nim +++ /dev/null @@ -1,1668 +0,0 @@ -{.used.} - -import - std/[options, sequtils, random, algorithm], testutils/unittests, chronos, chronicles -import - waku/waku_archive_legacy, - waku/waku_archive_legacy/driver/queue_driver, - waku/waku_core, - waku/waku_core/message/digest, - ../testlib/common, - ../testlib/wakucore - -logScope: - topics = "test archive queue_driver" - -# Initialize the random number generator -common.randomize() - -proc newTestSqliteDriver(): ArchiveDriver = - QueueDriver.new(capacity = 50) - -proc computeTestCursor(pubsubTopic: PubsubTopic, message: WakuMessage): ArchiveCursor = - ArchiveCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - hash: computeMessageHash(pubsubTopic, message), - ) - -suite "Queue driver - query by content topic": - test "no content topic": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages(maxPageSize = 5, ascendingOrder = true) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[0 .. 4] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "single content topic": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "single content topic - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[6 .. 7].reversed() - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "multiple content topic": - ## Given - const contentTopic1 = "test-content-topic-1" - const contentTopic2 = "test-content-topic-2" - const contentTopic3 = "test-content-topic-3" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], - maxPageSize = 2, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "single content topic - no results": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "content topic and max page size - not enough messages stored": - ## Given - const pageSize: uint = 50 - - let driver = newTestSqliteDriver() - - for t in 0 ..< 40: - let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[DefaultContentTopic], - maxPageSize = pageSize, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 40 - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - -suite "SQLite driver - query by pubsub topic": - test "pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "no pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages(maxPageSize = 2, ascendingOrder = true) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[0 .. 1] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "content topic and pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - maxPageSize = 2, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - -suite "Queue driver - query by cursor": - test "only cursor": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = waitFor driver.getMessages( - cursor = some(cursor), maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[5 .. 6] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "only cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = waitFor driver.getMessages( - cursor = some(cursor), maxPageSize = 2, ascendingOrder = false - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3].reversed() - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "only cursor - invalid": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - var messages = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) - let cursor = ArchiveCursor(hash: fakeCursor) - - ## When - let res = waitFor driver.getMessages( - includeData = true, - contentTopic = @[DefaultContentTopic], - pubsubTopic = none(PubsubTopic), - cursor = some(cursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hashes = @[], - maxPageSize = 5, - ascendingOrder = true, - ) - - ## Then - check: - res.isErr() - res.error == "invalid_cursor" - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "content topic and cursor": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[5 .. 6] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "content topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 5].reversed() - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "pubsub topic and cursor": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - let cursor = computeTestCursor(expected[5][0], expected[5][1]) - - ## When - let res = waitFor driver.getMessages( - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[6 .. 7] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "pubsub topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - let cursor = computeTestCursor(expected[6][0], expected[6][1]) - - ## When - let res = waitFor driver.getMessages( - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5].reversed() - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - -suite "Queue driver - query by time range": - test "start time only": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "end time only": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - endTime = some(ts(45, timeOrigin)), maxPageSize = 10, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[0 .. 4] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "start time and end time": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - startTime = some(ts(15, timeOrigin)), - endTime = some(ts(45, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[2 .. 4] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "invalid time range - no results": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(45, timeOrigin)), - endTime = some(ts(15, timeOrigin)), - maxPageSize = 2, - ascendingOrder = true, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - asynctest "time range start and content topic": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6] - - ## Cleanup - (await driver.close()).expect("driver to close") - - test "time range start and content topic - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6].reversed() - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - asynctest "time range start, single content topic and cursor": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[3]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[4 .. 9] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asynctest "time range start, single content topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - let retFut = await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - require retFut.isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[6]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[3 .. 4].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - - test "time range, content topic, pubsub topic and cursor": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - let cursor = computeTestCursor(DefaultPubsubTopic, expected[1][1]) - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(0, timeOrigin)), - endTime = some(ts(45, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[3 .. 4] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "time range, content topic, pubsub topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - let cursor = computeTestCursor(expected[7][0], expected[7][1]) - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5].reversed() - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - let cursor = computeTestCursor(expected[1][0], expected[1][1]) - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range, descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newTestSqliteDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - let retFut = waitFor driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - require retFut.isOk() - - let cursor = computeTestCursor(expected[1][0], expected[1][1]) - - ## When - let res = waitFor driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - ## Cleanup - (waitFor driver.close()).expect("driver to close") diff --git a/tests/waku_archive_legacy/test_driver_sqlite.nim b/tests/waku_archive_legacy/test_driver_sqlite.nim deleted file mode 100644 index 9d8c4d14b..000000000 --- a/tests/waku_archive_legacy/test_driver_sqlite.nim +++ /dev/null @@ -1,58 +0,0 @@ -{.used.} - -import std/sequtils, testutils/unittests, chronos -import - waku/waku_archive_legacy, - waku/waku_archive_legacy/driver/sqlite_driver, - waku/waku_core, - ../waku_archive_legacy/archive_utils, - ../testlib/wakucore - -suite "SQLite driver": - test "init driver and database": - ## Given - let database = newSqliteDatabase() - - ## When - let driverRes = SqliteDriver.new(database) - - ## Then - check: - driverRes.isOk() - - let driver: ArchiveDriver = driverRes.tryGet() - check: - not driver.isNil() - - ## Cleanup - (waitFor driver.close()).expect("driver to close") - - test "insert a message": - ## Given - const contentTopic = "test-content-topic" - const meta = "test meta" - - let driver = newSqliteArchiveDriver() - - let msg = fakeWakuMessage(contentTopic = contentTopic, meta = meta) - let msgHash = computeMessageHash(DefaultPubsubTopic, msg) - - ## When - let putRes = waitFor driver.put( - DefaultPubsubTopic, msg, computeDigest(msg), msgHash, msg.timestamp - ) - - ## Then - check: - putRes.isOk() - - let storedMsg = (waitFor driver.getAllMessages()).tryGet() - check: - storedMsg.len == 1 - storedMsg.all do(item: auto) -> bool: - let (pubsubTopic, actualMsg, _, _, hash) = item - actualMsg.contentTopic == contentTopic and pubsubTopic == DefaultPubsubTopic and - hash == msgHash and msg.meta == actualMsg.meta - - ## Cleanup - (waitFor driver.close()).expect("driver to close") diff --git a/tests/waku_archive_legacy/test_driver_sqlite_query.nim b/tests/waku_archive_legacy/test_driver_sqlite_query.nim deleted file mode 100644 index 3c3b55232..000000000 --- a/tests/waku_archive_legacy/test_driver_sqlite_query.nim +++ /dev/null @@ -1,1745 +0,0 @@ -{.used.} - -import - std/[options, sequtils, random, algorithm], testutils/unittests, chronos, chronicles - -import - waku/waku_archive_legacy, - waku/waku_core, - waku/waku_core/message/digest, - ../testlib/common, - ../testlib/wakucore, - ../waku_archive_legacy/archive_utils - -logScope: - topics = "test archive _driver" - -# Initialize the random number generator -common.randomize() - -suite "SQLite driver - query by content topic": - asyncTest "no content topic": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages(maxPageSize = 5, ascendingOrder = true) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[0 .. 4] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "single content topic": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "single content topic with meta field": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00), meta = "meta-0"), - fakeWakuMessage(@[byte 1], ts = ts(10), meta = "meta-1"), - fakeWakuMessage( - @[byte 2], contentTopic = contentTopic, ts = ts(20), meta = "meta-2" - ), - fakeWakuMessage( - @[byte 3], contentTopic = contentTopic, ts = ts(30), meta = "meta-3" - ), - fakeWakuMessage( - @[byte 4], contentTopic = contentTopic, ts = ts(40), meta = "meta-4" - ), - fakeWakuMessage( - @[byte 5], contentTopic = contentTopic, ts = ts(50), meta = "meta-5" - ), - fakeWakuMessage( - @[byte 6], contentTopic = contentTopic, ts = ts(60), meta = "meta-6" - ), - fakeWakuMessage( - @[byte 7], contentTopic = contentTopic, ts = ts(70), meta = "meta-7" - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "single content topic - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = false - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[6 .. 7].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "multiple content topic": - ## Given - const contentTopic1 = "test-content-topic-1" - const contentTopic2 = "test-content-topic-2" - const contentTopic3 = "test-content-topic-3" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic1, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic2, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic3, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic1, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic2, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic3, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic1, contentTopic2], - maxPageSize = 2, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "single content topic - no results": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = DefaultContentTopic, ts = ts(00)), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic, ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic, ts = ts(40)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "content topic and max page size - not enough messages stored": - ## Given - const pageSize: uint = 50 - - let driver = newSqliteArchiveDriver() - - for t in 0 ..< 40: - let msg = fakeWakuMessage(@[byte t], DefaultContentTopic, ts = ts(t)) - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[DefaultContentTopic], - maxPageSize = pageSize, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 40 - - ## Cleanup - (await driver.close()).expect("driver to close") - -suite "SQLite driver - query by pubsub topic": - asyncTest "pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages( - pubsubTopic = some(pubsubTopic), maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "no pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages(maxPageSize = 2, ascendingOrder = true) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[0 .. 1] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "content topic and pubsub topic": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - maxPageSize = 2, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - ## Cleanup - (await driver.close()).expect("driver to close") - -suite "SQLite driver - query by cursor": - asyncTest "only cursor": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = await driver.getMessages( - cursor = some(cursor), maxPageSize = 2, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[5 .. 6] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "only cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = await driver.getMessages( - cursor = some(cursor), maxPageSize = 2, ascendingOrder = false - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 3].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "only cursor - invalid": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - var messages = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let fakeCursor = computeMessageHash(DefaultPubsubTopic, fakeWakuMessage()) - let cursor = ArchiveCursor(hash: fakeCursor) - - ## When - let res = await driver.getMessages( - includeData = true, - contentTopic = @[DefaultContentTopic], - pubsubTopic = none(PubsubTopic), - cursor = some(cursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hashes = @[], - maxPageSize = 5, - ascendingOrder = true, - ) - - ## Then - check: - res.isErr() - res.error == "cursor not found" - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "content topic and cursor": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), # << cursor - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), - fakeWakuMessage(@[byte 7], ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[4]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[5 .. 6] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "content topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let expected = @[ - fakeWakuMessage(@[byte 0], ts = ts(00)), - fakeWakuMessage(@[byte 1], ts = ts(10)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60)), # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[6]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 5].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "pubsub topic and cursor": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeArchiveCursor(expected[5][0], expected[5][1]) - - ## When - let res = await driver.getMessages( - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[6 .. 7] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "pubsub topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), # << cursor - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeArchiveCursor(expected[6][0], expected[6][1]) - - ## When - let res = await driver.getMessages( - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - -suite "SQLite driver - query by time range": - asyncTest "start time only": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - startTime = some(ts(15, timeOrigin)), maxPageSize = 10, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "end time only": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - endTime = some(ts(45, timeOrigin)), maxPageSize = 10, ascendingOrder = true - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[0 .. 4] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "start time and end time": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # start_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - ## When - let res = await driver.getMessages( - startTime = some(ts(15, timeOrigin)), - endTime = some(ts(45, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[2 .. 4] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "invalid time range - no results": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - # end_time - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(45, timeOrigin)), - endTime = some(ts(15, timeOrigin)), - maxPageSize = 2, - ascendingOrder = true, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range start and content topic": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range start and content topic - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[2 .. 6].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range start, single content topic and cursor": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[3]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[4 .. 9] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range start, single content topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - fakeWakuMessage(@[byte 0], contentTopic = contentTopic, ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 1], contentTopic = contentTopic, ts = ts(10, timeOrigin)), - # start_time - fakeWakuMessage(@[byte 2], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 5], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 6], contentTopic = contentTopic, ts = ts(60, timeOrigin)), - # << cursor - fakeWakuMessage(@[byte 7], contentTopic = contentTopic, ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", sequence = messages.mapIt(it.payload) - - for msg in messages: - require ( - await driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[6]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - cursor = some(cursor), - startTime = some(ts(15, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expected[3 .. 4].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range, content topic, pubsub topic and cursor": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - # start_time - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - # end_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeArchiveCursor(DefaultPubsubTopic, expected[1][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(0, timeOrigin)), - endTime = some(ts(45, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[3 .. 4] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range, content topic, pubsub topic and cursor - descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeArchiveCursor(expected[7][0], expected[7][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5].reversed() - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeArchiveCursor(expected[1][0], expected[1][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = true, - ) - - ## Then - check: - res.isOk() - - let expectedMessages = expected.mapIt(it[1]) - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages == expectedMessages[4 .. 5] - - ## Cleanup - (await driver.close()).expect("driver to close") - - asyncTest "time range, content topic, pubsub topic and cursor - cursor timestamp out of time range, descending order": - ## Given - const contentTopic = "test-content-topic" - const pubsubTopic = "test-pubsub-topic" - - let driver = newSqliteArchiveDriver() - - let timeOrigin = now() - let expected = @[ - (DefaultPubsubTopic, fakeWakuMessage(@[byte 0], ts = ts(00, timeOrigin))), - (DefaultPubsubTopic, fakeWakuMessage(@[byte 1], ts = ts(10, timeOrigin))), - # << cursor - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 2], contentTopic = contentTopic, ts = ts(20, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 3], contentTopic = contentTopic, ts = ts(30, timeOrigin)), - ), - # start_time - ( - pubsubTopic, - fakeWakuMessage(@[byte 4], contentTopic = contentTopic, ts = ts(40, timeOrigin)), - ), - ( - pubsubTopic, - fakeWakuMessage(@[byte 5], contentTopic = contentTopic, ts = ts(50, timeOrigin)), - ), - (pubsubTopic, fakeWakuMessage(@[byte 6], ts = ts(60, timeOrigin))), - (pubsubTopic, fakeWakuMessage(@[byte 7], ts = ts(70, timeOrigin))), - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 8], contentTopic = contentTopic, ts = ts(80, timeOrigin)), - ), - # end_time - ( - DefaultPubsubTopic, - fakeWakuMessage(@[byte 9], contentTopic = contentTopic, ts = ts(90, timeOrigin)), - ), - ] - var messages = expected - - shuffle(messages) - info "randomized message insertion sequence", - sequence = messages.mapIt(it[1].payload) - - for row in messages: - let (topic, msg) = row - require ( - await driver.put( - topic, msg, computeDigest(msg), computeMessageHash(topic, msg), msg.timestamp - ) - ).isOk() - - let cursor = computeArchiveCursor(expected[1][0], expected[1][1]) - - ## When - let res = await driver.getMessages( - contentTopic = @[contentTopic], - pubsubTopic = some(pubsubTopic), - cursor = some(cursor), - startTime = some(ts(35, timeOrigin)), - endTime = some(ts(85, timeOrigin)), - maxPageSize = 10, - ascendingOrder = false, - ) - - ## Then - check: - res.isOk() - - let filteredMessages = res.tryGet().mapIt(it[1]) - check: - filteredMessages.len == 0 - - ## Cleanup - (await driver.close()).expect("driver to close") diff --git a/tests/waku_archive_legacy/test_waku_archive.nim b/tests/waku_archive_legacy/test_waku_archive.nim deleted file mode 100644 index f6373608d..000000000 --- a/tests/waku_archive_legacy/test_waku_archive.nim +++ /dev/null @@ -1,532 +0,0 @@ -{.used.} - -import std/[options, sequtils], testutils/unittests, chronos, libp2p/crypto/crypto - -import - waku/common/paging, - waku/waku_core, - waku/waku_core/message/digest, - waku/waku_archive_legacy, - ../waku_archive_legacy/archive_utils, - ../testlib/wakucore - -suite "Waku Archive - message handling": - test "it should archive a valid and non-ephemeral message": - ## Setup - let driver = newSqliteArchiveDriver() - let archive = newWakuArchive(driver) - - ## Given - let validSenderTime = now() - let message = fakeWakuMessage(ephemeral = false, ts = validSenderTime) - - ## When - waitFor archive.handleMessage(DefaultPubSubTopic, message) - - ## Then - check: - (waitFor driver.getMessagesCount()).tryGet() == 1 - - test "it should not archive ephemeral messages": - ## Setup - let driver = newSqliteArchiveDriver() - let archive = newWakuArchive(driver) - - ## Given - let msgList = @[ - fakeWakuMessage(ephemeral = false, payload = "1"), - fakeWakuMessage(ephemeral = true, payload = "2"), - fakeWakuMessage(ephemeral = true, payload = "3"), - fakeWakuMessage(ephemeral = true, payload = "4"), - fakeWakuMessage(ephemeral = false, payload = "5"), - ] - - ## When - for msg in msgList: - waitFor archive.handleMessage(DefaultPubsubTopic, msg) - - ## Then - check: - (waitFor driver.getMessagesCount()).tryGet() == 2 - - test "it should archive a message with no sender timestamp": - ## Setup - let driver = newSqliteArchiveDriver() - let archive = newWakuArchive(driver) - - ## Given - let invalidSenderTime = 0 - let message = fakeWakuMessage(ts = invalidSenderTime) - - ## When - waitFor archive.handleMessage(DefaultPubSubTopic, message) - - ## Then - check: - (waitFor driver.getMessagesCount()).tryGet() == 1 - - test "it should not archive a message with a sender time variance greater than max time variance (future)": - ## Setup - let driver = newSqliteArchiveDriver() - let archive = newWakuArchive(driver) - - ## Given - let - now = now() - invalidSenderTime = now + MaxMessageTimestampVariance + 1_000_000_000 - # 1 second over the max variance - - let message = fakeWakuMessage(ts = invalidSenderTime) - - ## When - waitFor archive.handleMessage(DefaultPubSubTopic, message) - - ## Then - check: - (waitFor driver.getMessagesCount()).tryGet() == 0 - - test "it should not archive a message with a sender time variance greater than max time variance (past)": - ## Setup - let driver = newSqliteArchiveDriver() - let archive = newWakuArchive(driver) - - ## Given - let - now = now() - invalidSenderTime = now - MaxMessageTimestampVariance - 1 - - let message = fakeWakuMessage(ts = invalidSenderTime) - - ## When - waitFor archive.handleMessage(DefaultPubSubTopic, message) - - ## Then - check: - (waitFor driver.getMessagesCount()).tryGet() == 0 - -procSuite "Waku Archive - find messages": - ## Fixtures - let timeOrigin = now() - let msgListA = @[ - fakeWakuMessage( - @[byte 00], contentTopic = ContentTopic("2"), ts = ts(00, timeOrigin) - ), - fakeWakuMessage( - @[byte 01], contentTopic = ContentTopic("1"), ts = ts(10, timeOrigin) - ), - fakeWakuMessage( - @[byte 02], contentTopic = ContentTopic("2"), ts = ts(20, timeOrigin) - ), - fakeWakuMessage( - @[byte 03], contentTopic = ContentTopic("1"), ts = ts(30, timeOrigin) - ), - fakeWakuMessage( - @[byte 04], contentTopic = ContentTopic("2"), ts = ts(40, timeOrigin) - ), - fakeWakuMessage( - @[byte 05], contentTopic = ContentTopic("1"), ts = ts(50, timeOrigin) - ), - fakeWakuMessage( - @[byte 06], contentTopic = ContentTopic("2"), ts = ts(60, timeOrigin) - ), - fakeWakuMessage( - @[byte 07], contentTopic = ContentTopic("1"), ts = ts(70, timeOrigin) - ), - fakeWakuMessage( - @[byte 08], contentTopic = ContentTopic("2"), ts = ts(80, timeOrigin) - ), - fakeWakuMessage( - @[byte 09], contentTopic = ContentTopic("1"), ts = ts(90, timeOrigin) - ), - ] - - let archiveA = block: - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - for msg in msgListA: - require ( - waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - archive - - test "handle query": - ## Setup - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - let topic = ContentTopic("1") - let - msg1 = fakeWakuMessage(contentTopic = topic) - msg2 = fakeWakuMessage() - - waitFor archive.handleMessage("foo", msg1) - waitFor archive.handleMessage("foo", msg2) - - ## Given - let req = ArchiveQuery(includeData: true, contentTopics: @[topic]) - - ## When - let queryRes = waitFor archive.findMessages(req) - - ## Then - check: - queryRes.isOk() - - let response = queryRes.tryGet() - check: - response.messages.len == 1 - response.messages == @[msg1] - - test "handle query with multiple content filters": - ## Setup - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - let - topic1 = ContentTopic("1") - topic2 = ContentTopic("2") - topic3 = ContentTopic("3") - - let - msg1 = fakeWakuMessage(contentTopic = topic1) - msg2 = fakeWakuMessage(contentTopic = topic2) - msg3 = fakeWakuMessage(contentTopic = topic3) - - waitFor archive.handleMessage("foo", msg1) - waitFor archive.handleMessage("foo", msg2) - waitFor archive.handleMessage("foo", msg3) - - ## Given - let req = ArchiveQuery(includeData: true, contentTopics: @[topic1, topic3]) - - ## When - let queryRes = waitFor archive.findMessages(req) - - ## Then - check: - queryRes.isOk() - - let response = queryRes.tryGet() - check: - response.messages.len() == 2 - response.messages.anyIt(it == msg1) - response.messages.anyIt(it == msg3) - - test "handle query with more than 10 content filters": - ## Setup - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - let queryTopics = toSeq(1 .. 15).mapIt(ContentTopic($it)) - - ## Given - let req = ArchiveQuery(contentTopics: queryTopics) - - ## When - let queryRes = waitFor archive.findMessages(req) - - ## Then - check: - queryRes.isErr() - - let error = queryRes.tryError() - check: - error.kind == ArchiveErrorKind.INVALID_QUERY - error.cause == "too many content topics" - - test "handle query with pubsub topic filter": - ## Setup - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - let - pubsubTopic1 = "queried-topic" - pubsubTopic2 = "non-queried-topic" - - let - contentTopic1 = ContentTopic("1") - contentTopic2 = ContentTopic("2") - contentTopic3 = ContentTopic("3") - - let - msg1 = fakeWakuMessage(contentTopic = contentTopic1) - msg2 = fakeWakuMessage(contentTopic = contentTopic2) - msg3 = fakeWakuMessage(contentTopic = contentTopic3) - - waitFor archive.handleMessage(pubsubtopic1, msg1) - waitFor archive.handleMessage(pubsubtopic2, msg2) - waitFor archive.handleMessage(pubsubtopic2, msg3) - - ## Given - # This query targets: pubsubtopic1 AND (contentTopic1 OR contentTopic3) - let req = ArchiveQuery( - includeData: true, - pubsubTopic: some(pubsubTopic1), - contentTopics: @[contentTopic1, contentTopic3], - ) - - ## When - let queryRes = waitFor archive.findMessages(req) - - ## Then - check: - queryRes.isOk() - - let response = queryRes.tryGet() - check: - response.messages.len() == 1 - response.messages.anyIt(it == msg1) - - test "handle query with pubsub topic filter - no match": - ## Setup - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - let - pubsubtopic1 = "queried-topic" - pubsubtopic2 = "non-queried-topic" - - let - msg1 = fakeWakuMessage() - msg2 = fakeWakuMessage() - msg3 = fakeWakuMessage() - - waitFor archive.handleMessage(pubsubtopic2, msg1) - waitFor archive.handleMessage(pubsubtopic2, msg2) - waitFor archive.handleMessage(pubsubtopic2, msg3) - - ## Given - let req = ArchiveQuery(pubsubTopic: some(pubsubTopic1)) - - ## When - let res = waitFor archive.findMessages(req) - - ## Then - check: - res.isOk() - - let response = res.tryGet() - check: - response.messages.len() == 0 - - test "handle query with pubsub topic filter - match the entire stored messages": - ## Setup - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - let pubsubTopic = "queried-topic" - - let - msg1 = fakeWakuMessage(payload = "TEST-1") - msg2 = fakeWakuMessage(payload = "TEST-2") - msg3 = fakeWakuMessage(payload = "TEST-3") - - waitFor archive.handleMessage(pubsubTopic, msg1) - waitFor archive.handleMessage(pubsubTopic, msg2) - waitFor archive.handleMessage(pubsubTopic, msg3) - - ## Given - let req = ArchiveQuery(includeData: true, pubsubTopic: some(pubsubTopic)) - - ## When - let res = waitFor archive.findMessages(req) - - ## Then - check: - res.isOk() - - let response = res.tryGet() - check: - response.messages.len() == 3 - response.messages.anyIt(it == msg1) - response.messages.anyIt(it == msg2) - response.messages.anyIt(it == msg3) - - test "handle query with forward pagination": - ## Given - let req = - ArchiveQuery(includeData: true, pageSize: 4, direction: PagingDirection.FORWARD) - - ## When - var nextReq = req # copy - - var pages = newSeq[seq[WakuMessage]](3) - var cursors = newSeq[Option[ArchiveCursor]](3) - - for i in 0 ..< 3: - let res = waitFor archiveA.findMessages(nextReq) - require res.isOk() - - # Keep query response content - let response = res.get() - pages[i] = response.messages - cursors[i] = response.cursor - - # Set/update the request cursor - nextReq.cursor = cursors[i] - - ## Then - check: - cursors[0] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[3])) - cursors[1] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[7])) - cursors[2] == none(ArchiveCursor) - - check: - pages[0] == msgListA[0 .. 3] - pages[1] == msgListA[4 .. 7] - pages[2] == msgListA[8 .. 9] - - test "handle query with backward pagination": - ## Given - let req = - ArchiveQuery(includeData: true, pageSize: 4, direction: PagingDirection.BACKWARD) - - ## When - var nextReq = req # copy - - var pages = newSeq[seq[WakuMessage]](3) - var cursors = newSeq[Option[ArchiveCursor]](3) - - for i in 0 ..< 3: - let res = waitFor archiveA.findMessages(nextReq) - require res.isOk() - - # Keep query response content - let response = res.get() - pages[i] = response.messages - cursors[i] = response.cursor - - # Set/update the request cursor - nextReq.cursor = cursors[i] - - ## Then - check: - cursors[0] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[6])) - cursors[1] == some(computeArchiveCursor(DefaultPubsubTopic, msgListA[2])) - cursors[2] == none(ArchiveCursor) - - check: - pages[0] == msgListA[6 .. 9] - pages[1] == msgListA[2 .. 5] - pages[2] == msgListA[0 .. 1] - - test "handle query with no paging info - auto-pagination": - ## Setup - let - driver = newSqliteArchiveDriver() - archive = newWakuArchive(driver) - - let msgList = @[ - fakeWakuMessage(@[byte 0], contentTopic = ContentTopic("2")), - fakeWakuMessage(@[byte 1], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 2], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 3], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 4], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 5], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 6], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 7], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 8], contentTopic = DefaultContentTopic), - fakeWakuMessage(@[byte 9], contentTopic = ContentTopic("2")), - ] - - for msg in msgList: - require ( - waitFor driver.put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - ).isOk() - - ## Given - let req = ArchiveQuery(includeData: true, contentTopics: @[DefaultContentTopic]) - - ## When - let res = waitFor archive.findMessages(req) - - ## Then - check: - res.isOk() - - let response = res.tryGet() - check: - ## No pagination specified. Response will be auto-paginated with - ## up to MaxPageSize messages per page. - response.messages.len() == 8 - response.cursor.isNone() - - test "handle temporal history query with a valid time window": - ## Given - let req = ArchiveQuery( - includeData: true, - contentTopics: @[ContentTopic("1")], - startTime: some(ts(15, timeOrigin)), - endTime: some(ts(55, timeOrigin)), - direction: PagingDirection.FORWARD, - ) - - ## When - let res = waitFor archiveA.findMessages(req) - - ## Then - check res.isOk() - - let response = res.tryGet() - check: - response.messages.len() == 2 - response.messages.mapIt(it.timestamp) == @[ts(30, timeOrigin), ts(50, timeOrigin)] - - test "handle temporal history query with a zero-size time window": - ## A zero-size window results in an empty list of history messages - ## Given - let req = ArchiveQuery( - contentTopics: @[ContentTopic("1")], - startTime: some(Timestamp(2)), - endTime: some(Timestamp(2)), - ) - - ## When - let res = waitFor archiveA.findMessages(req) - - ## Then - check res.isOk() - - let response = res.tryGet() - check: - response.messages.len == 0 - - test "handle temporal history query with an invalid time window": - ## A history query with an invalid time range results in an empty list of history messages - ## Given - let req = ArchiveQuery( - contentTopics: @[ContentTopic("1")], - startTime: some(Timestamp(5)), - endTime: some(Timestamp(2)), - ) - - ## When - let res = waitFor archiveA.findMessages(req) - - ## Then - check res.isOk() - - let response = res.tryGet() - check: - response.messages.len == 0 diff --git a/tests/waku_store_legacy/store_utils.nim b/tests/waku_store_legacy/store_utils.nim deleted file mode 100644 index a70ca9376..000000000 --- a/tests/waku_store_legacy/store_utils.nim +++ /dev/null @@ -1,33 +0,0 @@ -{.used.} - -import std/options, chronos - -import - waku/[node/peer_manager, waku_core, waku_store_legacy, waku_store_legacy/client], - ../testlib/[common, wakucore] - -proc newTestWakuStore*( - switch: Switch, handler: HistoryQueryHandler -): Future[WakuStore] {.async.} = - let - peerManager = PeerManager.new(switch) - proto = WakuStore.new(peerManager, rng, handler) - - await proto.start() - switch.mount(proto) - - return proto - -proc newTestWakuStoreClient*(switch: Switch): WakuStoreClient = - let peerManager = PeerManager.new(switch) - WakuStoreClient.new(peerManager, rng) - -proc computeHistoryCursor*( - pubsubTopic: PubsubTopic, message: WakuMessage -): HistoryCursor = - HistoryCursor( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - storeTime: message.timestamp, - digest: computeDigest(message), - ) diff --git a/tests/waku_store_legacy/test_all.nim b/tests/waku_store_legacy/test_all.nim deleted file mode 100644 index b495310f2..000000000 --- a/tests/waku_store_legacy/test_all.nim +++ /dev/null @@ -1,8 +0,0 @@ -{.used.} - -import - ./test_client, - ./test_resume, - ./test_rpc_codec, - ./test_waku_store, - ./test_wakunode_store diff --git a/tests/waku_store_legacy/test_client.nim b/tests/waku_store_legacy/test_client.nim deleted file mode 100644 index 2a8616375..000000000 --- a/tests/waku_store_legacy/test_client.nim +++ /dev/null @@ -1,214 +0,0 @@ -{.used.} - -import std/options, testutils/unittests, chronos, libp2p/crypto/crypto - -import - waku/[ - node/peer_manager, - waku_core, - waku_store_legacy, - waku_store_legacy/client, - common/paging, - ], - ../testlib/[wakucore, testasync, futures], - ./store_utils - -suite "Store Client": - var message1 {.threadvar.}: WakuMessage - var message2 {.threadvar.}: WakuMessage - var message3 {.threadvar.}: WakuMessage - var messageSeq {.threadvar.}: seq[WakuMessage] - var handlerFuture {.threadvar.}: Future[HistoryQuery] - var handler {.threadvar.}: HistoryQueryHandler - var historyQuery {.threadvar.}: HistoryQuery - - var serverSwitch {.threadvar.}: Switch - var clientSwitch {.threadvar.}: Switch - - var server {.threadvar.}: WakuStore - var client {.threadvar.}: WakuStoreClient - - var serverPeerInfo {.threadvar.}: RemotePeerInfo - var clientPeerInfo {.threadvar.}: RemotePeerInfo - - asyncSetup: - message1 = fakeWakuMessage(contentTopic = DefaultContentTopic) - message2 = fakeWakuMessage(contentTopic = DefaultContentTopic) - message3 = fakeWakuMessage(contentTopic = DefaultContentTopic) - messageSeq = @[message1, message2, message3] - handlerFuture = newLegacyHistoryFuture() - handler = proc(req: HistoryQuery): Future[HistoryResult] {.async, gcsafe.} = - handlerFuture.complete(req) - return ok(HistoryResponse(messages: messageSeq)) - historyQuery = HistoryQuery( - pubsubTopic: some(DefaultPubsubTopic), - contentTopics: @[DefaultContentTopic], - direction: PagingDirection.FORWARD, - requestId: "customRequestId", - ) - - serverSwitch = newTestSwitch() - clientSwitch = newTestSwitch() - - server = await newTestWakuStore(serverSwitch, handler = handler) - client = newTestWakuStoreClient(clientSwitch) - - await allFutures(serverSwitch.start(), clientSwitch.start()) - - ## The following sleep is aimed to prevent macos failures in CI - #[ -2024-05-16T13:24:45.5106200Z INF 2024-05-16 13:24:45.509+00:00 Stopping AutonatService topics="libp2p autonatservice" tid=53712 file=service.nim:203 -2024-05-16T13:24:45.5107960Z WRN 2024-05-16 13:24:45.509+00:00 service is already stopped topics="libp2p switch" tid=53712 file=switch.nim:86 -2024-05-16T13:24:45.5109010Z . (1.68s) -2024-05-16T13:24:45.5109320Z Store Client (0.00s) -2024-05-16T13:24:45.5109870Z SIGSEGV: Illegal storage access. (Attempt to read from nil?) -2024-05-16T13:24:45.5111470Z stack trace: (most recent call last) - ]# - await sleepAsync(500.millis) - - serverPeerInfo = serverSwitch.peerInfo.toRemotePeerInfo() - clientPeerInfo = clientSwitch.peerInfo.toRemotePeerInfo() - - asyncTeardown: - await allFutures(serverSwitch.stop(), clientSwitch.stop()) - - suite "HistoryQuery Creation and Execution": - asyncTest "Valid Queries": - # When a valid query is sent to the server - let queryResponse = await client.query(historyQuery, peer = serverPeerInfo) - - # Then the query is processed successfully - assert await handlerFuture.withTimeout(FUTURE_TIMEOUT) - check: - handlerFuture.read() == historyQuery - queryResponse.get().messages == messageSeq - - asyncTest "Invalid Queries": - # TODO: IMPROVE: We can't test "actual" invalid queries because - # it directly depends on the handler implementation, to achieve - # proper coverage we'd need an example implementation. - - # Given some invalid queries - let - invalidQuery1 = HistoryQuery( - pubsubTopic: some(DefaultPubsubTopic), - contentTopics: @[], - direction: PagingDirection.FORWARD, - requestId: "reqId1", - ) - invalidQuery2 = HistoryQuery( - pubsubTopic: PubsubTopic.none(), - contentTopics: @[DefaultContentTopic], - direction: PagingDirection.FORWARD, - requestId: "reqId2", - ) - invalidQuery3 = HistoryQuery( - pubsubTopic: some(DefaultPubsubTopic), - contentTopics: @[DefaultContentTopic], - pageSize: 0, - requestId: "reqId3", - ) - invalidQuery4 = HistoryQuery( - pubsubTopic: some(DefaultPubsubTopic), - contentTopics: @[DefaultContentTopic], - pageSize: 0, - requestId: "reqId4", - ) - invalidQuery5 = HistoryQuery( - pubsubTopic: some(DefaultPubsubTopic), - contentTopics: @[DefaultContentTopic], - startTime: some(0.Timestamp), - endTime: some(0.Timestamp), - requestId: "reqId5", - ) - invalidQuery6 = HistoryQuery( - pubsubTopic: some(DefaultPubsubTopic), - contentTopics: @[DefaultContentTopic], - startTime: some(0.Timestamp), - endTime: some(-1.Timestamp), - requestId: "reqId6", - ) - - # When the query is sent to the server - let queryResponse1 = await client.query(invalidQuery1, peer = serverPeerInfo) - - # Then the query is not processed - assert await handlerFuture.withTimeout(FUTURE_TIMEOUT) - check: - handlerFuture.read() == invalidQuery1 - queryResponse1.get().messages == messageSeq - - # When the query is sent to the server - handlerFuture = newLegacyHistoryFuture() - let queryResponse2 = await client.query(invalidQuery2, peer = serverPeerInfo) - - # Then the query is not processed - assert await handlerFuture.withTimeout(FUTURE_TIMEOUT) - check: - handlerFuture.read() == invalidQuery2 - queryResponse2.get().messages == messageSeq - - # When the query is sent to the server - handlerFuture = newLegacyHistoryFuture() - let queryResponse3 = await client.query(invalidQuery3, peer = serverPeerInfo) - - # Then the query is not processed - assert await handlerFuture.withTimeout(FUTURE_TIMEOUT) - check: - handlerFuture.read() == invalidQuery3 - queryResponse3.get().messages == messageSeq - - # When the query is sent to the server - handlerFuture = newLegacyHistoryFuture() - let queryResponse4 = await client.query(invalidQuery4, peer = serverPeerInfo) - - # Then the query is not processed - assert await handlerFuture.withTimeout(FUTURE_TIMEOUT) - check: - handlerFuture.read() == invalidQuery4 - queryResponse4.get().messages == messageSeq - - # When the query is sent to the server - handlerFuture = newLegacyHistoryFuture() - let queryResponse5 = await client.query(invalidQuery5, peer = serverPeerInfo) - - # Then the query is not processed - assert await handlerFuture.withTimeout(FUTURE_TIMEOUT) - check: - handlerFuture.read() == invalidQuery5 - queryResponse5.get().messages == messageSeq - - # When the query is sent to the server - handlerFuture = newLegacyHistoryFuture() - let queryResponse6 = await client.query(invalidQuery6, peer = serverPeerInfo) - - # Then the query is not processed - assert await handlerFuture.withTimeout(FUTURE_TIMEOUT) - check: - handlerFuture.read() == invalidQuery6 - queryResponse6.get().messages == messageSeq - - suite "Verification of HistoryResponse Payload": - asyncTest "Positive Responses": - # When a valid query is sent to the server - let queryResponse = await client.query(historyQuery, peer = serverPeerInfo) - - # Then the query is processed successfully, and is of the expected type - check: - await handlerFuture.withTimeout(FUTURE_TIMEOUT) - type(queryResponse.get()) is HistoryResponse - - asyncTest "Negative Responses - PeerDialFailure": - # Given a stopped peer - let - otherServerSwitch = newTestSwitch() - otherServerPeerInfo = otherServerSwitch.peerInfo.toRemotePeerInfo() - - # When a query is sent to the stopped peer - let queryResponse = await client.query(historyQuery, peer = otherServerPeerInfo) - - # Then the query is not processed - check: - not await handlerFuture.withTimeout(FUTURE_TIMEOUT) - queryResponse.isErr() - queryResponse.error.kind == HistoryErrorKind.PEER_DIAL_FAILURE diff --git a/tests/waku_store_legacy/test_resume.nim b/tests/waku_store_legacy/test_resume.nim deleted file mode 100644 index 0f8132f08..000000000 --- a/tests/waku_store_legacy/test_resume.nim +++ /dev/null @@ -1,338 +0,0 @@ -{.used.} - -when defined(waku_exp_store_resume): - # TODO: Review store resume test cases (#1282) - # Ongoing changes to test code base had ruin this test meanwhile, need to investigate and fix - - import - std/[options, tables, sets], - testutils/unittests, - chronos, - chronicles, - libp2p/crypto/crypto - import - waku/[ - common/databases/db_sqlite, - waku_archive_legacy/driver, - waku_archive_legacy/driver/sqlite_driver/sqlite_driver, - node/peer_manager, - waku_core, - waku_core/message/digest, - waku_store_legacy, - ], - ../waku_store_legacy/store_utils, - ../waku_archive_legacy/archive_utils, - ./testlib/common, - ./testlib/switch - - procSuite "Waku Store - resume store": - ## Fixtures - let storeA = block: - let store = newTestMessageStore() - let msgList = @[ - fakeWakuMessage( - payload = @[byte 0], contentTopic = ContentTopic("2"), ts = ts(0) - ), - fakeWakuMessage( - payload = @[byte 1], contentTopic = ContentTopic("1"), ts = ts(1) - ), - fakeWakuMessage( - payload = @[byte 2], contentTopic = ContentTopic("2"), ts = ts(2) - ), - fakeWakuMessage( - payload = @[byte 3], contentTopic = ContentTopic("1"), ts = ts(3) - ), - fakeWakuMessage( - payload = @[byte 4], contentTopic = ContentTopic("2"), ts = ts(4) - ), - fakeWakuMessage( - payload = @[byte 5], contentTopic = ContentTopic("1"), ts = ts(5) - ), - fakeWakuMessage( - payload = @[byte 6], contentTopic = ContentTopic("2"), ts = ts(6) - ), - fakeWakuMessage( - payload = @[byte 7], contentTopic = ContentTopic("1"), ts = ts(7) - ), - fakeWakuMessage( - payload = @[byte 8], contentTopic = ContentTopic("2"), ts = ts(8) - ), - fakeWakuMessage( - payload = @[byte 9], contentTopic = ContentTopic("1"), ts = ts(9) - ), - ] - - for msg in msgList: - require store - .put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - .isOk() - - store - - let storeB = block: - let store = newTestMessageStore() - let msgList2 = @[ - fakeWakuMessage( - payload = @[byte 0], contentTopic = ContentTopic("2"), ts = ts(0) - ), - fakeWakuMessage( - payload = @[byte 11], contentTopic = ContentTopic("1"), ts = ts(1) - ), - fakeWakuMessage( - payload = @[byte 12], contentTopic = ContentTopic("2"), ts = ts(2) - ), - fakeWakuMessage( - payload = @[byte 3], contentTopic = ContentTopic("1"), ts = ts(3) - ), - fakeWakuMessage( - payload = @[byte 4], contentTopic = ContentTopic("2"), ts = ts(4) - ), - fakeWakuMessage( - payload = @[byte 5], contentTopic = ContentTopic("1"), ts = ts(5) - ), - fakeWakuMessage( - payload = @[byte 13], contentTopic = ContentTopic("2"), ts = ts(6) - ), - fakeWakuMessage( - payload = @[byte 14], contentTopic = ContentTopic("1"), ts = ts(7) - ), - ] - - for msg in msgList2: - require store - .put( - DefaultPubsubTopic, - msg, - computeDigest(msg), - computeMessageHash(DefaultPubsubTopic, msg), - msg.timestamp, - ) - .isOk() - - store - - asyncTest "multiple query to multiple peers with pagination": - ## Setup - let - serverSwitchA = newTestSwitch() - serverSwitchB = newTestSwitch() - clientSwitch = newTestSwitch() - - await allFutures( - serverSwitchA.start(), serverSwitchB.start(), clientSwitch.start() - ) - - let - serverA = await newTestWakuStoreNode(serverSwitchA, store = testStore) - serverB = await newTestWakuStoreNode(serverSwitchB, store = testStore) - client = newTestWakuStoreClient(clientSwitch) - - ## Given - let peers = @[ - serverSwitchA.peerInfo.toRemotePeerInfo(), - serverSwitchB.peerInfo.toRemotePeerInfo(), - ] - let req = HistoryQuery(contentTopics: @[DefaultContentTopic], pageSize: 5) - - ## When - let res = await client.queryLoop(req, peers) - - ## Then - check: - res.isOk() - - let response = res.tryGet() - check: - response.len == 10 - - ## Cleanup - await allFutures(clientSwitch.stop(), serverSwitchA.stop(), serverSwitchB.stop()) - - asyncTest "resume message history": - ## Setup - let - serverSwitch = newTestSwitch() - clientSwitch = newTestSwitch() - - await allFutures(serverSwitch.start(), clientSwitch.start()) - - let - server = await newTestWakuStore(serverSwitch, store = storeA) - client = await newTestWakuStore(clientSwitch) - - client.setPeer(serverSwitch.peerInfo.toRemotePeerInfo()) - - ## When - let res = await client.resume() - - ## Then - check res.isOk() - - let resumedMessagesCount = res.tryGet() - let storedMessagesCount = client.store.getMessagesCount().tryGet() - check: - resumedMessagesCount == 10 - storedMessagesCount == 10 - - ## Cleanup - await allFutures(clientSwitch.stop(), serverSwitch.stop()) - - asyncTest "resume history from a list of candidates - offline peer": - ## Setup - let - clientSwitch = newTestSwitch() - offlineSwitch = newTestSwitch() - - await clientSwitch.start() - - let client = await newTestWakuStore(clientSwitch) - - ## Given - let peers = @[offlineSwitch.peerInfo.toRemotePeerInfo()] - - ## When - let res = await client.resume(some(peers)) - - ## Then - check res.isErr() - - ## Cleanup - await clientSwitch.stop() - - asyncTest "resume history from a list of candidates - online and offline peers": - ## Setup - let - offlineSwitch = newTestSwitch() - serverASwitch = newTestSwitch() - serverBSwitch = newTestSwitch() - clientSwitch = newTestSwitch() - - await allFutures( - serverASwitch.start(), serverBSwitch.start(), clientSwitch.start() - ) - - let - serverA = await newTestWakuStore(serverASwitch, store = storeA) - serverB = await newTestWakuStore(serverBSwitch, store = storeB) - client = await newTestWakuStore(clientSwitch) - - ## Given - let peers = @[ - offlineSwitch.peerInfo.toRemotePeerInfo(), - serverASwitch.peerInfo.toRemotePeerInfo(), - serverBSwitch.peerInfo.toRemotePeerInfo(), - ] - - ## When - let res = await client.resume(some(peers)) - - ## Then - # `client` is expected to retrieve 14 messages: - # - The store mounted on `serverB` holds 10 messages (see `storeA` fixture) - # - The store mounted on `serverB` holds 7 messages (see `storeB` fixture) - # Both stores share 3 messages, resulting in 14 unique messages in total - check res.isOk() - - let restoredMessagesCount = res.tryGet() - let storedMessagesCount = client.store.getMessagesCount().tryGet() - check: - restoredMessagesCount == 14 - storedMessagesCount == 14 - - ## Cleanup - await allFutures(serverASwitch.stop(), serverBSwitch.stop(), clientSwitch.stop()) - - suite "WakuNode - waku store": - asyncTest "Resume proc fetches the history": - ## Setup - let - serverKey = generateSecp256k1Key() - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - clientKey = generateSecp256k1Key() - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - await allFutures(client.start(), server.start()) - - let driver = newSqliteArchiveDriver() - server.mountArchive(some(driver), none(MessageValidator), none(RetentionPolicy)) - await server.mountStore() - - let clientStore = StoreQueueRef.new() - await client.mountStore(store = clientStore) - client.mountStoreClient(store = clientStore) - - ## Given - let message = fakeWakuMessage() - require server.wakuStore.store.put(DefaultPubsubTopic, message).isOk() - - let serverPeer = server.peerInfo.toRemotePeerInfo() - - ## When - await client.resume(some(@[serverPeer])) - - # Then - check: - client.wakuStore.store.getMessagesCount().tryGet() == 1 - - ## Cleanup - await allFutures(client.stop(), server.stop()) - - asyncTest "Resume proc discards duplicate messages": - ## Setup - let - serverKey = generateSecp256k1Key() - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - clientKey = generateSecp256k1Key() - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - await allFutures(server.start(), client.start()) - await server.mountStore(store = StoreQueueRef.new()) - - let clientStore = StoreQueueRef.new() - await client.mountStore(store = clientStore) - client.mountStoreClient(store = clientStore) - - ## Given - let timeOrigin = now() - let - msg1 = fakeWakuMessage( - payload = "hello world1", ts = (timeOrigin + getNanoSecondTime(1)) - ) - msg2 = fakeWakuMessage( - payload = "hello world2", ts = (timeOrigin + getNanoSecondTime(2)) - ) - msg3 = fakeWakuMessage( - payload = "hello world3", ts = (timeOrigin + getNanoSecondTime(3)) - ) - - require server.wakuStore.store.put(DefaultPubsubTopic, msg1).isOk() - require server.wakuStore.store.put(DefaultPubsubTopic, msg2).isOk() - - # Insert the same message in both node's store - let - receivedTime3 = now() + getNanosecondTime(10) - digest3 = computeDigest(msg3) - require server.wakuStore.store - .put(DefaultPubsubTopic, msg3, digest3, receivedTime3) - .isOk() - require client.wakuStore.store - .put(DefaultPubsubTopic, msg3, digest3, receivedTime3) - .isOk() - - let serverPeer = server.peerInfo.toRemotePeerInfo() - - ## When - await client.resume(some(@[serverPeer])) - - ## Then - check: - # If the duplicates are discarded properly, then the total number of messages after resume should be 3 - client.wakuStore.store.getMessagesCount().tryGet() == 3 - - await allFutures(client.stop(), server.stop()) diff --git a/tests/waku_store_legacy/test_rpc_codec.nim b/tests/waku_store_legacy/test_rpc_codec.nim deleted file mode 100644 index 2801cc9a8..000000000 --- a/tests/waku_store_legacy/test_rpc_codec.nim +++ /dev/null @@ -1,184 +0,0 @@ -{.used.} - -import std/options, testutils/unittests, chronos -import - waku/[ - common/protobuf, - common/paging, - waku_core, - waku_store_legacy/rpc, - waku_store_legacy/rpc_codec, - ], - ../testlib/wakucore - -procSuite "Waku Store - RPC codec": - test "PagingIndexRPC protobuf codec": - ## Given - let index = PagingIndexRPC.compute( - fakeWakuMessage(), receivedTime = ts(), pubsubTopic = DefaultPubsubTopic - ) - - ## When - let encodedIndex = index.encode() - let decodedIndexRes = PagingIndexRPC.decode(encodedIndex.buffer) - - ## Then - check: - decodedIndexRes.isOk() - - let decodedIndex = decodedIndexRes.tryGet() - check: - # The fields of decodedIndex must be the same as the original index - decodedIndex == index - - test "PagingIndexRPC protobuf codec - empty index": - ## Given - let emptyIndex = PagingIndexRPC() - - let encodedIndex = emptyIndex.encode() - let decodedIndexRes = PagingIndexRPC.decode(encodedIndex.buffer) - - ## Then - check: - decodedIndexRes.isOk() - - let decodedIndex = decodedIndexRes.tryGet() - check: - # Check the correctness of init and encode for an empty PagingIndexRPC - decodedIndex == emptyIndex - - test "PagingInfoRPC protobuf codec": - ## Given - let - index = PagingIndexRPC.compute( - fakeWakuMessage(), receivedTime = ts(), pubsubTopic = DefaultPubsubTopic - ) - pagingInfo = PagingInfoRPC( - pageSize: some(1'u64), - cursor: some(index), - direction: some(PagingDirection.FORWARD), - ) - - ## When - let pb = pagingInfo.encode() - let decodedPagingInfo = PagingInfoRPC.decode(pb.buffer) - - ## Then - check: - decodedPagingInfo.isOk() - - check: - # The fields of decodedPagingInfo must be the same as the original pagingInfo - decodedPagingInfo.value == pagingInfo - decodedPagingInfo.value.direction == pagingInfo.direction - - test "PagingInfoRPC protobuf codec - empty paging info": - ## Given - let emptyPagingInfo = PagingInfoRPC() - - ## When - let pb = emptyPagingInfo.encode() - let decodedEmptyPagingInfo = PagingInfoRPC.decode(pb.buffer) - - ## Then - check: - decodedEmptyPagingInfo.isOk() - - check: - # check the correctness of init and encode for an empty PagingInfoRPC - decodedEmptyPagingInfo.value == emptyPagingInfo - - test "HistoryQueryRPC protobuf codec": - ## Given - let - index = PagingIndexRPC.compute( - fakeWakuMessage(), receivedTime = ts(), pubsubTopic = DefaultPubsubTopic - ) - pagingInfo = PagingInfoRPC( - pageSize: some(1'u64), - cursor: some(index), - direction: some(PagingDirection.BACKWARD), - ) - query = HistoryQueryRPC( - contentFilters: @[ - HistoryContentFilterRPC(contentTopic: DefaultContentTopic), - HistoryContentFilterRPC(contentTopic: DefaultContentTopic), - ], - pagingInfo: some(pagingInfo), - startTime: some(Timestamp(10)), - endTime: some(Timestamp(11)), - ) - - ## When - let pb = query.encode() - let decodedQuery = HistoryQueryRPC.decode(pb.buffer) - - ## Then - check: - decodedQuery.isOk() - - check: - # the fields of decoded query decodedQuery must be the same as the original query query - decodedQuery.value == query - - test "HistoryQueryRPC protobuf codec - empty history query": - ## Given - let emptyQuery = HistoryQueryRPC() - - ## When - let pb = emptyQuery.encode() - let decodedEmptyQuery = HistoryQueryRPC.decode(pb.buffer) - - ## Then - check: - decodedEmptyQuery.isOk() - - check: - # check the correctness of init and encode for an empty HistoryQueryRPC - decodedEmptyQuery.value == emptyQuery - - test "HistoryResponseRPC protobuf codec": - ## Given - let - message = fakeWakuMessage() - index = PagingIndexRPC.compute( - message, receivedTime = ts(), pubsubTopic = DefaultPubsubTopic - ) - pagingInfo = PagingInfoRPC( - pageSize: some(1'u64), - cursor: some(index), - direction: some(PagingDirection.BACKWARD), - ) - res = HistoryResponseRPC( - messages: @[message], - pagingInfo: some(pagingInfo), - error: HistoryResponseErrorRPC.INVALID_CURSOR, - ) - - ## When - let pb = res.encode() - let decodedRes = HistoryResponseRPC.decode(pb.buffer) - - ## Then - check: - decodedRes.isOk() - - check: - # the fields of decoded response decodedRes must be the same as the original response res - decodedRes.value == res - - test "HistoryResponseRPC protobuf codec - empty history response": - ## Given - let emptyRes = HistoryResponseRPC() - - ## When - let pb = emptyRes.encode() - let decodedEmptyRes = HistoryResponseRPC.decode(pb.buffer) - - ## Then - check: - decodedEmptyRes.isOk() - - check: - # check the correctness of init and encode for an empty HistoryResponseRPC - decodedEmptyRes.value == emptyRes diff --git a/tests/waku_store_legacy/test_waku_store.nim b/tests/waku_store_legacy/test_waku_store.nim deleted file mode 100644 index b8dc835c8..000000000 --- a/tests/waku_store_legacy/test_waku_store.nim +++ /dev/null @@ -1,113 +0,0 @@ -{.used.} - -import testutils/unittests, chronos, libp2p/crypto/crypto - -import - waku/[ - common/paging, - node/peer_manager, - waku_core, - waku_store_legacy, - waku_store_legacy/client, - ], - ../testlib/wakucore, - ./store_utils - -suite "Waku Store - query handler legacy": - asyncTest "history query handler should be called": - ## Setup - let - serverSwitch = newTestSwitch() - clientSwitch = newTestSwitch() - - await allFutures(serverSwitch.start(), clientSwitch.start()) - - ## Given - let serverPeerInfo = serverSwitch.peerInfo.toRemotePeerInfo() - - let msg = fakeWakuMessage(contentTopic = DefaultContentTopic) - - var queryHandlerFut = newFuture[(HistoryQuery)]() - - let queryHandler = proc( - req: HistoryQuery - ): Future[HistoryResult] {.async, gcsafe.} = - queryHandlerFut.complete(req) - return ok(HistoryResponse(messages: @[msg])) - - let - server = await newTestWakuStore(serverSwitch, handler = queryhandler) - client = newTestWakuStoreClient(clientSwitch) - - let req = HistoryQuery( - contentTopics: @[DefaultContentTopic], - direction: PagingDirection.FORWARD, - requestId: "reqId", - ) - - ## When - let queryRes = await client.query(req, peer = serverPeerInfo) - - ## Then - check: - not queryHandlerFut.failed() - queryRes.isOk() - - let request = queryHandlerFut.read() - check: - request == req - - let response = queryRes.tryGet() - check: - response.messages.len == 1 - response.messages == @[msg] - - ## Cleanup - await allFutures(serverSwitch.stop(), clientSwitch.stop()) - - asyncTest "history query handler should be called and return an error": - ## Setup - let - serverSwitch = newTestSwitch() - clientSwitch = newTestSwitch() - - await allFutures(serverSwitch.start(), clientSwitch.start()) - - ## Given - let serverPeerInfo = serverSwitch.peerInfo.toRemotePeerInfo() - - var queryHandlerFut = newFuture[(HistoryQuery)]() - let queryHandler = proc( - req: HistoryQuery - ): Future[HistoryResult] {.async, gcsafe.} = - queryHandlerFut.complete(req) - return err(HistoryError(kind: HistoryErrorKind.BAD_REQUEST)) - - let - server = await newTestWakuStore(serverSwitch, handler = queryhandler) - client = newTestWakuStoreClient(clientSwitch) - - let req = HistoryQuery( - contentTopics: @[DefaultContentTopic], - direction: PagingDirection.FORWARD, - requestId: "reqId", - ) - - ## When - let queryRes = await client.query(req, peer = serverPeerInfo) - - ## Then - check: - not queryHandlerFut.failed() - queryRes.isErr() - - let request = queryHandlerFut.read() - check: - request == req - - let error = queryRes.tryError() - check: - error.kind == HistoryErrorKind.BAD_REQUEST - - ## Cleanup - await allFutures(serverSwitch.stop(), clientSwitch.stop()) diff --git a/tests/waku_store_legacy/test_wakunode_store.nim b/tests/waku_store_legacy/test_wakunode_store.nim deleted file mode 100644 index 58e3ca9e0..000000000 --- a/tests/waku_store_legacy/test_wakunode_store.nim +++ /dev/null @@ -1,315 +0,0 @@ -{.used.} - -import - std/net, - testutils/unittests, - chronos, - libp2p/crypto/crypto, - libp2p/peerid, - libp2p/multiaddress, - libp2p/switch, - libp2p/protocols/pubsub/pubsub, - libp2p/protocols/pubsub/gossipsub -import - waku/[ - common/paging, - waku_core, - waku_core/message/digest, - node/peer_manager, - waku_archive_legacy, - waku_filter_v2, - waku_filter_v2/client, - waku_store_legacy, - waku_node, - ], - ../waku_store_legacy/store_utils, - ../waku_archive_legacy/archive_utils, - ../testlib/wakucore, - ../testlib/wakunode - -procSuite "WakuNode - Store Legacy": - ## Fixtures - let timeOrigin = now() - let msgListA = @[ - fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)), - fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)), - fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)), - fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)), - fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)), - fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)), - fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)), - fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)), - fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)), - fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)), - ] - - let archiveA = block: - let driver = newSqliteArchiveDriver() - - for msg in msgListA: - let msg_digest = waku_archive_legacy.computeDigest(msg) - let msg_hash = computeMessageHash(DefaultPubsubTopic, msg) - require ( - waitFor driver.put(DefaultPubsubTopic, msg, msg_digest, msg_hash, msg.timestamp) - ).isOk() - - driver - - test "Store protocol returns expected messages": - ## Setup - let - serverKey = generateSecp256k1Key() - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - clientKey = generateSecp256k1Key() - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - waitFor allFutures(client.start(), server.start()) - - let mountArchiveRes = server.mountLegacyArchive(archiveA) - assert mountArchiveRes.isOk(), mountArchiveRes.error - - waitFor server.mountLegacyStore() - - client.mountLegacyStoreClient() - - ## Given - let req = HistoryQuery(contentTopics: @[DefaultContentTopic]) - let serverPeer = server.peerInfo.toRemotePeerInfo() - - ## When - let queryRes = waitFor client.query(req, peer = serverPeer) - - ## Then - check queryRes.isOk() - - let response = queryRes.get() - check: - response.messages == msgListA - - # Cleanup - waitFor allFutures(client.stop(), server.stop()) - - test "Store node history response - forward pagination": - ## Setup - let - serverKey = generateSecp256k1Key() - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - clientKey = generateSecp256k1Key() - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - waitFor allFutures(client.start(), server.start()) - - let mountArchiveRes = server.mountLegacyArchive(archiveA) - assert mountArchiveRes.isOk(), mountArchiveRes.error - - waitFor server.mountLegacyStore() - - client.mountLegacyStoreClient() - - ## Given - let req = HistoryQuery( - contentTopics: @[DefaultContentTopic], - pageSize: 7, - direction: PagingDirection.FORWARD, - ) - let serverPeer = server.peerInfo.toRemotePeerInfo() - - ## When - var nextReq = req # copy - - var pages = newSeq[seq[WakuMessage]](2) - var cursors = newSeq[Option[HistoryCursor]](2) - - for i in 0 ..< 2: - let res = waitFor client.query(nextReq, peer = serverPeer) - require res.isOk() - - # Keep query response content - let response = res.get() - pages[i] = response.messages - cursors[i] = response.cursor - - # Set/update the request cursor - nextReq.cursor = cursors[i] - - ## Then - check: - cursors[0] == some(computeHistoryCursor(DefaultPubsubTopic, msgListA[6])) - cursors[1] == none(HistoryCursor) - - check: - pages[0] == msgListA[0 .. 6] - pages[1] == msgListA[7 .. 9] - - # Cleanup - waitFor allFutures(client.stop(), server.stop()) - - test "Store node history response - backward pagination": - ## Setup - let - serverKey = generateSecp256k1Key() - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - clientKey = generateSecp256k1Key() - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - waitFor allFutures(client.start(), server.start()) - - let mountArchiveRes = server.mountLegacyArchive(archiveA) - assert mountArchiveRes.isOk(), mountArchiveRes.error - - waitFor server.mountLegacyStore() - - client.mountLegacyStoreClient() - - ## Given - let req = HistoryQuery( - contentTopics: @[DefaultContentTopic], - pageSize: 7, - direction: PagingDirection.BACKWARD, - ) - let serverPeer = server.peerInfo.toRemotePeerInfo() - - ## When - var nextReq = req # copy - - var pages = newSeq[seq[WakuMessage]](2) - var cursors = newSeq[Option[HistoryCursor]](2) - - for i in 0 ..< 2: - let res = waitFor client.query(nextReq, peer = serverPeer) - require res.isOk() - - # Keep query response content - let response = res.get() - pages[i] = response.messages - cursors[i] = response.cursor - - # Set/update the request cursor - nextReq.cursor = cursors[i] - - ## Then - check: - cursors[0] == some(computeHistoryCursor(DefaultPubsubTopic, msgListA[3])) - cursors[1] == none(HistoryCursor) - - check: - pages[0] == msgListA[3 .. 9] - pages[1] == msgListA[0 .. 2] - - # Cleanup - waitFor allFutures(client.stop(), server.stop()) - - test "Store protocol returns expected message when relay is disabled and filter enabled": - ## See nwaku issue #937: 'Store: ability to decouple store from relay' - ## Setup - let - filterSourceKey = generateSecp256k1Key() - filterSource = - newTestWakuNode(filterSourceKey, parseIpAddress("0.0.0.0"), Port(0)) - serverKey = generateSecp256k1Key() - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - clientKey = generateSecp256k1Key() - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - waitFor allFutures(client.start(), server.start(), filterSource.start()) - - waitFor filterSource.mountFilter() - let driver = newSqliteArchiveDriver() - - let mountArchiveRes = server.mountLegacyArchive(driver) - assert mountArchiveRes.isOk(), mountArchiveRes.error - - waitFor server.mountLegacyStore() - waitFor server.mountFilterClient() - client.mountLegacyStoreClient() - - ## Given - let message = fakeWakuMessage() - let - serverPeer = server.peerInfo.toRemotePeerInfo() - filterSourcePeer = filterSource.peerInfo.toRemotePeerInfo() - - ## Then - let filterFut = newFuture[(PubsubTopic, WakuMessage)]() - proc filterHandler( - pubsubTopic: PubsubTopic, msg: WakuMessage - ) {.async, gcsafe, closure.} = - await server.wakuLegacyArchive.handleMessage(pubsubTopic, msg) - filterFut.complete((pubsubTopic, msg)) - - server.wakuFilterClient.registerPushHandler(filterHandler) - let resp = waitFor server.filterSubscribe( - some(DefaultPubsubTopic), DefaultContentTopic, peer = filterSourcePeer - ) - - waitFor sleepAsync(100.millis) - - waitFor filterSource.wakuFilter.handleMessage(DefaultPubsubTopic, message) - - # Wait for the server filter to receive the push message - require waitFor filterFut.withTimeout(5.seconds) - - let res = waitFor client.query( - HistoryQuery(contentTopics: @[DefaultContentTopic]), peer = serverPeer - ) - - ## Then - check res.isOk() - - let response = res.get() - check: - response.messages.len == 1 - response.messages[0] == message - - let (handledPubsubTopic, handledMsg) = filterFut.read() - check: - handledPubsubTopic == DefaultPubsubTopic - handledMsg == message - - ## Cleanup - waitFor allFutures(client.stop(), server.stop(), filterSource.stop()) - - test "history query should return INVALID_CURSOR if the cursor has empty data in the request": - ## Setup - let - serverKey = generateSecp256k1Key() - server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0)) - clientKey = generateSecp256k1Key() - client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0)) - - waitFor allFutures(client.start(), server.start()) - - let mountArchiveRes = server.mountLegacyArchive(archiveA) - assert mountArchiveRes.isOk(), mountArchiveRes.error - - waitFor server.mountLegacyStore() - - client.mountLegacyStoreClient() - - ## Forcing a bad cursor with empty digest data - var data: array[32, byte] = [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - ] - let cursor = HistoryCursor( - pubsubTopic: "pubsubTopic", - senderTime: now(), - storeTime: now(), - digest: waku_archive_legacy.MessageDigest(data: data), - ) - - ## Given - let req = HistoryQuery(contentTopics: @[DefaultContentTopic], cursor: some(cursor)) - let serverPeer = server.peerInfo.toRemotePeerInfo() - - ## When - let queryRes = waitFor client.query(req, peer = serverPeer) - - ## Then - check not queryRes.isOk() - - check queryRes.error == - "legacy store client query error: BAD_REQUEST: invalid cursor" - - # Cleanup - waitFor allFutures(client.stop(), server.stop()) diff --git a/tests/wakunode_rest/test_rest_health.nim b/tests/wakunode_rest/test_rest_health.nim index 37abaf4f5..0bdb93123 100644 --- a/tests/wakunode_rest/test_rest_health.nim +++ b/tests/wakunode_rest/test_rest_health.nim @@ -86,7 +86,7 @@ suite "Waku v2 REST API - health": response.status == 200 $response.contentType == $MIMETYPE_JSON report.nodeHealth == HealthStatus.READY - report.protocolsHealth.len() == 15 + report.protocolsHealth.len() == 13 report.getHealth(RelayProtocol).health == HealthStatus.NOT_READY report.getHealth(RelayProtocol).desc == some("No connected peers") @@ -97,7 +97,6 @@ suite "Waku v2 REST API - health": report.getHealth(LegacyLightpushProtocol).health == HealthStatus.NOT_MOUNTED report.getHealth(FilterProtocol).health == HealthStatus.NOT_MOUNTED report.getHealth(StoreProtocol).health == HealthStatus.NOT_MOUNTED - report.getHealth(LegacyStoreProtocol).health == HealthStatus.NOT_MOUNTED report.getHealth(PeerExchangeProtocol).health == HealthStatus.NOT_MOUNTED report.getHealth(RendezvousProtocol).health == HealthStatus.NOT_MOUNTED report.getHealth(MixProtocol).health == HealthStatus.NOT_MOUNTED @@ -108,7 +107,6 @@ suite "Waku v2 REST API - health": report.getHealth(LegacyLightpushClientProtocol).health == HealthStatus.NOT_MOUNTED report.getHealth(StoreClientProtocol).health == HealthStatus.NOT_MOUNTED - report.getHealth(LegacyStoreClientProtocol).health == HealthStatus.NOT_MOUNTED report.getHealth(FilterClientProtocol).health == HealthStatus.NOT_READY report.getHealth(FilterClientProtocol).desc == diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim index 541215f76..a99ba43ee 100644 --- a/tools/confutils/cli_args.nim +++ b/tools/confutils/cli_args.nim @@ -348,12 +348,6 @@ hence would have reachability issues.""", desc: "Enable/disable waku store protocol", defaultValue: false, name: "store" .}: bool - legacyStore* {. - desc: "Enable/disable support of Waku Store v2 as a service", - defaultValue: false, - name: "legacy-store" - .}: bool - storenode* {. desc: "Peer multiaddress to query for storage", defaultValue: "", @@ -691,7 +685,7 @@ with the drawback of consuming some more bandwidth.""", desc: "Rate limit settings for different protocols." & "Format: protocol:volume/period" & - " Where 'protocol' can be one of: if not defined it means a global setting" & + " Where 'protocol' can be one of: if not defined it means a global setting" & " 'volume' and period must be an integer value. " & " 'unit' must be one of - hours, minutes, seconds, milliseconds respectively. " & "Argument may be repeated.", @@ -1045,7 +1039,6 @@ proc toWakuConf*(n: WakuNodeConf): ConfResult[WakuConf] = b.withContentTopics(n.contentTopics) b.storeServiceConf.withEnabled(n.store) - b.storeServiceConf.withSupportV2(n.legacyStore) b.storeServiceConf.withRetentionPolicies(n.storeMessageRetentionPolicy) b.storeServiceConf.withDbUrl(n.storeMessageDbUrl) b.storeServiceConf.withDbVacuum(n.storeMessageDbVacuum) diff --git a/waku/common/rate_limit/setting.nim b/waku/common/rate_limit/setting.nim index 70f0ee721..66ff79d17 100644 --- a/waku/common/rate_limit/setting.nim +++ b/waku/common/rate_limit/setting.nim @@ -7,7 +7,6 @@ type RateLimitSetting* = tuple[volume: int, period: Duration] type RateLimitedProtocol* = enum GLOBAL - STOREV2 STOREV3 LIGHTPUSH PEEREXCHG @@ -47,8 +46,6 @@ proc translate(sProtocol: string): RateLimitedProtocol {.raises: [ValueError].} case sProtocol of "global": return GLOBAL - of "storev2": - return STOREV2 of "storev3": return STOREV3 of "lightpush": @@ -65,7 +62,6 @@ proc fillSettingTable( ) {.raises: [ValueError].} = if sProtocol == "store": # generic store will only applies to version which is not listed directly - discard t.hasKeyOrPut(STOREV2, setting) discard t.hasKeyOrPut(STOREV3, setting) else: let protocol = translate(sProtocol) @@ -87,7 +83,7 @@ proc parse*( ## group4: Unit of period - only h:hour, m:minute, s:second, ms:millisecond allowed ## whitespaces are allowed lazily const parseRegex = - """^\s*((store|storev2|storev3|lightpush|px|filter)\s*:)?\s*(\d+)\s*\/\s*(\d+)\s*(s|h|m|ms)\s*$""" + """^\s*((store|storev3|lightpush|px|filter)\s*:)?\s*(\d+)\s*\/\s*(\d+)\s*(s|h|m|ms)\s*$""" const regexParseSize = re2(parseRegex) for settingStr in settings: let aSetting = settingStr.toLower() diff --git a/waku/common/waku_protocol.nim b/waku/common/waku_protocol.nim index 5063f4c98..76a8aded0 100644 --- a/waku/common/waku_protocol.nim +++ b/waku/common/waku_protocol.nim @@ -4,7 +4,6 @@ type WakuProtocol* {.pure.} = enum RelayProtocol = "Relay" RlnRelayProtocol = "Rln Relay" StoreProtocol = "Store" - LegacyStoreProtocol = "Legacy Store" FilterProtocol = "Filter" LightpushProtocol = "Lightpush" LegacyLightpushProtocol = "Legacy Lightpush" @@ -12,13 +11,12 @@ type WakuProtocol* {.pure.} = enum RendezvousProtocol = "Rendezvous" MixProtocol = "Mix" StoreClientProtocol = "Store Client" - LegacyStoreClientProtocol = "Legacy Store Client" FilterClientProtocol = "Filter Client" LightpushClientProtocol = "Lightpush Client" LegacyLightpushClientProtocol = "Legacy Lightpush Client" const RelayProtocols* = {RelayProtocol} - StoreClientProtocols* = {StoreClientProtocol, LegacyStoreClientProtocol} + StoreClientProtocols* = {StoreClientProtocol} LightpushClientProtocols* = {LightpushClientProtocol, LegacyLightpushClientProtocol} FilterClientProtocols* = {FilterClientProtocol} diff --git a/waku/factory/conf_builder/store_service_conf_builder.nim b/waku/factory/conf_builder/store_service_conf_builder.nim index 30c743e01..f1b0b1402 100644 --- a/waku/factory/conf_builder/store_service_conf_builder.nim +++ b/waku/factory/conf_builder/store_service_conf_builder.nim @@ -14,7 +14,6 @@ type StoreServiceConfBuilder* = object dbMigration*: Option[bool] dbURl*: Option[string] dbVacuum*: Option[bool] - supportV2*: Option[bool] maxNumDbConnections*: Option[int] retentionPolicies*: seq[string] resume*: Option[bool] @@ -35,9 +34,6 @@ proc withDbUrl*(b: var StoreServiceConfBuilder, dbUrl: string) = proc withDbVacuum*(b: var StoreServiceConfBuilder, dbVacuum: bool) = b.dbVacuum = some(dbVacuum) -proc withSupportV2*(b: var StoreServiceConfBuilder, supportV2: bool) = - b.supportV2 = some(supportV2) - proc withMaxNumDbConnections*( b: var StoreServiceConfBuilder, maxNumDbConnections: int ) = @@ -104,7 +100,6 @@ proc build*(b: StoreServiceConfBuilder): Result[Option[StoreServiceConf], string dbMigration: b.dbMigration.get(true), dbURl: b.dbUrl.get(), dbVacuum: b.dbVacuum.get(false), - supportV2: b.supportV2.get(false), maxNumDbConnections: b.maxNumDbConnections.get(50), retentionPolicies: retentionPolicies, resume: b.resume.get(false), diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index ebf91f415..52b719b8f 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -25,12 +25,8 @@ import ../waku_archive/retention_policy/builder as policy_builder, ../waku_archive/driver as driver, ../waku_archive/driver/builder as driver_builder, - ../waku_archive_legacy/driver as legacy_driver, - ../waku_archive_legacy/driver/builder as legacy_driver_builder, ../waku_store, ../waku_store/common as store_common, - ../waku_store_legacy, - ../waku_store_legacy/common as legacy_common, ../waku_filter_v2, ../waku_peer_exchange, ../discovery/waku_kademlia, @@ -38,8 +34,7 @@ import ../node/peer_manager/peer_store/waku_peer_storage, ../node/peer_manager/peer_store/migrations as peer_store_sqlite_migrations, ../waku_lightpush_legacy/common, - ../common/rate_limit/setting, - ../common/databases/dburl + ../common/rate_limit/setting ## Peer persistence @@ -198,42 +193,10 @@ proc setupProtocols( if conf.storeServiceConf.isSome(): let storeServiceConf = conf.storeServiceConf.get() - if storeServiceConf.supportV2: - let archiveDriver = ( - await legacy_driver.ArchiveDriver.new( - storeServiceConf.dbUrl, storeServiceConf.dbVacuum, - storeServiceConf.dbMigration, storeServiceConf.maxNumDbConnections, - onFatalErrorAction, - ) - ).valueOr: - return err("failed to setup legacy archive driver: " & error) - - node.mountLegacyArchive(archiveDriver).isOkOr: - return err("failed to mount waku legacy archive protocol: " & error) - - ## For now we always mount the future archive driver but if the legacy one is mounted, - ## then the legacy will be in charge of performing the archiving. - ## Regarding storage, the only diff between the current/future archive driver and the legacy - ## one, is that the legacy stores an extra field: the id (message digest.) - - ## TODO: remove this "migrate" variable once legacy store is removed - ## It is now necessary because sqlite's legacy store has an extra field: storedAt - ## This breaks compatibility between store's and legacy store's schemas in sqlite - ## So for now, we need to make sure that when legacy store is enabled and we use sqlite - ## that we migrate our db according to legacy store's schema to have the extra field - - let engine = dburl.getDbEngine(storeServiceConf.dbUrl).valueOr: - return err("error getting db engine in setupProtocols: " & error) - - let migrate = - if engine == "sqlite" and storeServiceConf.supportV2: - false - else: - storeServiceConf.dbMigration let archiveDriver = ( await driver.ArchiveDriver.new( - storeServiceConf.dbUrl, storeServiceConf.dbVacuum, migrate, + storeServiceConf.dbUrl, storeServiceConf.dbVacuum, storeServiceConf.dbMigration, storeServiceConf.maxNumDbConnections, onFatalErrorAction, ) ).valueOr: @@ -245,14 +208,6 @@ proc setupProtocols( node.mountArchive(archiveDriver, retPolicies).isOkOr: return err("failed to mount waku archive protocol: " & error) - if storeServiceConf.supportV2: - # Store legacy setup - try: - await mountLegacyStore(node, node.rateLimitSettings.getSetting(STOREV2)) - except CatchableError: - return - err("failed to mount waku legacy store protocol: " & getCurrentExceptionMsg()) - # Store setup try: await mountStore(node, node.rateLimitSettings.getSetting(STOREV3)) @@ -284,12 +239,6 @@ proc setupProtocols( return err("failed to set node waku store peer: " & error) node.peerManager.addServicePeer(storeNode, WakuStoreCodec) - mountLegacyStoreClient(node) - if conf.remoteStoreNode.isSome(): - let storeNode = parsePeerInfo(conf.remoteStoreNode.get()).valueOr: - return err("failed to set node waku legacy store peer: " & error) - node.peerManager.addServicePeer(storeNode, WakuLegacyStoreCodec) - if conf.storeServiceConf.isSome and conf.storeServiceConf.get().resume: node.setupStoreResume() diff --git a/waku/factory/waku_conf.nim b/waku/factory/waku_conf.nim index 6ed34e131..4934faccc 100644 --- a/waku/factory/waku_conf.nim +++ b/waku/factory/waku_conf.nim @@ -60,7 +60,6 @@ type StoreServiceConf* {.requiresInit.} = object dbMigration*: bool dbURl*: string dbVacuum*: bool - supportV2*: bool maxNumDbConnections*: int retentionPolicies*: seq[string] resume*: bool diff --git a/waku/node/health_monitor/node_health_monitor.nim b/waku/node/health_monitor/node_health_monitor.nim index ddba47ccb..79bf9f92a 100644 --- a/waku/node/health_monitor/node_health_monitor.nim +++ b/waku/node/health_monitor/node_health_monitor.nim @@ -163,17 +163,6 @@ proc getStoreHealth(hm: NodeHealthMonitor): ProtocolHealth = hm.strength[WakuProtocol.StoreProtocol] = peerCount return p.ready() -proc getLegacyStoreHealth(hm: NodeHealthMonitor): ProtocolHealth = - var p = ProtocolHealth.init(WakuProtocol.LegacyStoreProtocol) - - if isNil(hm.node.wakuLegacyStore): - hm.strength[WakuProtocol.LegacyStoreProtocol] = 0 - return p.notMounted() - - let peerCount = hm.countCapablePeers(WakuLegacyStoreCodec) - hm.strength[WakuProtocol.LegacyStoreProtocol] = peerCount - return p.ready() - proc getLightpushClientHealth(hm: NodeHealthMonitor): ProtocolHealth = var p = ProtocolHealth.init(WakuProtocol.LightpushClientProtocol) @@ -233,23 +222,6 @@ proc getStoreClientHealth(hm: NodeHealthMonitor): ProtocolHealth = "No Store service peer available yet, neither Store service set up for the node" ) -proc getLegacyStoreClientHealth(hm: NodeHealthMonitor): ProtocolHealth = - var p = ProtocolHealth.init(WakuProtocol.LegacyStoreClientProtocol) - - if isNil(hm.node.wakuLegacyStoreClient): - hm.strength[WakuProtocol.LegacyStoreClientProtocol] = 0 - return p.notMounted() - - let peerCount = countCapablePeers(hm, WakuLegacyStoreCodec) - hm.strength[WakuProtocol.LegacyStoreClientProtocol] = peerCount - - if peerCount > 0 or not isNil(hm.node.wakuLegacyStore): - return p.ready() - - return p.notReady( - "No Legacy Store service peers are available yet, neither Store service set up for the node" - ) - proc getPeerExchangeHealth(hm: NodeHealthMonitor): ProtocolHealth = var p = ProtocolHealth.init(WakuProtocol.PeerExchangeProtocol) @@ -294,8 +266,6 @@ proc getSyncProtocolHealthInfo*( return hm.getRelayHealth() of WakuProtocol.StoreProtocol: return hm.getStoreHealth() - of WakuProtocol.LegacyStoreProtocol: - return hm.getLegacyStoreHealth() of WakuProtocol.FilterProtocol: return hm.getFilterHealth(hm.getRelayHealth().health) of WakuProtocol.LightpushProtocol: @@ -310,8 +280,6 @@ proc getSyncProtocolHealthInfo*( return hm.getMixHealth() of WakuProtocol.StoreClientProtocol: return hm.getStoreClientHealth() - of WakuProtocol.LegacyStoreClientProtocol: - return hm.getLegacyStoreClientHealth() of WakuProtocol.FilterClientProtocol: return hm.getFilterClientHealth() of WakuProtocol.LightpushClientProtocol: @@ -349,7 +317,6 @@ proc getSyncAllProtocolHealthInfo(hm: NodeHealthMonitor): seq[ProtocolHealth] = protocols.add(hm.getLegacyLightpushHealth(relayHealth.health)) protocols.add(hm.getFilterHealth(relayHealth.health)) protocols.add(hm.getStoreHealth()) - protocols.add(hm.getLegacyStoreHealth()) protocols.add(hm.getPeerExchangeHealth()) protocols.add(hm.getRendezvousHealth()) protocols.add(hm.getMixHealth()) @@ -357,7 +324,6 @@ proc getSyncAllProtocolHealthInfo(hm: NodeHealthMonitor): seq[ProtocolHealth] = protocols.add(hm.getLightpushClientHealth()) protocols.add(hm.getLegacyLightpushClientHealth()) protocols.add(hm.getStoreClientHealth()) - protocols.add(hm.getLegacyStoreClientHealth()) protocols.add(hm.getFilterClientHealth()) return protocols diff --git a/waku/node/kernel_api/relay.nim b/waku/node/kernel_api/relay.nim index ec4d05ddd..c5a11ff02 100644 --- a/waku/node/kernel_api/relay.nim +++ b/waku/node/kernel_api/relay.nim @@ -24,7 +24,6 @@ import waku_core, waku_core/topics/sharding, waku_filter_v2, - waku_archive_legacy, waku_archive, waku_store_sync, waku_rln_relay, @@ -81,11 +80,6 @@ proc registerRelayHandler( await node.wakuFilter.handleMessage(topic, msg) proc archiveHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = - if not node.wakuLegacyArchive.isNil(): - ## we try to store with legacy archive - await node.wakuLegacyArchive.handleMessage(topic, msg) - return - if node.wakuArchive.isNil(): return diff --git a/waku/node/kernel_api/store.nim b/waku/node/kernel_api/store.nim index ca9917163..fcf0dfc89 100644 --- a/waku/node/kernel_api/store.nim +++ b/waku/node/kernel_api/store.nim @@ -20,17 +20,13 @@ import import ../waku_node, ../../waku_core, - ../../waku_store_legacy/protocol as legacy_store, - ../../waku_store_legacy/client as legacy_store_client, - ../../waku_store_legacy/common as legacy_store_common, ../../waku_store/protocol as store, ../../waku_store/client as store_client, ../../waku_store/common as store_common, ../../waku_store/resume, ../peer_manager, ../../common/rate_limit/setting, - ../../waku_archive, - ../../waku_archive_legacy + ../../waku_archive logScope: topics = "waku node store api" @@ -50,157 +46,6 @@ proc mountArchive*( return ok() -proc mountLegacyArchive*( - node: WakuNode, driver: waku_archive_legacy.ArchiveDriver -): Result[void, string] = - node.wakuLegacyArchive = waku_archive_legacy.WakuArchive.new(driver = driver).valueOr: - return err("error in mountLegacyArchive: " & error) - - return ok() - -## Legacy Waku Store - -# TODO: Review this mapping logic. Maybe, move it to the appplication code -proc toArchiveQuery( - request: legacy_store_common.HistoryQuery -): waku_archive_legacy.ArchiveQuery = - waku_archive_legacy.ArchiveQuery( - pubsubTopic: request.pubsubTopic, - contentTopics: request.contentTopics, - cursor: request.cursor.map( - proc(cursor: HistoryCursor): waku_archive_legacy.ArchiveCursor = - waku_archive_legacy.ArchiveCursor( - pubsubTopic: cursor.pubsubTopic, - senderTime: cursor.senderTime, - storeTime: cursor.storeTime, - digest: cursor.digest, - ) - ), - startTime: request.startTime, - endTime: request.endTime, - pageSize: request.pageSize.uint, - direction: request.direction, - requestId: request.requestId, - ) - -# TODO: Review this mapping logic. Maybe, move it to the appplication code -proc toHistoryResult*( - res: waku_archive_legacy.ArchiveResult -): legacy_store_common.HistoryResult = - let response = res.valueOr: - case error.kind - of waku_archive_legacy.ArchiveErrorKind.DRIVER_ERROR, - waku_archive_legacy.ArchiveErrorKind.INVALID_QUERY: - return err(HistoryError(kind: HistoryErrorKind.BAD_REQUEST, cause: error.cause)) - else: - return err(HistoryError(kind: HistoryErrorKind.UNKNOWN)) - return ok( - HistoryResponse( - messages: response.messages, - cursor: response.cursor.map( - proc(cursor: waku_archive_legacy.ArchiveCursor): HistoryCursor = - HistoryCursor( - pubsubTopic: cursor.pubsubTopic, - senderTime: cursor.senderTime, - storeTime: cursor.storeTime, - digest: cursor.digest, - ) - ), - ) - ) - -proc mountLegacyStore*( - node: WakuNode, rateLimit: RateLimitSetting = DefaultGlobalNonRelayRateLimit -) {.async.} = - info "mounting waku legacy store protocol" - - if node.wakuLegacyArchive.isNil(): - error "failed to mount waku legacy store protocol", error = "waku archive not set" - return - - # TODO: Review this handler logic. Maybe, move it to the appplication code - let queryHandler: HistoryQueryHandler = proc( - request: HistoryQuery - ): Future[legacy_store_common.HistoryResult] {.async.} = - if request.cursor.isSome(): - ?request.cursor.get().checkHistCursor() - - let request = request.toArchiveQuery() - let response = await node.wakuLegacyArchive.findMessagesV2(request) - return response.toHistoryResult() - - node.wakuLegacyStore = legacy_store.WakuStore.new( - node.peerManager, node.rng, queryHandler, some(rateLimit) - ) - - if node.started: - # Node has started already. Let's start store too. - await node.wakuLegacyStore.start() - - node.switch.mount( - node.wakuLegacyStore, protocolMatcher(legacy_store_common.WakuLegacyStoreCodec) - ) - -proc mountLegacyStoreClient*(node: WakuNode) = - info "mounting legacy store client" - - node.wakuLegacyStoreClient = - legacy_store_client.WakuStoreClient.new(node.peerManager, node.rng) - -proc query*( - node: WakuNode, query: legacy_store_common.HistoryQuery, peer: RemotePeerInfo -): Future[legacy_store_common.WakuStoreResult[legacy_store_common.HistoryResponse]] {. - async, gcsafe -.} = - ## Queries known nodes for historical messages - if node.wakuLegacyStoreClient.isNil(): - return err("waku legacy store client is nil") - - let response = (await node.wakuLegacyStoreClient.query(query, peer)).valueOr: - return err("legacy store client query error: " & $error) - - return ok(response) - -# TODO: Move to application module (e.g., wakunode2.nim) -proc query*( - node: WakuNode, query: legacy_store_common.HistoryQuery -): Future[legacy_store_common.WakuStoreResult[legacy_store_common.HistoryResponse]] {. - async, gcsafe, deprecated: "Use 'node.query()' with peer destination instead" -.} = - ## Queries known nodes for historical messages - if node.wakuLegacyStoreClient.isNil(): - return err("waku legacy store client is nil") - - let peerOpt = node.peerManager.selectPeer(legacy_store_common.WakuLegacyStoreCodec) - if peerOpt.isNone(): - error "no suitable remote peers" - return err("peer_not_found_failure") - - return await node.query(query, peerOpt.get()) - -when defined(waku_exp_store_resume): - # TODO: Move to application module (e.g., wakunode2.nim) - proc resume*( - node: WakuNode, peerList: Option[seq[RemotePeerInfo]] = none(seq[RemotePeerInfo]) - ) {.async, gcsafe.} = - ## resume proc retrieves the history of waku messages published on the default waku pubsub topic since the last time the waku node has been online - ## for resume to work properly the waku node must have the store protocol mounted in the full mode (i.e., persisting messages) - ## messages are stored in the wakuStore's messages field and in the message db - ## the offline time window is measured as the difference between the current time and the timestamp of the most recent persisted waku message - ## an offset of 20 second is added to the time window to count for nodes asynchrony - ## peerList indicates the list of peers to query from. The history is fetched from the first available peer in this list. Such candidates should be found through a discovery method (to be developed). - ## if no peerList is passed, one of the peers in the underlying peer manager unit of the store protocol is picked randomly to fetch the history from. - ## The history gets fetched successfully if the dialed peer has been online during the queried time window. - if node.wakuLegacyStoreClient.isNil(): - return - - let retrievedMessages = (await node.wakuLegacyStoreClient.resume(peerList)).valueOr: - error "failed to resume store", error = error - return - - info "the number of retrieved messages since the last online time: ", - number = retrievedMessages.value - ## Waku Store proc toArchiveQuery(request: StoreQueryRequest): waku_archive.ArchiveQuery = diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 0c6cb7ac4..92528c7b9 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -32,10 +32,6 @@ import waku_core/topics/sharding, waku_relay, waku_archive, - waku_archive_legacy, - waku_store_legacy/protocol as legacy_store, - waku_store_legacy/client as legacy_store_client, - waku_store_legacy/common as legacy_store_common, waku_store/protocol as store, waku_store/client as store_client, waku_store/common as store_common, @@ -116,9 +112,6 @@ type switch*: Switch wakuRelay*: WakuRelay wakuArchive*: waku_archive.WakuArchive - wakuLegacyArchive*: waku_archive_legacy.WakuArchive - wakuLegacyStore*: legacy_store.WakuStore - wakuLegacyStoreClient*: legacy_store_client.WakuStoreClient wakuStore*: store.WakuStore wakuStoreClient*: store_client.WakuStoreClient wakuStoreResume*: StoreResume diff --git a/waku/rest_api/endpoint/admin/handlers.nim b/waku/rest_api/endpoint/admin/handlers.nim index 1efbf7d04..1904d43f9 100644 --- a/waku/rest_api/endpoint/admin/handlers.nim +++ b/waku/rest_api/endpoint/admin/handlers.nim @@ -12,7 +12,6 @@ import waku/[ waku_core, waku_core/topics/pubsub_topic, - waku_store_legacy/common, waku_store/common, waku_filter_v2, waku_lightpush_legacy/common, @@ -172,7 +171,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = let peers = populateAdminPeerInfoForCodecs( node, @[ - WakuRelayCodec, WakuFilterSubscribeCodec, WakuStoreCodec, WakuLegacyStoreCodec, + WakuRelayCodec, WakuFilterSubscribeCodec, WakuStoreCodec, WakuLegacyLightPushCodec, WakuLightPushCodec, WakuPeerExchangeCodec, WakuReconciliationCodec, WakuTransferCodec, ], @@ -366,8 +365,6 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = protoStats[WakuFilterPushCodec] = peers.countIt(it.protocols.contains(WakuFilterPushCodec)) protoStats[WakuStoreCodec] = peers.countIt(it.protocols.contains(WakuStoreCodec)) - protoStats[WakuLegacyStoreCodec] = - peers.countIt(it.protocols.contains(WakuLegacyStoreCodec)) protoStats[WakuLightPushCodec] = peers.countIt(it.protocols.contains(WakuLightPushCodec)) protoStats[WakuLegacyLightPushCodec] = diff --git a/waku/rest_api/endpoint/builder.nim b/waku/rest_api/endpoint/builder.nim index cb23bc284..9b4ecf662 100644 --- a/waku/rest_api/endpoint/builder.nim +++ b/waku/rest_api/endpoint/builder.nim @@ -14,7 +14,6 @@ import waku/rest_api/endpoint/legacy_lightpush/handlers as rest_legacy_lightpush_endpoint, waku/rest_api/endpoint/lightpush/handlers as rest_lightpush_endpoint, waku/rest_api/endpoint/store/handlers as rest_store_endpoint, - waku/rest_api/endpoint/legacy_store/handlers as rest_store_legacy_endpoint, waku/rest_api/endpoint/health/handlers as rest_health_endpoint, waku/rest_api/endpoint/admin/handlers as rest_admin_endpoint, waku/waku_core/topics, @@ -195,7 +194,6 @@ proc startRestServerProtocolSupport*( none(DiscoveryHandler) rest_store_endpoint.installStoreApiHandlers(router, node, storeDiscoHandler) - rest_store_legacy_endpoint.installStoreApiHandlers(router, node, storeDiscoHandler) ## Light push API ## Install it either if client is mounted) diff --git a/waku/rest_api/endpoint/legacy_store/client.nim b/waku/rest_api/endpoint/legacy_store/client.nim deleted file mode 100644 index 24ad38d9a..000000000 --- a/waku/rest_api/endpoint/legacy_store/client.nim +++ /dev/null @@ -1,75 +0,0 @@ -{.push raises: [].} - -import - chronicles, json_serialization, json_serialization/std/options, presto/[route, client] -import ../../../waku_store_legacy/common, ../serdes, ../responses, ./types - -export types - -logScope: - topics = "waku node rest legacy store_api" - -proc decodeBytes*( - t: typedesc[StoreResponseRest], - data: openArray[byte], - contentType: Opt[ContentTypeData], -): RestResult[StoreResponseRest] = - if MediaType.init($contentType) == MIMETYPE_JSON: - let decoded = ?decodeFromJsonBytes(StoreResponseRest, data) - return ok(decoded) - - if MediaType.init($contentType) == MIMETYPE_TEXT: - var res: string - if len(data) > 0: - res = newString(len(data)) - copyMem(addr res[0], unsafeAddr data[0], len(data)) - - return ok( - StoreResponseRest( - messages: newSeq[StoreWakuMessage](0), - cursor: none(HistoryCursorRest), - # field that contain error information - errorMessage: some(res), - ) - ) - - # If everything goes wrong - return err(cstring("Unsupported contentType " & $contentType)) - -proc getStoreMessagesV1*( - # URL-encoded reference to the store-node - peerAddr: string = "", - pubsubTopic: string = "", - # URL-encoded comma-separated list of content topics - contentTopics: string = "", - startTime: string = "", - endTime: string = "", - - # Optional cursor fields - senderTime: string = "", - storeTime: string = "", - digest: string = "", # base64-encoded digest - pageSize: string = "", - ascending: string = "", -): RestResponse[StoreResponseRest] {. - rest, endpoint: "/store/v1/messages", meth: HttpMethod.MethodGet -.} - -proc getStoreMessagesV1*( - # URL-encoded reference to the store-node - peerAddr: Option[string], - pubsubTopic: string = "", - # URL-encoded comma-separated list of content topics - contentTopics: string = "", - startTime: string = "", - endTime: string = "", - - # Optional cursor fields - senderTime: string = "", - storeTime: string = "", - digest: string = "", # base64-encoded digest - pageSize: string = "", - ascending: string = "", -): RestResponse[StoreResponseRest] {. - rest, endpoint: "/store/v1/messages", meth: HttpMethod.MethodGet -.} diff --git a/waku/rest_api/endpoint/legacy_store/handlers.nim b/waku/rest_api/endpoint/legacy_store/handlers.nim deleted file mode 100644 index 4ed58f799..000000000 --- a/waku/rest_api/endpoint/legacy_store/handlers.nim +++ /dev/null @@ -1,246 +0,0 @@ -{.push raises: [].} - -import - std/[strformat, sugar], results, chronicles, uri, json_serialization, presto/route -import - ../../../waku_core, - ../../../waku_store_legacy/common, - ../../../waku_store_legacy/self_req_handler, - ../../../waku_node, - ../../../node/peer_manager, - ../../../common/paging, - ../../handlers, - ../responses, - ../serdes, - ./types - -export types - -logScope: - topics = "waku node rest legacy store_api" - -const futTimeout* = 5.seconds # Max time to wait for futures - -const NoPeerNoDiscError* = - RestApiResponse.preconditionFailed("No suitable service peer & no discovery method") - -# Queries the store-node with the query parameters and -# returns a RestApiResponse that is sent back to the api client. -proc performHistoryQuery( - selfNode: WakuNode, histQuery: HistoryQuery, storePeer: RemotePeerInfo -): Future[RestApiResponse] {.async.} = - let queryFut = selfNode.query(histQuery, storePeer) - if not await queryFut.withTimeout(futTimeout): - const msg = "No history response received (timeout)" - error msg - return RestApiResponse.internalServerError(msg) - - let storeResp = queryFut.read().map(res => res.toStoreResponseRest()).valueOr: - const msg = "Error occurred in queryFut.read()" - error msg, error = error - return RestApiResponse.internalServerError(fmt("{msg} [{error}]")) - - let resp = RestApiResponse.jsonResponse(storeResp, status = Http200).valueOr: - const msg = "Error building the json respose" - error msg, error = error - return RestApiResponse.internalServerError(fmt("{msg} [{error}]")) - - return resp - -# Converts a string time representation into an Option[Timestamp]. -# Only positive time is considered a valid Timestamp in the request -proc parseTime(input: Option[string]): Result[Option[Timestamp], string] = - if input.isSome() and input.get() != "": - try: - let time = parseInt(input.get()) - if time > 0: - return ok(some(Timestamp(time))) - except ValueError: - return err("Problem parsing time [" & getCurrentExceptionMsg() & "]") - - return ok(none(Timestamp)) - -# Generates a history query cursor as per the given params -proc parseCursor( - parsedPubsubTopic: Option[string], - senderTime: Option[string], - storeTime: Option[string], - digest: Option[string], -): Result[Option[HistoryCursor], string] = - # Parse sender time - let parsedSenderTime = ?parseTime(senderTime) - - # Parse store time - let parsedStoreTime = ?parseTime(storeTime) - - # Parse message digest - let parsedMsgDigest = ?parseMsgDigest(digest) - - # Parse cursor information - if parsedPubsubTopic.isSome() and parsedSenderTime.isSome() and - parsedStoreTime.isSome() and parsedMsgDigest.isSome(): - return ok( - some( - HistoryCursor( - pubsubTopic: parsedPubsubTopic.get(), - senderTime: parsedSenderTime.get(), - storeTime: parsedStoreTime.get(), - digest: parsedMsgDigest.get(), - ) - ) - ) - else: - return ok(none(HistoryCursor)) - -# Creates a HistoryQuery from the given params -proc createHistoryQuery( - pubsubTopic: Option[string], - contentTopics: Option[string], - senderTime: Option[string], - storeTime: Option[string], - digest: Option[string], - startTime: Option[string], - endTime: Option[string], - pageSize: Option[string], - direction: Option[string], -): Result[HistoryQuery, string] = - # Parse pubsubTopic parameter - var parsedPubsubTopic = none(string) - if pubsubTopic.isSome(): - let decodedPubsubTopic = decodeUrl(pubsubTopic.get()) - if decodedPubsubTopic != "": - parsedPubsubTopic = some(decodedPubsubTopic) - - # Parse the content topics - var parsedContentTopics = newSeq[ContentTopic](0) - if contentTopics.isSome(): - let ctList = decodeUrl(contentTopics.get()) - if ctList != "": - for ct in ctList.split(','): - parsedContentTopics.add(ct) - - # Parse cursor information - let parsedCursor = ?parseCursor(parsedPubsubTopic, senderTime, storeTime, digest) - - # Parse page size field - var parsedPagedSize = DefaultPageSize - if pageSize.isSome() and pageSize.get() != "": - try: - parsedPagedSize = uint64(parseInt(pageSize.get())) - except CatchableError: - return err("Problem parsing page size [" & getCurrentExceptionMsg() & "]") - - # Parse start time - let parsedStartTime = ?parseTime(startTime) - - # Parse end time - let parsedEndTime = ?parseTime(endTime) - - # Parse ascending field - var parsedDirection = default() - if direction.isSome() and direction.get() != "": - parsedDirection = direction.get().into() - - return ok( - HistoryQuery( - pubsubTopic: parsedPubsubTopic, - contentTopics: parsedContentTopics, - startTime: parsedStartTime, - endTime: parsedEndTime, - direction: parsedDirection, - pageSize: parsedPagedSize, - cursor: parsedCursor, - ) - ) - -# Simple type conversion. The "Option[Result[string, cstring]]" -# type is used by the nim-presto library. -proc toOpt(self: Option[Result[string, cstring]]): Option[string] = - if not self.isSome() or self.get().value == "": - return none(string) - if self.isSome() and self.get().value != "": - return some(self.get().value) - -proc retrieveMsgsFromSelfNode( - self: WakuNode, histQuery: HistoryQuery -): Future[RestApiResponse] {.async.} = - ## Performs a "store" request to the local node (self node.) - ## Notice that this doesn't follow the regular store libp2p channel because a node - ## it is not allowed to libp2p-dial a node to itself, by default. - ## - - let selfResp = (await self.wakuLegacyStore.handleSelfStoreRequest(histQuery)).valueOr: - return RestApiResponse.internalServerError($error) - - let storeResp = selfResp.toStoreResponseRest() - let resp = RestApiResponse.jsonResponse(storeResp, status = Http200).valueOr: - const msg = "Error building the json respose" - let e = $error - error msg, error = e - return RestApiResponse.internalServerError(fmt("{msg} [{e}]")) - - return resp - -# Subscribes the rest handler to attend "/store/v1/messages" requests -proc installStoreApiHandlers*( - router: var RestRouter, - node: WakuNode, - discHandler: Option[DiscoveryHandler] = none(DiscoveryHandler), -) = - # Handles the store-query request according to the passed parameters - router.api(MethodGet, "/store/v1/messages") do( - peerAddr: Option[string], - pubsubTopic: Option[string], - contentTopics: Option[string], - senderTime: Option[string], - storeTime: Option[string], - digest: Option[string], - startTime: Option[string], - endTime: Option[string], - pageSize: Option[string], - ascending: Option[string] - ) -> RestApiResponse: - info "REST-GET /store/v1/messages ", peer_addr = $peerAddr - - # All the GET parameters are URL-encoded (https://en.wikipedia.org/wiki/URL_encoding) - # Example: - # /store/v1/messages?peerAddr=%2Fip4%2F127.0.0.1%2Ftcp%2F60001%2Fp2p%2F16Uiu2HAmVFXtAfSj4EiR7mL2KvL4EE2wztuQgUSBoj2Jx2KeXFLN\&pubsubTopic=my-waku-topic - - # Parse the rest of the parameters and create a HistoryQuery - let histQuery = createHistoryQuery( - pubsubTopic.toOpt(), - contentTopics.toOpt(), - senderTime.toOpt(), - storeTime.toOpt(), - digest.toOpt(), - startTime.toOpt(), - endTime.toOpt(), - pageSize.toOpt(), - ascending.toOpt(), - ).valueOr: - return RestApiResponse.badRequest(error) - - if peerAddr.isNone() and not node.wakuLegacyStore.isNil(): - ## The user didn't specify a peer address and self-node is configured as a store node. - ## In this case we assume that the user is willing to retrieve the messages stored by - ## the local/self store node. - return await node.retrieveMsgsFromSelfNode(histQuery) - - # Parse the peer address parameter - let parsedPeerAddr = parseUrlPeerAddr(peerAddr.toOpt()).valueOr: - return RestApiResponse.badRequest(error) - - let peerAddr = parsedPeerAddr.valueOr: - node.peerManager.selectPeer(WakuLegacyStoreCodec).valueOr: - let handler = discHandler.valueOr: - return NoPeerNoDiscError - - let peerOp = (await handler()).valueOr: - return RestApiResponse.internalServerError($error) - - peerOp.valueOr: - return RestApiResponse.preconditionFailed( - "No suitable service peer & none discovered" - ) - - return await node.performHistoryQuery(histQuery, peerAddr) diff --git a/waku/rest_api/endpoint/legacy_store/types.nim b/waku/rest_api/endpoint/legacy_store/types.nim deleted file mode 100644 index 0c547c7cc..000000000 --- a/waku/rest_api/endpoint/legacy_store/types.nim +++ /dev/null @@ -1,375 +0,0 @@ -{.push raises: [].} - -import - std/[sets, strformat, uri], - stew/byteutils, - chronicles, - json_serialization, - json_serialization/std/options, - presto/[route, client, common] -import - ../../../waku_store_legacy/common as waku_store_common, - ../../../common/base64, - ../../../waku_core, - ../serdes - -#### Types - -type - HistoryCursorRest* = object - pubsubTopic*: PubsubTopic - senderTime*: Timestamp - storeTime*: Timestamp - digest*: waku_store_common.MessageDigest - - StoreRequestRest* = object - # inspired by https://github.com/waku-org/nwaku/blob/f95147f5b7edfd45f914586f2d41cd18fb0e0d18/waku/v2//waku_store/common.nim#L52 - pubsubTopic*: Option[PubsubTopic] - contentTopics*: seq[ContentTopic] - cursor*: Option[HistoryCursorRest] - startTime*: Option[Timestamp] - endTime*: Option[Timestamp] - pageSize*: uint64 - ascending*: bool - - StoreWakuMessage* = object - payload*: Base64String - contentTopic*: Option[ContentTopic] - version*: Option[uint32] - timestamp*: Option[Timestamp] - ephemeral*: Option[bool] - meta*: Option[Base64String] - - StoreResponseRest* = object # inspired by https://rfc.vac.dev/spec/16/#storeresponse - messages*: seq[StoreWakuMessage] - cursor*: Option[HistoryCursorRest] - # field that contains error information - errorMessage*: Option[string] - -createJsonFlavor RestJson - -Json.setWriter JsonWriter, PreferredOutput = string - -#### Type conversion - -# Converts a URL-encoded-base64 string into a 'MessageDigest' -proc parseMsgDigest*( - input: Option[string] -): Result[Option[waku_store_common.MessageDigest], string] = - if not input.isSome() or input.get() == "": - return ok(none(waku_store_common.MessageDigest)) - - let decodedUrl = decodeUrl(input.get()) - let base64DecodedArr = ?base64.decode(Base64String(decodedUrl)) - - var messageDigest = waku_store_common.MessageDigest() - - # Next snippet inspired by "nwaku/waku/waku_archive/archive.nim" - # TODO: Improve coherence of MessageDigest type - messageDigest = block: - var data: array[32, byte] - for i in 0 ..< min(base64DecodedArr.len, 32): - data[i] = base64DecodedArr[i] - - waku_store_common.MessageDigest(data: data) - - return ok(some(messageDigest)) - -# Converts a given MessageDigest object into a suitable -# Base64-URL-encoded string suitable to be transmitted in a Rest -# request-response. The MessageDigest is first base64 encoded -# and this result is URL-encoded. -proc toRestStringMessageDigest*(self: waku_store_common.MessageDigest): string = - let base64Encoded = base64.encode(self.data) - encodeUrl($base64Encoded) - -proc toWakuMessage*(message: StoreWakuMessage): WakuMessage = - WakuMessage( - payload: base64.decode(message.payload).get(), - contentTopic: message.contentTopic.get(), - version: message.version.get(), - timestamp: message.timestamp.get(), - ephemeral: message.ephemeral.get(), - meta: message.meta.get(Base64String("")).decode().get(), - ) - -# Converts a 'HistoryResponse' object to an 'StoreResponseRest' -# that can be serialized to a json object. -proc toStoreResponseRest*(histResp: HistoryResponse): StoreResponseRest = - proc toStoreWakuMessage(message: WakuMessage): StoreWakuMessage = - StoreWakuMessage( - payload: base64.encode(message.payload), - contentTopic: some(message.contentTopic), - version: some(message.version), - timestamp: some(message.timestamp), - ephemeral: some(message.ephemeral), - meta: - if message.meta.len > 0: - some(base64.encode(message.meta)) - else: - none(Base64String), - ) - - var storeWakuMsgs: seq[StoreWakuMessage] - for m in histResp.messages: - storeWakuMsgs.add(m.toStoreWakuMessage()) - - var cursor = none(HistoryCursorRest) - if histResp.cursor.isSome: - cursor = some( - HistoryCursorRest( - pubsubTopic: histResp.cursor.get().pubsubTopic, - senderTime: histResp.cursor.get().senderTime, - storeTime: histResp.cursor.get().storeTime, - digest: histResp.cursor.get().digest, - ) - ) - - StoreResponseRest(messages: storeWakuMsgs, cursor: cursor) - -## Beginning of StoreWakuMessage serde - -proc writeValue*( - writer: var JsonWriter, value: StoreWakuMessage -) {.gcsafe, raises: [IOError].} = - writer.beginRecord() - writer.writeField("payload", $value.payload) - if value.contentTopic.isSome(): - writer.writeField("contentTopic", value.contentTopic.get()) - if value.version.isSome(): - writer.writeField("version", value.version.get()) - if value.timestamp.isSome(): - writer.writeField("timestamp", value.timestamp.get()) - if value.ephemeral.isSome(): - writer.writeField("ephemeral", value.ephemeral.get()) - if value.meta.isSome(): - writer.writeField("meta", value.meta.get()) - writer.endRecord() - -proc readValue*( - reader: var JsonReader, value: var StoreWakuMessage -) {.gcsafe, raises: [SerializationError, IOError].} = - var - payload = none(Base64String) - contentTopic = none(ContentTopic) - version = none(uint32) - timestamp = none(Timestamp) - ephemeral = none(bool) - meta = none(Base64String) - - var keys = initHashSet[string]() - for fieldName in readObjectFields(reader): - # Check for reapeated keys - if keys.containsOrIncl(fieldName): - let err = - try: - fmt"Multiple `{fieldName}` fields found" - except CatchableError: - "Multiple fields with the same name found" - reader.raiseUnexpectedField(err, "StoreWakuMessage") - - case fieldName - of "payload": - payload = some(reader.readValue(Base64String)) - of "contentTopic": - contentTopic = some(reader.readValue(ContentTopic)) - of "version": - version = some(reader.readValue(uint32)) - of "timestamp": - timestamp = some(reader.readValue(Timestamp)) - of "ephemeral": - ephemeral = some(reader.readValue(bool)) - of "meta": - meta = some(reader.readValue(Base64String)) - else: - reader.raiseUnexpectedField("Unrecognided field", cstring(fieldName)) - - if payload.isNone(): - reader.raiseUnexpectedValue("Field `payload` is missing") - - value = StoreWakuMessage( - payload: payload.get(), - contentTopic: contentTopic, - version: version, - timestamp: timestamp, - ephemeral: ephemeral, - meta: meta, - ) - -## End of StoreWakuMessage serde - -## Beginning of MessageDigest serde - -proc writeValue*( - writer: var JsonWriter, value: waku_store_common.MessageDigest -) {.gcsafe, raises: [IOError].} = - writer.beginRecord() - writer.writeField("data", base64.encode(value.data)) - writer.endRecord() - -proc readValue*( - reader: var JsonReader, value: var waku_store_common.MessageDigest -) {.gcsafe, raises: [SerializationError, IOError].} = - var data = none(seq[byte]) - - for fieldName in readObjectFields(reader): - case fieldName - of "data": - if data.isSome(): - reader.raiseUnexpectedField("Multiple `data` fields found", "MessageDigest") - let decoded = base64.decode(reader.readValue(Base64String)).valueOr: - reader.raiseUnexpectedField("Failed decoding data", "MessageDigest") - data = some(decoded) - else: - reader.raiseUnexpectedField("Unrecognided field", cstring(fieldName)) - - if data.isNone(): - reader.raiseUnexpectedValue("Field `data` is missing") - - for i in 0 ..< 32: - value.data[i] = data.get()[i] - -## End of MessageDigest serde - -## Beginning of HistoryCursorRest serde - -proc writeValue*( - writer: var JsonWriter, value: HistoryCursorRest -) {.gcsafe, raises: [IOError].} = - writer.beginRecord() - writer.writeField("pubsubTopic", value.pubsubTopic) - writer.writeField("senderTime", value.senderTime) - writer.writeField("storeTime", value.storeTime) - writer.writeField("digest", value.digest) - writer.endRecord() - -proc readValue*( - reader: var JsonReader, value: var HistoryCursorRest -) {.gcsafe, raises: [SerializationError, IOError].} = - var - pubsubTopic = none(PubsubTopic) - senderTime = none(Timestamp) - storeTime = none(Timestamp) - digest = none(waku_store_common.MessageDigest) - - for fieldName in readObjectFields(reader): - case fieldName - of "pubsubTopic": - if pubsubTopic.isSome(): - reader.raiseUnexpectedField( - "Multiple `pubsubTopic` fields found", "HistoryCursorRest" - ) - pubsubTopic = some(reader.readValue(PubsubTopic)) - of "senderTime": - if senderTime.isSome(): - reader.raiseUnexpectedField( - "Multiple `senderTime` fields found", "HistoryCursorRest" - ) - senderTime = some(reader.readValue(Timestamp)) - of "storeTime": - if storeTime.isSome(): - reader.raiseUnexpectedField( - "Multiple `storeTime` fields found", "HistoryCursorRest" - ) - storeTime = some(reader.readValue(Timestamp)) - of "digest": - if digest.isSome(): - reader.raiseUnexpectedField( - "Multiple `digest` fields found", "HistoryCursorRest" - ) - digest = some(reader.readValue(waku_store_common.MessageDigest)) - else: - reader.raiseUnexpectedField("Unrecognided field", cstring(fieldName)) - - if pubsubTopic.isNone(): - reader.raiseUnexpectedValue("Field `pubsubTopic` is missing") - - if senderTime.isNone(): - reader.raiseUnexpectedValue("Field `senderTime` is missing") - - if storeTime.isNone(): - reader.raiseUnexpectedValue("Field `storeTime` is missing") - - if digest.isNone(): - reader.raiseUnexpectedValue("Field `digest` is missing") - - value = HistoryCursorRest( - pubsubTopic: pubsubTopic.get(), - senderTime: senderTime.get(), - storeTime: storeTime.get(), - digest: digest.get(), - ) - -## End of HistoryCursorRest serde - -## Beginning of StoreResponseRest serde - -proc writeValue*( - writer: var JsonWriter, value: StoreResponseRest -) {.gcsafe, raises: [IOError].} = - writer.beginRecord() - writer.writeField("messages", value.messages) - if value.cursor.isSome(): - writer.writeField("cursor", value.cursor.get()) - if value.errorMessage.isSome(): - writer.writeField("errorMessage", value.errorMessage.get()) - writer.endRecord() - -proc readValue*( - reader: var JsonReader, value: var StoreResponseRest -) {.gcsafe, raises: [SerializationError, IOError].} = - var - messages = none(seq[StoreWakuMessage]) - cursor = none(HistoryCursorRest) - errorMessage = none(string) - - for fieldName in readObjectFields(reader): - case fieldName - of "messages": - if messages.isSome(): - reader.raiseUnexpectedField( - "Multiple `messages` fields found", "StoreResponseRest" - ) - messages = some(reader.readValue(seq[StoreWakuMessage])) - of "cursor": - if cursor.isSome(): - reader.raiseUnexpectedField( - "Multiple `cursor` fields found", "StoreResponseRest" - ) - cursor = some(reader.readValue(HistoryCursorRest)) - of "errorMessage": - if errorMessage.isSome(): - reader.raiseUnexpectedField( - "Multiple `errorMessage` fields found", "StoreResponseRest" - ) - errorMessage = some(reader.readValue(string)) - else: - reader.raiseUnexpectedField("Unrecognided field", cstring(fieldName)) - - if messages.isNone(): - reader.raiseUnexpectedValue("Field `messages` is missing") - - value = StoreResponseRest( - messages: messages.get(), cursor: cursor, errorMessage: errorMessage - ) - -## End of StoreResponseRest serde - -## Beginning of StoreRequestRest serde - -proc writeValue*( - writer: var JsonWriter, value: StoreRequestRest -) {.gcsafe, raises: [IOError].} = - writer.beginRecord() - if value.pubsubTopic.isSome(): - writer.writeField("pubsubTopic", value.pubsubTopic.get()) - writer.writeField("contentTopics", value.contentTopics) - if value.startTime.isSome(): - writer.writeField("startTime", value.startTime.get()) - if value.endTime.isSome(): - writer.writeField("endTime", value.endTime.get()) - writer.writeField("pageSize", value.pageSize) - writer.writeField("ascending", value.ascending) - writer.endRecord() - -## End of StoreRequestRest serde diff --git a/waku/waku_archive_legacy.nim b/waku/waku_archive_legacy.nim deleted file mode 100644 index bcb6b6a54..000000000 --- a/waku/waku_archive_legacy.nim +++ /dev/null @@ -1,6 +0,0 @@ -import - ./waku_archive_legacy/common, - ./waku_archive_legacy/archive, - ./waku_archive_legacy/driver - -export common, archive, driver diff --git a/waku/waku_archive_legacy/archive.nim b/waku/waku_archive_legacy/archive.nim deleted file mode 100644 index 7bf5685a5..000000000 --- a/waku/waku_archive_legacy/archive.nim +++ /dev/null @@ -1,285 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import - std/[times, options, sequtils, algorithm], - stew/byteutils, - chronicles, - chronos, - metrics, - results -import - ../common/paging, - ./driver, - ../waku_core, - ../waku_core/message/digest, - ./common, - ./archive_metrics - -logScope: - topics = "waku archive" - -const - DefaultPageSize*: uint = 20 - MaxPageSize*: uint = 100 - - # Retention policy - WakuArchiveDefaultRetentionPolicyInterval* = chronos.minutes(30) - - # Metrics reporting - WakuArchiveDefaultMetricsReportInterval* = chronos.minutes(30) - - # Message validation - # 20 seconds maximum allowable sender timestamp "drift" - MaxMessageTimestampVariance* = getNanoSecondTime(20) - -type MessageValidator* = - proc(msg: WakuMessage): Result[void, string] {.closure, gcsafe, raises: [].} - -## Archive - -type WakuArchive* = ref object - driver: ArchiveDriver - - validator: MessageValidator - -proc validate*(msg: WakuMessage): Result[void, string] = - if msg.ephemeral: - # Ephemeral message, do not store - return - - if msg.timestamp == 0: - return ok() - - let - now = getNanosecondTime(getTime().toUnixFloat()) - lowerBound = now - MaxMessageTimestampVariance - upperBound = now + MaxMessageTimestampVariance - - if msg.timestamp < lowerBound: - return err(invalidMessageOld) - - if upperBound < msg.timestamp: - return err(invalidMessageFuture) - - return ok() - -proc new*( - T: type WakuArchive, driver: ArchiveDriver, validator: MessageValidator = validate -): Result[T, string] = - if driver.isNil(): - return err("archive driver is Nil") - - let archive = WakuArchive(driver: driver, validator: validator) - - return ok(archive) - -proc handleMessage*( - self: WakuArchive, pubsubTopic: PubsubTopic, msg: WakuMessage -) {.async.} = - let - msgDigest = computeDigest(msg) - msgDigestHex = msgDigest.data.to0xHex() - msgHash = computeMessageHash(pubsubTopic, msg) - msgHashHex = msgHash.to0xHex() - msgTimestamp = - if msg.timestamp > 0: - msg.timestamp - else: - getNanosecondTime(getTime().toUnixFloat()) - - trace "handling message", - msg_hash = msgHashHex, - pubsubTopic = pubsubTopic, - contentTopic = msg.contentTopic, - msgTimestamp = msg.timestamp, - digest = msgDigestHex - - self.validator(msg).isOkOr: - waku_legacy_archive_errors.inc(labelValues = [error]) - trace "invalid message", - msg_hash = msgHashHex, - pubsubTopic = pubsubTopic, - contentTopic = msg.contentTopic, - timestamp = msg.timestamp, - error = error - return - - let insertStartTime = getTime().toUnixFloat() - - (await self.driver.put(pubsubTopic, msg, msgDigest, msgHash, msgTimestamp)).isOkOr: - waku_legacy_archive_errors.inc(labelValues = [insertFailure]) - error "failed to insert message", - msg_hash = msgHashHex, - pubsubTopic = pubsubTopic, - contentTopic = msg.contentTopic, - timestamp = msg.timestamp, - error = error - return - - let insertDuration = getTime().toUnixFloat() - insertStartTime - waku_legacy_archive_insert_duration_seconds.observe(insertDuration) - - info "message archived", - msg_hash = msgHashHex, - pubsubTopic = pubsubTopic, - contentTopic = msg.contentTopic, - msgTimestamp = msg.timestamp, - digest = msgDigestHex, - insertDuration = insertDuration - -proc findMessages*( - self: WakuArchive, query: ArchiveQuery -): Future[ArchiveResult] {.async, gcsafe.} = - ## Search the archive to return a single page of messages matching the query criteria - - let maxPageSize = - if query.pageSize <= 0: - DefaultPageSize - else: - min(query.pageSize, MaxPageSize) - - let isAscendingOrder = query.direction.into() - - if query.contentTopics.len > 10: - return err(ArchiveError.invalidQuery("too many content topics")) - - if query.cursor.isSome() and query.cursor.get().hash.len != 32: - return err(ArchiveError.invalidQuery("invalid cursor hash length")) - - let queryStartTime = getTime().toUnixFloat() - - let rows = ( - await self.driver.getMessages( - includeData = query.includeData, - contentTopic = query.contentTopics, - pubsubTopic = query.pubsubTopic, - cursor = query.cursor, - startTime = query.startTime, - endTime = query.endTime, - hashes = query.hashes, - maxPageSize = maxPageSize + 1, - ascendingOrder = isAscendingOrder, - requestId = query.requestId, - ) - ).valueOr: - return err(ArchiveError(kind: ArchiveErrorKind.DRIVER_ERROR, cause: error)) - - let queryDuration = getTime().toUnixFloat() - queryStartTime - waku_legacy_archive_query_duration_seconds.observe(queryDuration) - - var hashes = newSeq[WakuMessageHash]() - var messages = newSeq[WakuMessage]() - var topics = newSeq[PubsubTopic]() - var cursor = none(ArchiveCursor) - - if rows.len == 0: - return ok(ArchiveResponse(hashes: hashes, messages: messages, cursor: cursor)) - - ## Messages - let pageSize = min(rows.len, int(maxPageSize)) - - if query.includeData: - topics = rows[0 ..< pageSize].mapIt(it[0]) - messages = rows[0 ..< pageSize].mapIt(it[1]) - - hashes = rows[0 ..< pageSize].mapIt(it[4]) - - ## Cursor - if rows.len > int(maxPageSize): - ## Build last message cursor - ## The cursor is built from the last message INCLUDED in the response - ## (i.e. the second last message in the rows list) - - let (pubsubTopic, message, digest, storeTimestamp, hash) = rows[^2] - - cursor = some( - ArchiveCursor( - digest: MessageDigest.fromBytes(digest), - storeTime: storeTimestamp, - sendertime: message.timestamp, - pubsubTopic: pubsubTopic, - hash: hash, - ) - ) - - # All messages MUST be returned in chronological order - if not isAscendingOrder: - reverse(hashes) - reverse(messages) - reverse(topics) - - return ok( - ArchiveResponse(hashes: hashes, messages: messages, topics: topics, cursor: cursor) - ) - -proc findMessagesV2*( - self: WakuArchive, query: ArchiveQuery -): Future[ArchiveResult] {.async, deprecated, gcsafe.} = - ## Search the archive to return a single page of messages matching the query criteria - - let maxPageSize = - if query.pageSize <= 0: - DefaultPageSize - else: - min(query.pageSize, MaxPageSize) - - let isAscendingOrder = query.direction.into() - - if query.contentTopics.len > 10: - return err(ArchiveError.invalidQuery("too many content topics")) - - let queryStartTime = getTime().toUnixFloat() - - let rows = ( - await self.driver.getMessagesV2( - contentTopic = query.contentTopics, - pubsubTopic = query.pubsubTopic, - cursor = query.cursor, - startTime = query.startTime, - endTime = query.endTime, - maxPageSize = maxPageSize + 1, - ascendingOrder = isAscendingOrder, - requestId = query.requestId, - ) - ).valueOr: - return err(ArchiveError(kind: ArchiveErrorKind.DRIVER_ERROR, cause: error)) - - let queryDuration = getTime().toUnixFloat() - queryStartTime - waku_legacy_archive_query_duration_seconds.observe(queryDuration) - - var messages = newSeq[WakuMessage]() - var cursor = none(ArchiveCursor) - - if rows.len == 0: - return ok(ArchiveResponse(messages: messages, cursor: cursor)) - - ## Messages - let pageSize = min(rows.len, int(maxPageSize)) - - messages = rows[0 ..< pageSize].mapIt(it[1]) - - ## Cursor - if rows.len > int(maxPageSize): - ## Build last message cursor - ## The cursor is built from the last message INCLUDED in the response - ## (i.e. the second last message in the rows list) - - let (pubsubTopic, message, digest, storeTimestamp, _) = rows[^2] - - cursor = some( - ArchiveCursor( - digest: MessageDigest.fromBytes(digest), - storeTime: storeTimestamp, - sendertime: message.timestamp, - pubsubTopic: pubsubTopic, - ) - ) - - # All messages MUST be returned in chronological order - if not isAscendingOrder: - reverse(messages) - - return ok(ArchiveResponse(messages: messages, cursor: cursor)) diff --git a/waku/waku_archive_legacy/archive_metrics.nim b/waku/waku_archive_legacy/archive_metrics.nim deleted file mode 100644 index c3569a1ea..000000000 --- a/waku/waku_archive_legacy/archive_metrics.nim +++ /dev/null @@ -1,22 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import metrics - -declarePublicGauge waku_legacy_archive_messages, - "number of historical messages", ["type"] -declarePublicCounter waku_legacy_archive_errors, - "number of store protocol errors", ["type"] -declarePublicHistogram waku_legacy_archive_insert_duration_seconds, - "message insertion duration" -declarePublicHistogram waku_legacy_archive_query_duration_seconds, - "history query duration" - -# Error types (metric label values) -const - invalidMessageOld* = "invalid_message_too_old" - invalidMessageFuture* = "invalid_message_future_timestamp" - insertFailure* = "insert_failure" - retPolicyFailure* = "retpolicy_failure" diff --git a/waku/waku_archive_legacy/common.nim b/waku/waku_archive_legacy/common.nim deleted file mode 100644 index ed2b7272d..000000000 --- a/waku/waku_archive_legacy/common.nim +++ /dev/null @@ -1,88 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import std/options, results, stew/byteutils, stew/arrayops, nimcrypto/sha2 -import ../waku_core, ../common/paging - -## Waku message digest - -type MessageDigest* = MDigest[256] - -proc fromBytes*(T: type MessageDigest, src: seq[byte]): T = - var data: array[32, byte] - - let byteCount = copyFrom[byte](data, src) - - assert byteCount == 32 - - return MessageDigest(data: data) - -proc computeDigest*(msg: WakuMessage): MessageDigest = - var ctx: sha256 - ctx.init() - defer: - ctx.clear() - - ctx.update(msg.contentTopic.toBytes()) - ctx.update(msg.payload) - - # Computes the hash - return ctx.finish() - -## API types - -type - #TODO Once Store v2 is removed, the cursor becomes the hash of the last message - ArchiveCursor* = object - digest*: MessageDigest - storeTime*: Timestamp - senderTime*: Timestamp - pubsubTopic*: PubsubTopic - hash*: WakuMessageHash - - ArchiveQuery* = object - includeData*: bool # indicate if messages should be returned in addition to hashes. - pubsubTopic*: Option[PubsubTopic] - contentTopics*: seq[ContentTopic] - cursor*: Option[ArchiveCursor] - startTime*: Option[Timestamp] - endTime*: Option[Timestamp] - hashes*: seq[WakuMessageHash] - pageSize*: uint - direction*: PagingDirection - requestId*: string - - ArchiveResponse* = object - hashes*: seq[WakuMessageHash] - messages*: seq[WakuMessage] - topics*: seq[PubsubTopic] - cursor*: Option[ArchiveCursor] - - ArchiveErrorKind* {.pure.} = enum - UNKNOWN = uint32(0) - DRIVER_ERROR = uint32(1) - INVALID_QUERY = uint32(2) - - ArchiveError* = object - case kind*: ArchiveErrorKind - of DRIVER_ERROR, INVALID_QUERY: - # TODO: Add an enum to be able to distinguish between error causes - cause*: string - else: - discard - - ArchiveResult* = Result[ArchiveResponse, ArchiveError] - -proc `$`*(err: ArchiveError): string = - case err.kind - of ArchiveErrorKind.DRIVER_ERROR: - "DRIVER_ERROR: " & err.cause - of ArchiveErrorKind.INVALID_QUERY: - "INVALID_QUERY: " & err.cause - of ArchiveErrorKind.UNKNOWN: - "UNKNOWN" - -proc invalidQuery*(T: type ArchiveError, cause: string): T = - ArchiveError(kind: ArchiveErrorKind.INVALID_QUERY, cause: cause) diff --git a/waku/waku_archive_legacy/driver.nim b/waku/waku_archive_legacy/driver.nim deleted file mode 100644 index 8ff8df029..000000000 --- a/waku/waku_archive_legacy/driver.nim +++ /dev/null @@ -1,121 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import std/options, results, chronos -import ../waku_core, ./common - -const DefaultPageSize*: uint = 25 - -type - ArchiveDriverResult*[T] = Result[T, string] - ArchiveDriver* = ref object of RootObj - -#TODO Once Store v2 is removed keep only messages and hashes -type ArchiveRow* = (PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash) - -# ArchiveDriver interface - -method put*( - driver: ArchiveDriver, - pubsubTopic: PubsubTopic, - message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, -): Future[ArchiveDriverResult[void]] {.base, async.} = - discard - -method getAllMessages*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, async.} = - discard - -method getMessagesV2*( - driver: ArchiveDriver, - contentTopic = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId: string, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, deprecated, async.} = - discard - -method getMessages*( - driver: ArchiveDriver, - includeData = true, - contentTopic = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hashes = newSeq[WakuMessageHash](0), - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId = "", -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.base, async.} = - discard - -method getMessagesCount*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[int64]] {.base, async.} = - discard - -method getPagesCount*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[int64]] {.base, async.} = - discard - -method getPagesSize*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[int64]] {.base, async.} = - discard - -method getDatabaseSize*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[int64]] {.base, async.} = - discard - -method performVacuum*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[void]] {.base, async.} = - discard - -method getOldestMessageTimestamp*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[Timestamp]] {.base, async.} = - discard - -method getNewestMessageTimestamp*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[Timestamp]] {.base, async.} = - discard - -method deleteMessagesOlderThanTimestamp*( - driver: ArchiveDriver, ts: Timestamp -): Future[ArchiveDriverResult[void]] {.base, async.} = - discard - -method deleteOldestMessagesNotWithinLimit*( - driver: ArchiveDriver, limit: int -): Future[ArchiveDriverResult[void]] {.base, async.} = - discard - -method decreaseDatabaseSize*( - driver: ArchiveDriver, targetSizeInBytes: int64, forceRemoval: bool = false -): Future[ArchiveDriverResult[void]] {.base, async.} = - discard - -method close*( - driver: ArchiveDriver -): Future[ArchiveDriverResult[void]] {.base, async.} = - discard - -method existsTable*( - driver: ArchiveDriver, tableName: string -): Future[ArchiveDriverResult[bool]] {.base, async.} = - discard diff --git a/waku/waku_archive_legacy/driver/builder.nim b/waku/waku_archive_legacy/driver/builder.nim deleted file mode 100644 index 0f19b3669..000000000 --- a/waku/waku_archive_legacy/driver/builder.nim +++ /dev/null @@ -1,89 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import results, chronicles, chronos -import - ../driver, - ../../common/databases/dburl, - ../../common/databases/db_sqlite, - ../../common/error_handling, - ./sqlite_driver, - ./sqlite_driver/migrations as archive_driver_sqlite_migrations, - ./queue_driver - -export sqlite_driver, queue_driver - -when defined(postgres): - import ## These imports add dependency with an external libpq library - ./postgres_driver - export postgres_driver - -proc new*( - T: type ArchiveDriver, - url: string, - vacuum: bool, - migrate: bool, - maxNumConn: int, - onFatalErrorAction: OnFatalErrorHandler, -): Future[Result[T, string]] {.async.} = - ## url - string that defines the database - ## vacuum - if true, a cleanup operation will be applied to the database - ## migrate - if true, the database schema will be updated - ## maxNumConn - defines the maximum number of connections to handle simultaneously (Postgres) - ## onFatalErrorAction - called if, e.g., the connection with db got lost - - dburl.validateDbUrl(url).isOkOr: - return err("DbUrl failure in ArchiveDriver.new: " & error) - - let engine = dburl.getDbEngine(url).valueOr: - return err("error getting db engine in setupWakuArchiveDriver: " & error) - - case engine - of "sqlite": - let path = dburl.getDbPath(url).valueOr: - return err("error get path in setupWakuArchiveDriver: " & error) - - let db = SqliteDatabase.new(path).valueOr: - return err("error in setupWakuArchiveDriver: " & error) - - # SQLite vacuum - let (pageSize, pageCount, freelistCount) = db.gatherSqlitePageStats().valueOr: - return err("error while gathering sqlite stats: " & $error) - - info "sqlite database page stats", - pageSize = pageSize, pages = pageCount, freePages = freelistCount - - if vacuum and (pageCount > 0 and freelistCount > 0): - db.performSqliteVacuum().isOkOr: - return err("error in vacuum sqlite: " & $error) - - # Database migration - if migrate: - archive_driver_sqlite_migrations.migrate(db).isOkOr: - return err("error in migrate sqlite: " & $error) - - info "setting up sqlite waku archive driver" - let res = SqliteDriver.new(db).valueOr: - return err("failed to init sqlite archive driver: " & error) - - return ok(res) - of "postgres": - when defined(postgres): - let driver = PostgresDriver.new( - dbUrl = url, - maxConnections = maxNumConn, - onFatalErrorAction = onFatalErrorAction, - ).valueOr: - return err("failed to init postgres archive driver: " & error) - - return ok(driver) - else: - return err( - "Postgres has been configured but not been compiled. Check compiler definitions." - ) - else: - info "setting up in-memory waku archive driver" - let driver = QueueDriver.new() # Defaults to a capacity of 25.000 messages - return ok(driver) diff --git a/waku/waku_archive_legacy/driver/postgres_driver.nim b/waku/waku_archive_legacy/driver/postgres_driver.nim deleted file mode 100644 index 496005cbe..000000000 --- a/waku/waku_archive_legacy/driver/postgres_driver.nim +++ /dev/null @@ -1,8 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import ./postgres_driver/postgres_driver - -export postgres_driver diff --git a/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim deleted file mode 100644 index a6784e4f8..000000000 --- a/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim +++ /dev/null @@ -1,976 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import - std/[options, sequtils, strutils, strformat, times], - stew/[byteutils, arrayops], - results, - chronos, - db_connector/[postgres, db_common], - chronicles -import - ../../../common/error_handling, - ../../../waku_core, - ../../common, - ../../driver, - ./postgres_healthcheck, - ../../../common/databases/db_postgres as waku_postgres - -type PostgresDriver* = ref object of ArchiveDriver - ## Establish a separate pools for read/write operations - writeConnPool: PgAsyncPool - readConnPool: PgAsyncPool - -const InsertRowStmtName = "InsertRow" -const InsertRowStmtDefinition = # TODO: get the sql queries from a file - """INSERT INTO messages (id, messageHash, contentTopic, payload, pubsubTopic, - version, timestamp, meta) VALUES ($1, $2, $3, $4, $5, $6, $7, CASE WHEN $8 = '' THEN NULL ELSE $8 END) ON CONFLICT DO NOTHING;""" - -const InsertRowInMessagesLookupStmtName = "InsertRowMessagesLookup" -const InsertRowInMessagesLookupStmtDefinition = - """INSERT INTO messages_lookup (messageHash, timestamp) VALUES ($1, $2) ON CONFLICT DO NOTHING;""" - -const SelectNoCursorAscStmtName = "SelectWithoutCursorAsc" -const SelectNoCursorAscStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - timestamp >= $4 AND - timestamp <= $5 - ORDER BY timestamp ASC, messageHash ASC LIMIT $6;""" - -const SelectNoCursorDescStmtName = "SelectWithoutCursorDesc" -const SelectNoCursorDescStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - timestamp >= $4 AND - timestamp <= $5 - ORDER BY timestamp DESC, messageHash DESC LIMIT $6;""" - -const SelectWithCursorDescStmtName = "SelectWithCursorDesc" -const SelectWithCursorDescStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - (timestamp, messageHash) < ($4,$5) AND - timestamp >= $6 AND - timestamp <= $7 - ORDER BY timestamp DESC, messageHash DESC LIMIT $8;""" - -const SelectWithCursorAscStmtName = "SelectWithCursorAsc" -const SelectWithCursorAscStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - messageHash IN ($2) AND - pubsubTopic = $3 AND - (timestamp, messageHash) > ($4,$5) AND - timestamp >= $6 AND - timestamp <= $7 - ORDER BY timestamp ASC, messageHash ASC LIMIT $8;""" - -const SelectMessageByHashName = "SelectMessageByHash" -const SelectMessageByHashDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages WHERE messageHash = $1""" - -const SelectNoCursorV2AscStmtName = "SelectWithoutCursorV2Asc" -const SelectNoCursorV2AscStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - timestamp >= $3 AND - timestamp <= $4 - ORDER BY timestamp ASC LIMIT $5;""" - -const SelectNoCursorV2DescStmtName = "SelectWithoutCursorV2Desc" -const SelectNoCursorV2DescStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - timestamp >= $3 AND - timestamp <= $4 - ORDER BY timestamp DESC LIMIT $5;""" - -const SelectWithCursorV2DescStmtName = "SelectWithCursorV2Desc" -const SelectWithCursorV2DescStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - (timestamp, id) < ($3,$4) AND - timestamp >= $5 AND - timestamp <= $6 - ORDER BY timestamp DESC LIMIT $7;""" - -const SelectWithCursorV2AscStmtName = "SelectWithCursorV2Asc" -const SelectWithCursorV2AscStmtDef = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages - WHERE contentTopic IN ($1) AND - pubsubTopic = $2 AND - (timestamp, id) > ($3,$4) AND - timestamp >= $5 AND - timestamp <= $6 - ORDER BY timestamp ASC LIMIT $7;""" - -const DefaultMaxNumConns = 50 - -proc new*( - T: type PostgresDriver, - dbUrl: string, - maxConnections = DefaultMaxNumConns, - onFatalErrorAction: OnFatalErrorHandler = nil, -): ArchiveDriverResult[T] = - ## Very simplistic split of max connections - let maxNumConnOnEachPool = int(maxConnections / 2) - - let readConnPool = PgAsyncPool.new(dbUrl, maxNumConnOnEachPool).valueOr: - return err("error creating read conn pool PgAsyncPool") - - let writeConnPool = PgAsyncPool.new(dbUrl, maxNumConnOnEachPool).valueOr: - return err("error creating write conn pool PgAsyncPool") - - if not isNil(onFatalErrorAction): - asyncSpawn checkConnectivity(readConnPool, onFatalErrorAction) - - if not isNil(onFatalErrorAction): - asyncSpawn checkConnectivity(writeConnPool, onFatalErrorAction) - - let driver = PostgresDriver(writeConnPool: writeConnPool, readConnPool: readConnPool) - return ok(driver) - -proc reset*(s: PostgresDriver): Future[ArchiveDriverResult[void]] {.async.} = - ## Clear the database partitions - let targetSize = 0 - let forceRemoval = true - let ret = await s.decreaseDatabaseSize(targetSize, forceRemoval) - return ret - -proc rowCallbackImpl( - pqResult: ptr PGresult, - outRows: var seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)], -) = - ## Proc aimed to contain the logic of the callback passed to the `psasyncpool`. - ## That callback is used in "SELECT" queries. - ## - ## pqResult - contains the query results - ## outRows - seq of Store-rows. This is populated from the info contained in pqResult - - let numFields = pqResult.pqnfields() - if numFields != 8: - error "Wrong number of fields, expected 8", numFields - return - - for iRow in 0 ..< pqResult.pqNtuples(): - var wakuMessage: WakuMessage - var timestamp: Timestamp - var version: uint - var pubSubTopic: string - var contentTopic: string - var digest: string - var payload: string - var hashHex: string - var msgHash: WakuMessageHash - var meta: string - - try: - contentTopic = $(pqgetvalue(pqResult, iRow, 0)) - payload = parseHexStr($(pqgetvalue(pqResult, iRow, 1))) - pubSubTopic = $(pqgetvalue(pqResult, iRow, 2)) - version = parseUInt($(pqgetvalue(pqResult, iRow, 3))) - timestamp = parseInt($(pqgetvalue(pqResult, iRow, 4))) - digest = parseHexStr($(pqgetvalue(pqResult, iRow, 5))) - hashHex = parseHexStr($(pqgetvalue(pqResult, iRow, 6))) - meta = parseHexStr($(pqgetvalue(pqResult, iRow, 7))) - msgHash = fromBytes(hashHex.toOpenArrayByte(0, 31)) - except ValueError: - error "could not parse correctly", error = getCurrentExceptionMsg() - - wakuMessage.timestamp = timestamp - wakuMessage.version = uint32(version) - wakuMessage.contentTopic = contentTopic - wakuMessage.payload = @(payload.toOpenArrayByte(0, payload.high)) - wakuMessage.meta = @(meta.toOpenArrayByte(0, meta.high)) - - outRows.add( - ( - pubSubTopic, - wakuMessage, - @(digest.toOpenArrayByte(0, digest.high)), - timestamp, - msgHash, - ) - ) - -method put*( - s: PostgresDriver, - pubsubTopic: PubsubTopic, - message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, -): Future[ArchiveDriverResult[void]] {.async.} = - let digest = byteutils.toHex(digest.data) - let messageHash = byteutils.toHex(messageHash) - let contentTopic = message.contentTopic - let payload = byteutils.toHex(message.payload) - let version = $message.version - let timestamp = $message.timestamp - let meta = byteutils.toHex(message.meta) - - trace "put PostgresDriver", timestamp = timestamp - - ( - await s.writeConnPool.runStmt( - InsertRowStmtName, - InsertRowStmtDefinition, - @[ - digest, messageHash, contentTopic, payload, pubsubTopic, version, timestamp, - meta, - ], - @[ - int32(digest.len), - int32(messageHash.len), - int32(contentTopic.len), - int32(payload.len), - int32(pubsubTopic.len), - int32(version.len), - int32(timestamp.len), - int32(meta.len), - ], - @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], - ) - ).isOkOr: - return err("could not put msg in messages table: " & $error) - - ## Now add the row to messages_lookup - return await s.writeConnPool.runStmt( - InsertRowInMessagesLookupStmtName, - InsertRowInMessagesLookupStmtDefinition, - @[messageHash, timestamp], - @[int32(messageHash.len), int32(timestamp.len)], - @[int32(0), int32(0)], - ) - -method getAllMessages*( - s: PostgresDriver -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - ## Retrieve all messages from the store. - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) - - ( - await s.readConnPool.pgQuery( - """SELECT contentTopic, - payload, pubsubTopic, version, timestamp, - id, messageHash, meta FROM messages ORDER BY timestamp ASC""", - newSeq[string](0), - rowCallback, - ) - ).isOkOr: - return err("failed in query: " & $error) - - return ok(rows) - -proc getMessagesArbitraryQuery( - s: PostgresDriver, - contentTopic: seq[ContentTopic] = @[], - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hexHashes: seq[string] = @[], - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId: string, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - ## This proc allows to handle atypical queries. We don't use prepared statements for those. - - var query = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages""" - var statements: seq[string] - var args: seq[string] - - if contentTopic.len > 0: - let cstmt = "contentTopic IN (" & "?".repeat(contentTopic.len).join(",") & ")" - statements.add(cstmt) - for t in contentTopic: - args.add(t) - - if hexHashes.len > 0: - let cstmt = "messageHash IN (" & "?".repeat(hexHashes.len).join(",") & ")" - statements.add(cstmt) - for t in hexHashes: - args.add(t) - - if pubsubTopic.isSome(): - statements.add("pubsubTopic = ?") - args.add(pubsubTopic.get()) - - if cursor.isSome(): - let hashHex = byteutils.toHex(cursor.get().hash) - - var entree: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc entreeCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, entree) - - ( - await s.readConnPool.runStmt( - SelectMessageByHashName, - SelectMessageByHashDef, - @[hashHex], - @[int32(hashHex.len)], - @[int32(0)], - entreeCallback, - requestId, - ) - ).isOkOr: - return err("failed to run query with cursor: " & $error) - - if entree.len == 0: - return ok(entree) - - let storetime = entree[0][3] - - let comp = if ascendingOrder: ">" else: "<" - statements.add("(timestamp, messageHash) " & comp & " (?,?)") - args.add($storetime) - args.add(hashHex) - - if startTime.isSome(): - statements.add("timestamp >= ?") - args.add($startTime.get()) - - if endTime.isSome(): - statements.add("timestamp <= ?") - args.add($endTime.get()) - - if statements.len > 0: - query &= " WHERE " & statements.join(" AND ") - - var direction: string - if ascendingOrder: - direction = "ASC" - else: - direction = "DESC" - - query &= " ORDER BY timestamp " & direction & ", messageHash " & direction - - query &= " LIMIT ?" - args.add($maxPageSize) - - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) - - (await s.readConnPool.pgQuery(query, args, rowCallback, requestId)).isOkOr: - return err("failed to run query: " & $error) - - return ok(rows) - -proc getMessagesV2ArbitraryQuery( - s: PostgresDriver, - contentTopic: seq[ContentTopic] = @[], - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId: string, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = - ## This proc allows to handle atypical queries. We don't use prepared statements for those. - - var query = - """SELECT contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta FROM messages""" - var statements: seq[string] - var args: seq[string] - - if contentTopic.len > 0: - let cstmt = "contentTopic IN (" & "?".repeat(contentTopic.len).join(",") & ")" - statements.add(cstmt) - for t in contentTopic: - args.add(t) - - if pubsubTopic.isSome(): - statements.add("pubsubTopic = ?") - args.add(pubsubTopic.get()) - - if cursor.isSome(): - let comp = if ascendingOrder: ">" else: "<" - statements.add("(timestamp, id) " & comp & " (?,?)") - args.add($cursor.get().storeTime) - args.add(toHex(cursor.get().digest.data)) - - if startTime.isSome(): - statements.add("timestamp >= ?") - args.add($startTime.get()) - - if endTime.isSome(): - statements.add("timestamp <= ?") - args.add($endTime.get()) - - if statements.len > 0: - query &= " WHERE " & statements.join(" AND ") - - var direction: string - if ascendingOrder: - direction = "ASC" - else: - direction = "DESC" - - query &= " ORDER BY timestamp " & direction & ", id " & direction - - query &= " LIMIT ?" - args.add($maxPageSize) - - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) - - (await s.readConnPool.pgQuery(query, args, rowCallback, requestId)).isOkOr: - return err("failed to run query: " & $error) - - return ok(rows) - -proc getMessagesPreparedStmt( - s: PostgresDriver, - contentTopic: string, - pubsubTopic: PubsubTopic, - cursor = none(ArchiveCursor), - startTime: Timestamp, - endTime: Timestamp, - hashes: string, - maxPageSize = DefaultPageSize, - ascOrder = true, - requestId: string, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - ## This proc aims to run the most typical queries in a more performant way, i.e. by means of - ## prepared statements. - ## - ## contentTopic - string with list of conten topics. e.g: "'ctopic1','ctopic2','ctopic3'" - - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) - - let startTimeStr = $startTime - let endTimeStr = $endTime - let limit = $maxPageSize - - if cursor.isSome(): - let hash = byteutils.toHex(cursor.get().hash) - - var entree: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - - proc entreeCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, entree) - - ( - await s.readConnPool.runStmt( - SelectMessageByHashName, - SelectMessageByHashDef, - @[hash], - @[int32(hash.len)], - @[int32(0)], - entreeCallback, - requestId, - ) - ).isOkOr: - return err("failed to run query with cursor: " & $error) - - if entree.len == 0: - return ok(entree) - - let timestamp = $entree[0][3] - - var stmtName = - if ascOrder: SelectWithCursorAscStmtName else: SelectWithCursorDescStmtName - var stmtDef = - if ascOrder: SelectWithCursorAscStmtDef else: SelectWithCursorDescStmtDef - - ( - await s.readConnPool.runStmt( - stmtName, - stmtDef, - @[ - contentTopic, hashes, pubsubTopic, timestamp, hash, startTimeStr, endTimeStr, - limit, - ], - @[ - int32(contentTopic.len), - int32(hashes.len), - int32(pubsubTopic.len), - int32(timestamp.len), - int32(hash.len), - int32(startTimeStr.len), - int32(endTimeStr.len), - int32(limit.len), - ], - @[ - int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0) - ], - rowCallback, - requestId, - ) - ).isOkOr: - return err("failed to run query with cursor: " & $error) - else: - var stmtName = - if ascOrder: SelectNoCursorAscStmtName else: SelectNoCursorDescStmtName - var stmtDef = if ascOrder: SelectNoCursorAscStmtDef else: SelectNoCursorDescStmtDef - - ( - await s.readConnPool.runStmt( - stmtName, - stmtDef, - @[contentTopic, hashes, pubsubTopic, startTimeStr, endTimeStr, limit], - @[ - int32(contentTopic.len), - int32(hashes.len), - int32(pubsubTopic.len), - int32(startTimeStr.len), - int32(endTimeStr.len), - int32(limit.len), - ], - @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], - rowCallback, - requestId, - ) - ).isOkOr: - return err("failed to run query without cursor: " & $error) - - return ok(rows) - -proc getMessagesV2PreparedStmt( - s: PostgresDriver, - contentTopic: string, - pubsubTopic: PubsubTopic, - cursor = none(ArchiveCursor), - startTime: Timestamp, - endTime: Timestamp, - maxPageSize = DefaultPageSize, - ascOrder = true, - requestId: string, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = - ## This proc aims to run the most typical queries in a more performant way, i.e. by means of - ## prepared statements. - ## - ## contentTopic - string with list of conten topics. e.g: "'ctopic1','ctopic2','ctopic3'" - - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) - - let startTimeStr = $startTime - let endTimeStr = $endTime - let limit = $maxPageSize - - if cursor.isSome(): - var stmtName = - if ascOrder: SelectWithCursorV2AscStmtName else: SelectWithCursorV2DescStmtName - var stmtDef = - if ascOrder: SelectWithCursorV2AscStmtDef else: SelectWithCursorV2DescStmtDef - - let digest = byteutils.toHex(cursor.get().digest.data) - let timestamp = $cursor.get().storeTime - - ( - await s.readConnPool.runStmt( - stmtName, - stmtDef, - @[contentTopic, pubsubTopic, timestamp, digest, startTimeStr, endTimeStr, limit], - @[ - int32(contentTopic.len), - int32(pubsubTopic.len), - int32(timestamp.len), - int32(digest.len), - int32(startTimeStr.len), - int32(endTimeStr.len), - int32(limit.len), - ], - @[int32(0), int32(0), int32(0), int32(0), int32(0), int32(0), int32(0)], - rowCallback, - requestId, - ) - ).isOkOr: - return err("failed to run query with cursor: " & $error) - else: - var stmtName = - if ascOrder: SelectNoCursorV2AscStmtName else: SelectNoCursorV2DescStmtName - var stmtDef = - if ascOrder: SelectNoCursorV2AscStmtDef else: SelectNoCursorV2DescStmtDef - - ( - await s.readConnPool.runStmt( - stmtName, - stmtDef, - @[contentTopic, pubsubTopic, startTimeStr, endTimeStr, limit], - @[ - int32(contentTopic.len), - int32(pubsubTopic.len), - int32(startTimeStr.len), - int32(endTimeStr.len), - int32(limit.len), - ], - @[int32(0), int32(0), int32(0), int32(0), int32(0)], - rowCallback, - requestId, - ) - ).isOkOr: - return err("failed to run query without cursor: " & $error) - - return ok(rows) - -proc getMessagesByMessageHashes( - s: PostgresDriver, hashes: string, maxPageSize: uint, requestId: string -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - ## Retrieves information only filtering by a given messageHashes list. - ## This proc levarages on the messages_lookup table to have better query performance - ## and only query the desired partitions in the partitioned messages table - var query = - fmt""" - WITH min_timestamp AS ( - SELECT MIN(timestamp) AS min_ts - FROM messages_lookup - WHERE messagehash IN ( - {hashes} - ) - ) - SELECT contentTopic, payload, pubsubTopic, version, m.timestamp, id, m.messageHash, meta - FROM messages m - INNER JOIN - messages_lookup l - ON - m.timestamp = l.timestamp - AND m.messagehash = l.messagehash - WHERE - l.timestamp >= (SELECT min_ts FROM min_timestamp) - AND l.messagehash IN ( - {hashes} - ) - ORDER BY - m.timestamp DESC, - m.messagehash DESC - LIMIT {maxPageSize}; - """ - - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc rowCallback(pqResult: ptr PGresult) = - rowCallbackImpl(pqResult, rows) - - ( - await s.readConnPool.pgQuery( - query = query, rowCallback = rowCallback, requestId = requestId - ) - ).isOkOr: - return err("failed to run query: " & $error) - - return ok(rows) - -method getMessages*( - s: PostgresDriver, - includeData = true, - contentTopicSeq = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hashes = newSeq[WakuMessageHash](0), - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId = "", -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - let hexHashes = hashes.mapIt(toHex(it)) - - if cursor.isNone() and pubsubTopic.isNone() and contentTopicSeq.len == 0 and - startTime.isNone() and endTime.isNone() and hexHashes.len > 0: - return await s.getMessagesByMessageHashes( - "'" & hexHashes.join("','") & "'", maxPageSize, requestId - ) - - if contentTopicSeq.len == 1 and hexHashes.len == 1 and pubsubTopic.isSome() and - startTime.isSome() and endTime.isSome(): - ## Considered the most common query. Therefore, we use prepared statements to optimize it. - return await s.getMessagesPreparedStmt( - contentTopicSeq.join(","), - PubsubTopic(pubsubTopic.get()), - cursor, - startTime.get(), - endTime.get(), - hexHashes.join(","), - maxPageSize, - ascendingOrder, - requestId, - ) - else: - ## We will run atypical query. In this case we don't use prepared statemets - return await s.getMessagesArbitraryQuery( - contentTopicSeq, pubsubTopic, cursor, startTime, endTime, hexHashes, maxPageSize, - ascendingOrder, requestId, - ) - -method getMessagesV2*( - s: PostgresDriver, - contentTopicSeq = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId: string, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = - if contentTopicSeq.len == 1 and pubsubTopic.isSome() and startTime.isSome() and - endTime.isSome(): - ## Considered the most common query. Therefore, we use prepared statements to optimize it. - return await s.getMessagesV2PreparedStmt( - contentTopicSeq.join(","), - PubsubTopic(pubsubTopic.get()), - cursor, - startTime.get(), - endTime.get(), - maxPageSize, - ascendingOrder, - requestId, - ) - else: - ## We will run atypical query. In this case we don't use prepared statemets - return await s.getMessagesV2ArbitraryQuery( - contentTopicSeq, pubsubTopic, cursor, startTime, endTime, maxPageSize, - ascendingOrder, requestId, - ) - -proc getStr( - s: PostgresDriver, query: string -): Future[ArchiveDriverResult[string]] {.async.} = - # Performs a query that is expected to return a single string - - var ret: string - proc rowCallback(pqResult: ptr PGresult) = - if pqResult.pqnfields() != 1: - error "Wrong number of fields in getStr" - return - - if pqResult.pqNtuples() != 1: - error "Wrong number of rows in getStr" - return - - ret = $(pqgetvalue(pqResult, 0, 0)) - - (await s.readConnPool.pgQuery(query, newSeq[string](0), rowCallback)).isOkOr: - return err("failed in getRow: " & $error) - - return ok(ret) - -proc getInt( - s: PostgresDriver, query: string -): Future[ArchiveDriverResult[int64]] {.async.} = - # Performs a query that is expected to return a single numeric value (int64) - - var retInt = 0'i64 - let str = (await s.getStr(query)).valueOr: - return err("could not get str in getInt: " & $error) - - try: - retInt = parseInt(str) - except ValueError: - return err( - "exception in getInt, parseInt, str: " & str & " query: " & query & " exception: " & - getCurrentExceptionMsg() - ) - - return ok(retInt) - -method getDatabaseSize*( - s: PostgresDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - let intRes = (await s.getInt("SELECT pg_database_size(current_database())")).valueOr: - return err("error in getDatabaseSize: " & error) - - let databaseSize: int64 = int64(intRes) - return ok(databaseSize) - -method getMessagesCount*( - s: PostgresDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - let intRes = (await s.getInt("SELECT COUNT(1) FROM messages")).valueOr: - return err("error in getMessagesCount: " & error) - - return ok(intRes) - -method getOldestMessageTimestamp*( - s: PostgresDriver -): Future[ArchiveDriverResult[Timestamp]] {.async.} = - return err("not implemented because legacy will get deprecated") - -method getNewestMessageTimestamp*( - s: PostgresDriver -): Future[ArchiveDriverResult[Timestamp]] {.async.} = - let intRes = (await s.getInt("SELECT MAX(timestamp) FROM messages")).valueOr: - return err("error in getNewestMessageTimestamp: " & error) - - return ok(Timestamp(intRes)) - -method deleteOldestMessagesNotWithinLimit*( - s: PostgresDriver, limit: int -): Future[ArchiveDriverResult[void]] {.async.} = - ## Will be completely removed when deprecating store legacy - # let execRes = await s.writeConnPool.pgQuery( - # """DELETE FROM messages WHERE id NOT IN - # ( - # SELECT id FROM messages ORDER BY timestamp DESC LIMIT ? - # );""", - # @[$limit], - # ) - # if execRes.isErr(): - # return err("error in deleteOldestMessagesNotWithinLimit: " & execRes.error) - - return ok() - -method close*(s: PostgresDriver): Future[ArchiveDriverResult[void]] {.async.} = - ## Close the database connection - let writeCloseRes = await s.writeConnPool.close() - let readCloseRes = await s.readConnPool.close() - - writeCloseRes.isOkOr: - return err("error closing write pool: " & $error) - - readCloseRes.isOkOr: - return err("error closing read pool: " & $error) - - return ok() - -proc sleep*( - s: PostgresDriver, seconds: int -): Future[ArchiveDriverResult[void]] {.async.} = - # This is for testing purposes only. It is aimed to test the proper - # implementation of asynchronous requests. It merely triggers a sleep in the - # database for the amount of seconds given as a parameter. - - proc rowCallback(result: ptr PGresult) = - ## We are not interested in any value in this case - discard - - try: - let params = @[$seconds] - (await s.writeConnPool.pgQuery("SELECT pg_sleep(?)", params, rowCallback)).isOkOr: - return err("error in postgres_driver sleep: " & $error) - except DbError: - # This always raises an exception although the sleep works - return err("exception sleeping: " & getCurrentExceptionMsg()) - - return ok() - -proc performWriteQuery*( - s: PostgresDriver, query: string -): Future[ArchiveDriverResult[void]] {.async.} = - ## Performs a query that somehow changes the state of the database - - (await s.writeConnPool.pgQuery(query)).isOkOr: - return err("error in performWriteQuery: " & $error) - - return ok() - -method decreaseDatabaseSize*( - driver: PostgresDriver, targetSizeInBytes: int64, forceRemoval: bool = false -): Future[ArchiveDriverResult[void]] {.async.} = - ## This is completely disabled and only the non-legacy driver - ## will take care of that - # var dbSize = (await driver.getDatabaseSize()).valueOr: - # return err("decreaseDatabaseSize failed to get database size: " & $error) - - # ## database size in bytes - # var totalSizeOfDB: int64 = int64(dbSize) - - # if totalSizeOfDB <= targetSizeInBytes: - # return ok() - - # info "start reducing database size", - # targetSize = $targetSizeInBytes, currentSize = $totalSizeOfDB - - # while totalSizeOfDB > targetSizeInBytes and driver.containsAnyPartition(): - # (await driver.removeOldestPartition(forceRemoval)).isOkOr: - # return err( - # "decreaseDatabaseSize inside loop failed to remove oldest partition: " & $error - # ) - - # dbSize = (await driver.getDatabaseSize()).valueOr: - # return - # err("decreaseDatabaseSize inside loop failed to get database size: " & $error) - - # let newCurrentSize = int64(dbSize) - # if newCurrentSize == totalSizeOfDB: - # return err("the previous partition removal didn't clear database size") - - # totalSizeOfDB = newCurrentSize - - # info "reducing database size", - # targetSize = $targetSizeInBytes, newCurrentSize = $totalSizeOfDB - - return ok() - -method existsTable*( - s: PostgresDriver, tableName: string -): Future[ArchiveDriverResult[bool]] {.async.} = - let query: string = - fmt""" - SELECT EXISTS ( - SELECT FROM - pg_tables - WHERE - tablename = '{tableName}' - ); - """ - - var exists: string - proc rowCallback(pqResult: ptr PGresult) = - if pqResult.pqnfields() != 1: - error "Wrong number of fields in existsTable" - return - - if pqResult.pqNtuples() != 1: - error "Wrong number of rows in existsTable" - return - - exists = $(pqgetvalue(pqResult, 0, 0)) - - (await s.readConnPool.pgQuery(query, newSeq[string](0), rowCallback)).isOkOr: - return err("existsTable failed in getRow: " & $error) - - return ok(exists == "t") - -proc getCurrentVersion*( - s: PostgresDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - let existsVersionTable = (await s.existsTable("version")).valueOr: - return err("error in getCurrentVersion-existsTable: " & $error) - - if not existsVersionTable: - return ok(0) - - let res = (await s.getInt(fmt"SELECT version FROM version")).valueOr: - return err("error in getMessagesCount: " & $error) - - return ok(res) - -method deleteMessagesOlderThanTimestamp*( - s: PostgresDriver, tsNanoSec: Timestamp -): Future[ArchiveDriverResult[void]] {.async.} = - ## First of all, let's remove the older partitions so that we can reduce - ## the database size. - # (await s.removePartitionsOlderThan(tsNanoSec)).isOkOr: - # return err("error while removing older partitions: " & $error) - - # ( - # await s.writeConnPool.pgQuery( - # "DELETE FROM messages WHERE timestamp < " & $tsNanoSec - # ) - # ).isOkOr: - # return err("error in deleteMessagesOlderThanTimestamp: " & $error) - - return ok() diff --git a/waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim b/waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim deleted file mode 100644 index 23678538e..000000000 --- a/waku/waku_archive_legacy/driver/postgres_driver/postgres_healthcheck.nim +++ /dev/null @@ -1,37 +0,0 @@ -{.push raises: [].} - -import chronos, chronicles, results -import ../../../common/databases/db_postgres, ../../../common/error_handling - -## Simple query to validate that the postgres is working and attending requests -const HealthCheckQuery = "SELECT version();" -const CheckConnectivityInterval = 60.seconds -const MaxNumTrials = 20 -const TrialInterval = 1.seconds - -proc checkConnectivity*( - connPool: PgAsyncPool, onFatalErrorAction: OnFatalErrorHandler -) {.async.} = - while true: - (await connPool.pgQuery(HealthCheckQuery)).isOkOr: - ## The connection failed once. Let's try reconnecting for a while. - ## Notice that the 'pgQuery' proc tries to establish a new connection. - - block errorBlock: - ## Force close all the opened connections. No need to close gracefully. - (await connPool.resetConnPool()).isOkOr: - onFatalErrorAction("checkConnectivity legacy resetConnPool error: " & error) - - var numTrial = 0 - while numTrial < MaxNumTrials: - (await connPool.pgQuery(HealthCheckQuery)).isErrOr: - ## Connection resumed. Let's go back to the normal healthcheck. - break errorBlock - - await sleepAsync(TrialInterval) - numTrial.inc() - - ## The connection couldn't be resumed. Let's inform the upper layers. - onFatalErrorAction("postgres legacy health check error: " & error) - - await sleepAsync(CheckConnectivityInterval) diff --git a/waku/waku_archive_legacy/driver/queue_driver.nim b/waku/waku_archive_legacy/driver/queue_driver.nim deleted file mode 100644 index 1ea8a29d3..000000000 --- a/waku/waku_archive_legacy/driver/queue_driver.nim +++ /dev/null @@ -1,8 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import ./queue_driver/queue_driver, ./queue_driver/index - -export queue_driver, index diff --git a/waku/waku_archive_legacy/driver/queue_driver/index.nim b/waku/waku_archive_legacy/driver/queue_driver/index.nim deleted file mode 100644 index 2328870d0..000000000 --- a/waku/waku_archive_legacy/driver/queue_driver/index.nim +++ /dev/null @@ -1,91 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import nimcrypto/sha2 -import ../../../waku_core, ../../common - -type Index* = object - ## This type contains the description of an Index used in the pagination of WakuMessages - pubsubTopic*: string - senderTime*: Timestamp # the time at which the message is generated - receiverTime*: Timestamp - digest*: MessageDigest # calculated over payload and content topic - hash*: WakuMessageHash - -proc compute*( - T: type Index, msg: WakuMessage, receivedTime: Timestamp, pubsubTopic: PubsubTopic -): T = - ## Takes a WakuMessage with received timestamp and returns its Index. - let - digest = computeDigest(msg) - senderTime = msg.timestamp - hash = computeMessageHash(pubsubTopic, msg) - - return Index( - pubsubTopic: pubsubTopic, - senderTime: senderTime, - receiverTime: receivedTime, - digest: digest, - hash: hash, - ) - -proc tohistoryCursor*(index: Index): ArchiveCursor = - return ArchiveCursor( - pubsubTopic: index.pubsubTopic, - senderTime: index.senderTime, - storeTime: index.receiverTime, - digest: index.digest, - hash: index.hash, - ) - -proc toIndex*(index: ArchiveCursor): Index = - return Index( - pubsubTopic: index.pubsubTopic, - senderTime: index.senderTime, - receiverTime: index.storeTime, - digest: index.digest, - hash: index.hash, - ) - -proc `==`*(x, y: Index): bool = - ## receiverTime plays no role in index equality - return - ( - (x.senderTime == y.senderTime) and (x.digest == y.digest) and - (x.pubsubTopic == y.pubsubTopic) - ) or (x.hash == y.hash) # this applies to store v3 queries only - -proc cmp*(x, y: Index): int = - ## compares x and y - ## returns 0 if they are equal - ## returns -1 if x < y - ## returns 1 if x > y - ## - ## Default sorting order priority is: - ## 1. senderTimestamp - ## 2. receiverTimestamp (a fallback only if senderTimestamp unset on either side, and all other fields unequal) - ## 3. message digest - ## 4. pubsubTopic - - if x == y: - # Quick exit ensures receiver time does not affect index equality - return 0 - - # Timestamp has a higher priority for comparison - let - # Use receiverTime where senderTime is unset - xTimestamp = if x.senderTime == 0: x.receiverTime else: x.senderTime - yTimestamp = if y.senderTime == 0: y.receiverTime else: y.senderTime - - let timecmp = cmp(xTimestamp, yTimestamp) - if timecmp != 0: - return timecmp - - # Continue only when timestamps are equal - let digestcmp = cmp(x.digest.data, y.digest.data) - if digestcmp != 0: - return digestcmp - - return cmp(x.pubsubTopic, y.pubsubTopic) diff --git a/waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim b/waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim deleted file mode 100644 index 530a84034..000000000 --- a/waku/waku_archive_legacy/driver/queue_driver/queue_driver.nim +++ /dev/null @@ -1,363 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import std/options, results, stew/sorted_set, chronicles, chronos -import ../../../waku_core, ../../common, ../../driver, ./index - -logScope: - topics = "waku archive queue_store" - -const QueueDriverDefaultMaxCapacity* = 25_000 - -type - QueryFilterMatcher = - proc(index: Index, msg: WakuMessage): bool {.gcsafe, raises: [], closure.} - - QueueDriver* = ref object of ArchiveDriver - ## Bounded repository for indexed messages - ## - ## The store queue will keep messages up to its - ## configured capacity. As soon as this capacity - ## is reached and a new message is added, the oldest - ## item will be removed to make space for the new one. - ## This implies both a `delete` and `add` operation - ## for new items. - - # TODO: a circular/ring buffer may be a more efficient implementation - items: SortedSet[Index, WakuMessage] # sorted set of stored messages - capacity: int # Maximum amount of messages to keep - - QueueDriverErrorKind {.pure.} = enum - INVALID_CURSOR - - QueueDriverGetPageResult = Result[seq[ArchiveRow], QueueDriverErrorKind] - -proc `$`(error: QueueDriverErrorKind): string = - case error - of INVALID_CURSOR: "invalid_cursor" - -### Helpers - -proc walkToCursor( - w: SortedSetWalkRef[Index, WakuMessage], startCursor: Index, forward: bool -): SortedSetResult[Index, WakuMessage] = - ## Walk to util we find the cursor - ## TODO: Improve performance here with a binary/tree search - - var nextItem = - if forward: - w.first() - else: - w.last() - - ## Fast forward until we reach the startCursor - while nextItem.isOk(): - if nextItem.value.key == startCursor: - break - - # Not yet at cursor. Continue advancing - nextItem = - if forward: - w.next() - else: - w.prev() - - return nextItem - -#### API - -proc new*(T: type QueueDriver, capacity: int = QueueDriverDefaultMaxCapacity): T = - var items = SortedSet[Index, WakuMessage].init() - return QueueDriver(items: items, capacity: capacity) - -proc contains*(driver: QueueDriver, index: Index): bool = - ## Return `true` if the store queue already contains the `index`, `false` otherwise. - return driver.items.eq(index).isOk() - -proc len*(driver: QueueDriver): int {.noSideEffect.} = - return driver.items.len - -proc getPage( - driver: QueueDriver, - pageSize: uint = 0, - forward: bool = true, - cursor: Option[Index] = none(Index), - predicate: QueryFilterMatcher = nil, -): QueueDriverGetPageResult {.raises: [].} = - ## Populate a single page in forward direction - ## Start at the `startCursor` (exclusive), or first entry (inclusive) if not defined. - ## Page size must not exceed `maxPageSize` - ## Each entry must match the `pred` - var outSeq: seq[ArchiveRow] - - var w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) - defer: - w.destroy() - - var currentEntry: SortedSetResult[Index, WakuMessage] - - # Find starting entry - if cursor.isSome(): - w.walkToCursor(cursor.get(), forward).isOkOr: - return err(QueueDriverErrorKind.INVALID_CURSOR) - - # Advance walker once more - currentEntry = - if forward: - w.next() - else: - w.prev() - else: - # Start from the beginning of the queue - currentEntry = - if forward: - w.first() - else: - w.last() - - trace "Starting page query", currentEntry = currentEntry - - ## This loop walks forward over the queue: - ## 1. from the given cursor (or first/last entry, if not provided) - ## 2. adds entries matching the predicate function to output page - ## 3. until either the end of the queue or maxPageSize is reached - var numberOfItems: uint = 0 - while currentEntry.isOk() and numberOfItems < pageSize: - trace "Continuing page query", - currentEntry = currentEntry, numberOfItems = numberOfItems - - let - key = currentEntry.value.key - data = currentEntry.value.data - - if predicate.isNil() or predicate(key, data): - numberOfItems += 1 - - outSeq.add( - (key.pubsubTopic, data, @(key.digest.data), key.receiverTime, key.hash) - ) - - currentEntry = - if forward: - w.next() - else: - w.prev() - - trace "Successfully retrieved page", len = outSeq.len - - return ok(outSeq) - -## --- SortedSet accessors --- - -iterator fwdIterator*(driver: QueueDriver): (Index, WakuMessage) = - ## Forward iterator over the entire store queue - var - w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) - res = w.first() - - while res.isOk(): - yield (res.value.key, res.value.data) - res = w.next() - - w.destroy() - -iterator bwdIterator*(driver: QueueDriver): (Index, WakuMessage) = - ## Backwards iterator over the entire store queue - var - w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) - res = w.last() - - while res.isOk(): - yield (res.value.key, res.value.data) - res = w.prev() - - w.destroy() - -proc first*(driver: QueueDriver): ArchiveDriverResult[Index] = - var - w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) - res = w.first() - w.destroy() - - res.isOkOr: - return err("Not found") - - return ok(res.value.key) - -proc last*(driver: QueueDriver): ArchiveDriverResult[Index] = - var - w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) - res = w.last() - w.destroy() - - res.isOkOr: - return err("Not found") - - return ok(res.value.key) - -## --- Queue API --- - -proc add*( - driver: QueueDriver, index: Index, msg: WakuMessage -): ArchiveDriverResult[void] = - ## Add a message to the queue - ## - ## If we're at capacity, we will be removing, the oldest (first) item - if driver.contains(index): - trace "could not add item to store queue. Index already exists", index = index - return err("duplicate") - - # TODO: the below delete block can be removed if we convert to circular buffer - if driver.items.len >= driver.capacity: - var - w = SortedSetWalkRef[Index, WakuMessage].init(driver.items) - firstItem = w.first - - if cmp(index, firstItem.value.key) < 0: - # When at capacity, we won't add if message index is smaller (older) than our oldest item - w.destroy # Clean up walker - return err("too_old") - - discard driver.items.delete(firstItem.value.key) - w.destroy # better to destroy walker after a delete operation - - driver.items.insert(index).value.data = msg - - return ok() - -method put*( - driver: QueueDriver, - pubsubTopic: PubsubTopic, - message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, -): Future[ArchiveDriverResult[void]] {.async.} = - let index = Index( - pubsubTopic: pubsubTopic, - senderTime: message.timestamp, - receiverTime: receivedTime, - digest: digest, - hash: messageHash, - ) - - return driver.add(index, message) - -method getAllMessages*( - driver: QueueDriver -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - # TODO: Implement this message_store method - return err("interface method not implemented") - -method existsTable*( - driver: QueueDriver, tableName: string -): Future[ArchiveDriverResult[bool]] {.async.} = - return err("interface method not implemented") - -method getMessages*( - driver: QueueDriver, - includeData = true, - contentTopic: seq[ContentTopic] = @[], - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hashes: seq[WakuMessageHash] = @[], - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId = "", -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - let cursor = cursor.map(toIndex) - - let matchesQuery: QueryFilterMatcher = - func (index: Index, msg: WakuMessage): bool = - if pubsubTopic.isSome() and index.pubsubTopic != pubsubTopic.get(): - return false - - if contentTopic.len > 0 and msg.contentTopic notin contentTopic: - return false - - if startTime.isSome() and msg.timestamp < startTime.get(): - return false - - if endTime.isSome() and msg.timestamp > endTime.get(): - return false - - if hashes.len > 0 and index.hash notin hashes: - return false - - return true - - var pageRes: QueueDriverGetPageResult - try: - pageRes = driver.getPage(maxPageSize, ascendingOrder, cursor, matchesQuery) - except CatchableError, Exception: - return err(getCurrentExceptionMsg()) - - pageRes.isOkOr: - return err($error) - - return ok(pageRes.value) - -method getMessagesCount*( - driver: QueueDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - return ok(int64(driver.len())) - -method getPagesCount*( - driver: QueueDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - return ok(int64(driver.len())) - -method getPagesSize*( - driver: QueueDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - return ok(int64(driver.len())) - -method getDatabaseSize*( - driver: QueueDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - return ok(int64(driver.len())) - -method performVacuum*( - driver: QueueDriver -): Future[ArchiveDriverResult[void]] {.async.} = - return err("interface method not implemented") - -method getOldestMessageTimestamp*( - driver: QueueDriver -): Future[ArchiveDriverResult[Timestamp]] {.async.} = - return driver.first().map( - proc(index: Index): Timestamp = - index.receiverTime - ) - -method getNewestMessageTimestamp*( - driver: QueueDriver -): Future[ArchiveDriverResult[Timestamp]] {.async.} = - return driver.last().map( - proc(index: Index): Timestamp = - index.receiverTime - ) - -method deleteMessagesOlderThanTimestamp*( - driver: QueueDriver, ts: Timestamp -): Future[ArchiveDriverResult[void]] {.async.} = - # TODO: Implement this message_store method - return err("interface method not implemented") - -method deleteOldestMessagesNotWithinLimit*( - driver: QueueDriver, limit: int -): Future[ArchiveDriverResult[void]] {.async.} = - # TODO: Implement this message_store method - return err("interface method not implemented") - -method decreaseDatabaseSize*( - driver: QueueDriver, targetSizeInBytes: int64, forceRemoval: bool = false -): Future[ArchiveDriverResult[void]] {.async.} = - return err("interface method not implemented") - -method close*(driver: QueueDriver): Future[ArchiveDriverResult[void]] {.async.} = - return ok() diff --git a/waku/waku_archive_legacy/driver/sqlite_driver.nim b/waku/waku_archive_legacy/driver/sqlite_driver.nim deleted file mode 100644 index 027e00488..000000000 --- a/waku/waku_archive_legacy/driver/sqlite_driver.nim +++ /dev/null @@ -1,8 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import ./sqlite_driver/sqlite_driver - -export sqlite_driver diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/cursor.nim b/waku/waku_archive_legacy/driver/sqlite_driver/cursor.nim deleted file mode 100644 index 9729f0ff7..000000000 --- a/waku/waku_archive_legacy/driver/sqlite_driver/cursor.nim +++ /dev/null @@ -1,11 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import ../../../waku_core, ../../common - -type DbCursor* = (Timestamp, seq[byte], PubsubTopic) - -proc toDbCursor*(c: ArchiveCursor): DbCursor = - (c.storeTime, @(c.digest.data), c.pubsubTopic) diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim b/waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim deleted file mode 100644 index 3d8905e7e..000000000 --- a/waku/waku_archive_legacy/driver/sqlite_driver/migrations.nim +++ /dev/null @@ -1,71 +0,0 @@ -{.push raises: [].} - -import - std/[tables, strutils, os], results, chronicles, sqlite3_abi # sqlite3_column_int64 -import ../../../common/databases/db_sqlite, ../../../common/databases/common - -logScope: - topics = "waku archive migration" - -const SchemaVersion* = 9 # increase this when there is an update in the database schema - -template projectRoot(): string = - currentSourcePath.rsplit(DirSep, 1)[0] / ".." / ".." / ".." / ".." - -const MessageStoreMigrationPath: string = projectRoot / "migrations" / "message_store" - -proc isSchemaVersion7*(db: SqliteDatabase): DatabaseResult[bool] = - ## Temporary proc created to analyse when the table actually belongs to the SchemaVersion 7. - ## - ## During many nwaku versions, 0.14.0 until 0.18.0, the SchemaVersion wasn't set or checked. - ## Docker `nwaku` nodes that start working from these versions, 0.14.0 until 0.18.0, they started - ## with this discrepancy: `user_version`== 0 (not set) but Message table with SchemaVersion 7. - ## - ## We found issues where `user_version` (SchemaVersion) was set to 0 in the database even though - ## its scheme structure reflected SchemaVersion 7. In those cases, when `nwaku` re-started to - ## apply the migration scripts (in 0.19.0) the node didn't start properly because it tried to - ## migrate a database that already had the Schema structure #7, so it failed when changing the PK. - ## - ## TODO: This was added in version 0.20.0. We might remove this in version 0.30.0, as we - ## could consider that many users use +0.20.0. - - var pkColumns = newSeq[string]() - proc queryRowCallback(s: ptr sqlite3_stmt) = - let colName = cstring sqlite3_column_text(s, 0) - pkColumns.add($colName) - - let query = - """SELECT l.name FROM pragma_table_info("Message") as l WHERE l.pk != 0;""" - db.query(query, queryRowCallback).isOkOr: - return err("failed to determine the current SchemaVersion: " & $error) - - if pkColumns == @["pubsubTopic", "id", "storedAt"]: - return ok(true) - else: - info "Not considered schema version 7" - return ok(false) - -proc migrate*(db: SqliteDatabase, targetVersion = SchemaVersion): DatabaseResult[void] = - ## Compares the `user_version` of the sqlite database with the provided `targetVersion`, then - ## it runs migration scripts if the `user_version` is outdated. The `migrationScriptsDir` path - ## points to the directory holding the migrations scripts once the db is updated, it sets the - ## `user_version` to the `tragetVersion`. - ## - ## If not `targetVersion` is provided, it defaults to `SchemaVersion`. - ## - ## NOTE: Down migration it is not currently supported - info "starting message store's sqlite database migration" - - let userVersion = ?db.getUserVersion() - let isSchemaVersion7 = ?db.isSchemaVersion7() - - if userVersion == 0'i64 and isSchemaVersion7: - info "We found user_version 0 but the database schema reflects the user_version 7" - ## Force the correct schema version - ?db.setUserVersion(7) - - migrate(db, targetVersion, migrationsScriptsDir = MessageStoreMigrationPath).isOkOr: - return err("failed to execute migration scripts: " & error) - - info "finished message store's sqlite database migration" - return ok() diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim b/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim deleted file mode 100644 index 4590a8df1..000000000 --- a/waku/waku_archive_legacy/driver/sqlite_driver/queries.nim +++ /dev/null @@ -1,729 +0,0 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import std/[options, sequtils], stew/byteutils, sqlite3_abi, results -import - ../../../common/databases/db_sqlite, - ../../../common/databases/common, - ../../../waku_core, - ./cursor - -const DbTable = "Message" - -type SqlQueryStr = string - -### SQLite column helper methods - -proc queryRowWakuMessageCallback( - s: ptr sqlite3_stmt, - contentTopicCol, payloadCol, versionCol, senderTimestampCol, metaCol: cint, -): WakuMessage = - let - topic = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, contentTopicCol)) - topicLength = sqlite3_column_bytes(s, contentTopicCol) - contentTopic = string.fromBytes(@(toOpenArray(topic, 0, topicLength - 1))) - - p = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, payloadCol)) - m = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, metaCol)) - - payloadLength = sqlite3_column_bytes(s, payloadCol) - metaLength = sqlite3_column_bytes(s, metaCol) - payload = @(toOpenArray(p, 0, payloadLength - 1)) - version = sqlite3_column_int64(s, versionCol) - senderTimestamp = sqlite3_column_int64(s, senderTimestampCol) - meta = @(toOpenArray(m, 0, metaLength - 1)) - - return WakuMessage( - contentTopic: ContentTopic(contentTopic), - payload: payload, - version: uint32(version), - timestamp: Timestamp(senderTimestamp), - meta: meta, - ) - -proc queryRowReceiverTimestampCallback( - s: ptr sqlite3_stmt, storedAtCol: cint -): Timestamp = - let storedAt = sqlite3_column_int64(s, storedAtCol) - return Timestamp(storedAt) - -proc queryRowPubsubTopicCallback( - s: ptr sqlite3_stmt, pubsubTopicCol: cint -): PubsubTopic = - let - pubsubTopicPointer = - cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, pubsubTopicCol)) - pubsubTopicLength = sqlite3_column_bytes(s, pubsubTopicCol) - pubsubTopic = - string.fromBytes(@(toOpenArray(pubsubTopicPointer, 0, pubsubTopicLength - 1))) - - return pubsubTopic - -proc queryRowDigestCallback(s: ptr sqlite3_stmt, digestCol: cint): seq[byte] = - let - digestPointer = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, digestCol)) - digestLength = sqlite3_column_bytes(s, digestCol) - digest = @(toOpenArray(digestPointer, 0, digestLength - 1)) - - return digest - -proc queryRowWakuMessageHashCallback( - s: ptr sqlite3_stmt, hashCol: cint -): WakuMessageHash = - let - hashPointer = cast[ptr UncheckedArray[byte]](sqlite3_column_blob(s, hashCol)) - hashLength = sqlite3_column_bytes(s, hashCol) - hash = fromBytes(toOpenArray(hashPointer, 0, hashLength - 1)) - - return hash - -### SQLite queries - -## Create table - -proc createTableQuery(table: string): SqlQueryStr = - "CREATE TABLE IF NOT EXISTS " & table & " (" & " pubsubTopic BLOB NOT NULL," & - " contentTopic BLOB NOT NULL," & " payload BLOB," & " version INTEGER NOT NULL," & - " timestamp INTEGER NOT NULL," & " id BLOB," & " messageHash BLOB," & - " storedAt INTEGER NOT NULL," & " meta BLOB," & - " CONSTRAINT messageIndex PRIMARY KEY (messageHash)" & ") WITHOUT ROWID;" - -proc createTable*(db: SqliteDatabase): DatabaseResult[void] = - let query = createTableQuery(DbTable) - discard ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) - return ok() - -## Create indices - -proc createOldestMessageTimestampIndexQuery(table: string): SqlQueryStr = - "CREATE INDEX IF NOT EXISTS i_ts ON " & table & " (storedAt);" - -proc createOldestMessageTimestampIndex*(db: SqliteDatabase): DatabaseResult[void] = - let query = createOldestMessageTimestampIndexQuery(DbTable) - discard ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) - return ok() - -proc createHistoryQueryIndexQuery(table: string): SqlQueryStr = - "CREATE INDEX IF NOT EXISTS i_query ON " & table & - " (contentTopic, pubsubTopic, storedAt, id);" - -proc createHistoryQueryIndex*(db: SqliteDatabase): DatabaseResult[void] = - let query = createHistoryQueryIndexQuery(DbTable) - discard ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) - return ok() - -## Insert message -type InsertMessageParams* = ( - seq[byte], - seq[byte], - Timestamp, - seq[byte], - seq[byte], - seq[byte], - int64, - Timestamp, - seq[byte], -) - -proc insertMessageQuery(table: string): SqlQueryStr = - return - "INSERT INTO " & table & - "(id, messageHash, storedAt, contentTopic, payload, pubsubTopic, version, timestamp, meta)" & - " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);" - -proc prepareInsertMessageStmt*( - db: SqliteDatabase -): SqliteStmt[InsertMessageParams, void] = - let query = insertMessageQuery(DbTable) - return - db.prepareStmt(query, InsertMessageParams, void).expect("this is a valid statement") - -## Count table messages - -proc countMessagesQuery(table: string): SqlQueryStr = - return "SELECT COUNT(*) FROM " & table - -proc getMessageCount*(db: SqliteDatabase): DatabaseResult[int64] = - var count: int64 - proc queryRowCallback(s: ptr sqlite3_stmt) = - count = sqlite3_column_int64(s, 0) - - let query = countMessagesQuery(DbTable) - db.query(query, queryRowCallback).isOkOr: - return err("failed to count number of messages in the database") - - return ok(count) - -## Get oldest message receiver timestamp - -proc selectOldestMessageTimestampQuery(table: string): SqlQueryStr = - return "SELECT MIN(storedAt) FROM " & table - -proc selectOldestReceiverTimestamp*( - db: SqliteDatabase -): DatabaseResult[Timestamp] {.inline.} = - var timestamp: Timestamp - proc queryRowCallback(s: ptr sqlite3_stmt) = - timestamp = queryRowReceiverTimestampCallback(s, 0) - - let query = selectOldestMessageTimestampQuery(DbTable) - db.query(query, queryRowCallback).isOkOr: - return err("failed to get the oldest receiver timestamp from the database") - - return ok(timestamp) - -## Get newest message receiver timestamp - -proc selectNewestMessageTimestampQuery(table: string): SqlQueryStr = - return "SELECT MAX(storedAt) FROM " & table - -proc selectNewestReceiverTimestamp*( - db: SqliteDatabase -): DatabaseResult[Timestamp] {.inline.} = - var timestamp: Timestamp - proc queryRowCallback(s: ptr sqlite3_stmt) = - timestamp = queryRowReceiverTimestampCallback(s, 0) - - let query = selectNewestMessageTimestampQuery(DbTable) - db.query(query, queryRowCallback).isOkOr: - return err("failed to get the newest receiver timestamp from the database") - - return ok(timestamp) - -## Delete messages older than timestamp - -proc deleteMessagesOlderThanTimestampQuery(table: string, ts: Timestamp): SqlQueryStr = - return "DELETE FROM " & table & " WHERE storedAt < " & $ts - -proc deleteMessagesOlderThanTimestamp*( - db: SqliteDatabase, ts: int64 -): DatabaseResult[void] = - let query = deleteMessagesOlderThanTimestampQuery(DbTable, ts) - discard ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) - return ok() - -## Delete oldest messages not within limit - -proc deleteOldestMessagesNotWithinLimitQuery(table: string, limit: int): SqlQueryStr = - return - "DELETE FROM " & table & " WHERE (storedAt, id, pubsubTopic) NOT IN (" & - " SELECT storedAt, id, pubsubTopic FROM " & table & - " ORDER BY storedAt DESC, id DESC" & " LIMIT " & $limit & ");" - -proc deleteOldestMessagesNotWithinLimit*( - db: SqliteDatabase, limit: int -): DatabaseResult[void] = - # NOTE: The word `limit` here refers the store capacity/maximum number-of-messages allowed limit - let query = deleteOldestMessagesNotWithinLimitQuery(DbTable, limit = limit) - discard ?db.query( - query, - proc(s: ptr sqlite3_stmt) = - discard, - ) - return ok() - -## Select all messages - -proc selectAllMessagesQuery(table: string): SqlQueryStr = - return - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" & - " FROM " & table & " ORDER BY storedAt ASC" - -proc selectAllMessages*( - db: SqliteDatabase -): DatabaseResult[ - seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] -] {.gcsafe.} = - ## Retrieve all messages from the store. - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] - proc queryRowCallback(s: ptr sqlite3_stmt) = - let - pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) - wakuMessage = queryRowWakuMessageCallback( - s, - contentTopicCol = 1, - payloadCol = 2, - versionCol = 4, - senderTimestampCol = 5, - metaCol = 8, - ) - digest = queryRowDigestCallback(s, digestCol = 6) - storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) - hash = queryRowWakuMessageHashCallback(s, hashCol = 7) - - rows.add((pubsubTopic, wakuMessage, digest, storedAt, hash)) - - let query = selectAllMessagesQuery(DbTable) - discard ?db.query(query, queryRowCallback) - - return ok(rows) - -## Select messages by history query with limit - -proc combineClauses(clauses: varargs[Option[string]]): Option[string] = - let whereSeq = @clauses.filterIt(it.isSome()).mapIt(it.get()) - if whereSeq.len <= 0: - return none(string) - - var where: string = whereSeq[0] - for clause in whereSeq[1 ..^ 1]: - where &= " AND " & clause - return some(where) - -proc whereClausev2( - cursor: bool, - pubsubTopic: Option[PubsubTopic], - contentTopic: seq[ContentTopic], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - ascending: bool, -): Option[string] {.deprecated.} = - let cursorClause = - if cursor: - let comp = if ascending: ">" else: "<" - - some("(storedAt, id) " & comp & " (?, ?)") - else: - none(string) - - let pubsubTopicClause = - if pubsubTopic.isNone(): - none(string) - else: - some("pubsubTopic = (?)") - - let contentTopicClause = - if contentTopic.len <= 0: - none(string) - else: - var where = "contentTopic IN (" - where &= "?" - for _ in 1 ..< contentTopic.len: - where &= ", ?" - where &= ")" - some(where) - - let startTimeClause = - if startTime.isNone(): - none(string) - else: - some("storedAt >= (?)") - - let endTimeClause = - if endTime.isNone(): - none(string) - else: - some("storedAt <= (?)") - - return combineClauses( - cursorClause, pubsubTopicClause, contentTopicClause, startTimeClause, endTimeClause - ) - -proc selectMessagesWithLimitQueryv2( - table: string, where: Option[string], limit: uint, ascending = true, v3 = false -): SqlQueryStr {.deprecated.} = - let order = if ascending: "ASC" else: "DESC" - - var query: string - - query = - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" - query &= " FROM " & table - - if where.isSome(): - query &= " WHERE " & where.get() - - query &= " ORDER BY storedAt " & order & ", id " & order - - query &= " LIMIT " & $limit & ";" - - return query - -proc prepareStmt( - db: SqliteDatabase, stmt: string -): DatabaseResult[SqliteStmt[void, void]] = - var s: RawStmtPtr - checkErr sqlite3_prepare_v2(db.env, stmt, stmt.len.cint, addr s, nil) - return ok(SqliteStmt[void, void](s)) - -proc execSelectMessagesV2WithLimitStmt( - s: SqliteStmt, - cursor: Option[DbCursor], - pubsubTopic: Option[PubsubTopic], - contentTopic: seq[ContentTopic], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - onRowCallback: DataProc, -): DatabaseResult[void] {.deprecated.} = - let s = RawStmtPtr(s) - - # Bind params - var paramIndex = 1 - - if cursor.isSome(): - let (storedAt, id, _) = cursor.get() - checkErr bindParam(s, paramIndex, storedAt) - paramIndex += 1 - checkErr bindParam(s, paramIndex, id) - paramIndex += 1 - - if pubsubTopic.isSome(): - let pubsubTopic = toBytes(pubsubTopic.get()) - checkErr bindParam(s, paramIndex, pubsubTopic) - paramIndex += 1 - - for topic in contentTopic: - checkErr bindParam(s, paramIndex, topic.toBytes()) - paramIndex += 1 - - if startTime.isSome(): - let time = startTime.get() - checkErr bindParam(s, paramIndex, time) - paramIndex += 1 - - if endTime.isSome(): - let time = endTime.get() - checkErr bindParam(s, paramIndex, time) - paramIndex += 1 - - try: - while true: - let v = sqlite3_step(s) - case v - of SQLITE_ROW: - onRowCallback(s) - of SQLITE_DONE: - return ok() - else: - return err($sqlite3_errstr(v)) - except Exception, CatchableError: - # release implicit transaction - discard sqlite3_reset(s) # same return information as step - discard sqlite3_clear_bindings(s) # no errors possible - -proc selectMessagesByHistoryQueryWithLimit*( - db: SqliteDatabase, - contentTopic: seq[ContentTopic], - pubsubTopic: Option[PubsubTopic], - cursor: Option[DbCursor], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - limit: uint, - ascending: bool, -): DatabaseResult[ - seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] -] {.deprecated.} = - var messages: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] = - @[] - - proc queryRowCallback(s: ptr sqlite3_stmt) = - let - pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) - message = queryRowWakuMessageCallback( - s, - contentTopicCol = 1, - payloadCol = 2, - versionCol = 4, - senderTimestampCol = 5, - metaCol = 8, - ) - digest = queryRowDigestCallback(s, digestCol = 6) - storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) - hash = queryRowWakuMessageHashCallback(s, hashCol = 7) - - messages.add((pubsubTopic, message, digest, storedAt, hash)) - - let query = block: - let where = whereClausev2( - cursor.isSome(), pubsubTopic, contentTopic, startTime, endTime, ascending - ) - - selectMessagesWithLimitQueryv2(DbTable, where, limit, ascending) - - let dbStmt = ?db.prepareStmt(query) - ?dbStmt.execSelectMessagesV2WithLimitStmt( - cursor, pubsubTopic, contentTopic, startTime, endTime, queryRowCallback - ) - dbStmt.dispose() - - return ok(messages) - -### Store v3 ### - -proc execSelectMessageByHash( - s: SqliteStmt, hash: WakuMessageHash, onRowCallback: DataProc -): DatabaseResult[void] = - let s = RawStmtPtr(s) - - checkErr bindParam(s, 1, toSeq(hash)) - - try: - while true: - let v = sqlite3_step(s) - case v - of SQLITE_ROW: - onRowCallback(s) - of SQLITE_DONE: - return ok() - else: - return err($sqlite3_errstr(v)) - except Exception, CatchableError: - # release implicit transaction - discard sqlite3_reset(s) # same return information as step - discard sqlite3_clear_bindings(s) # no errors possible - -proc selectMessageByHashQuery(): SqlQueryStr = - var query: string - - query = "SELECT contentTopic, payload, version, timestamp, meta, messageHash" - query &= " FROM " & DbTable - query &= " WHERE messageHash = (?)" - - return query - -proc whereClause( - cursor: bool, - pubsubTopic: Option[PubsubTopic], - contentTopic: seq[ContentTopic], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - hashes: seq[WakuMessageHash], - ascending: bool, -): Option[string] = - let cursorClause = - if cursor: - let comp = if ascending: ">" else: "<" - - some("(timestamp, messageHash) " & comp & " (?, ?)") - else: - none(string) - - let pubsubTopicClause = - if pubsubTopic.isNone(): - none(string) - else: - some("pubsubTopic = (?)") - - let contentTopicClause = - if contentTopic.len <= 0: - none(string) - else: - var where = "contentTopic IN (" - where &= "?" - for _ in 1 ..< contentTopic.len: - where &= ", ?" - where &= ")" - some(where) - - let startTimeClause = - if startTime.isNone(): - none(string) - else: - some("storedAt >= (?)") - - let endTimeClause = - if endTime.isNone(): - none(string) - else: - some("storedAt <= (?)") - - let hashesClause = - if hashes.len <= 0: - none(string) - else: - var where = "messageHash IN (" - where &= "?" - for _ in 1 ..< hashes.len: - where &= ", ?" - where &= ")" - some(where) - - return combineClauses( - cursorClause, pubsubTopicClause, contentTopicClause, startTimeClause, endTimeClause, - hashesClause, - ) - -proc execSelectMessagesWithLimitStmt( - s: SqliteStmt, - cursor: Option[(Timestamp, WakuMessageHash)], - pubsubTopic: Option[PubsubTopic], - contentTopic: seq[ContentTopic], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - hashes: seq[WakuMessageHash], - onRowCallback: DataProc, -): DatabaseResult[void] = - let s = RawStmtPtr(s) - - # Bind params - var paramIndex = 1 - - if cursor.isSome(): - let (time, hash) = cursor.get() - checkErr bindParam(s, paramIndex, time) - paramIndex += 1 - checkErr bindParam(s, paramIndex, toSeq(hash)) - paramIndex += 1 - - if pubsubTopic.isSome(): - let pubsubTopic = toBytes(pubsubTopic.get()) - checkErr bindParam(s, paramIndex, pubsubTopic) - paramIndex += 1 - - for topic in contentTopic: - checkErr bindParam(s, paramIndex, topic.toBytes()) - paramIndex += 1 - - for hash in hashes: - checkErr bindParam(s, paramIndex, toSeq(hash)) - paramIndex += 1 - - if startTime.isSome(): - let time = startTime.get() - checkErr bindParam(s, paramIndex, time) - paramIndex += 1 - - if endTime.isSome(): - let time = endTime.get() - checkErr bindParam(s, paramIndex, time) - paramIndex += 1 - - try: - while true: - let v = sqlite3_step(s) - case v - of SQLITE_ROW: - onRowCallback(s) - of SQLITE_DONE: - return ok() - else: - return err($sqlite3_errstr(v)) - except Exception, CatchableError: - # release implicit transaction - discard sqlite3_reset(s) # same return information as step - discard sqlite3_clear_bindings(s) # no errors possible - -proc selectMessagesWithLimitQuery( - table: string, where: Option[string], limit: uint, ascending = true, v3 = false -): SqlQueryStr = - let order = if ascending: "ASC" else: "DESC" - - var query: string - - query = - "SELECT storedAt, contentTopic, payload, pubsubTopic, version, timestamp, id, messageHash, meta" - query &= " FROM " & table - - if where.isSome(): - query &= " WHERE " & where.get() - - query &= " ORDER BY storedAt " & order & ", messageHash " & order - - query &= " LIMIT " & $limit & ";" - - return query - -proc selectMessagesByStoreQueryWithLimit*( - db: SqliteDatabase, - contentTopic: seq[ContentTopic], - pubsubTopic: Option[PubsubTopic], - cursor: Option[WakuMessageHash], - startTime: Option[Timestamp], - endTime: Option[Timestamp], - hashes: seq[WakuMessageHash], - limit: uint, - ascending: bool, -): DatabaseResult[ - seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] -] = - # Must first get the message timestamp before paginating by time - let newCursor = - if cursor.isSome() and cursor.get() != EmptyWakuMessageHash: - let hash: WakuMessageHash = cursor.get() - - var wakuMessage: Option[WakuMessage] - - proc queryRowCallback(s: ptr sqlite3_stmt) = - wakuMessage = some( - queryRowWakuMessageCallback( - s, - contentTopicCol = 0, - payloadCol = 1, - versionCol = 2, - senderTimestampCol = 3, - metaCol = 4, - ) - ) - - let query = selectMessageByHashQuery() - let dbStmt = ?db.prepareStmt(query) - ?dbStmt.execSelectMessageByHash(hash, queryRowCallback) - dbStmt.dispose() - - if wakuMessage.isSome(): - let time = wakuMessage.get().timestamp - - some((time, hash)) - else: - return err("cursor not found") - else: - none((Timestamp, WakuMessageHash)) - - var messages: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] = - @[] - - proc queryRowCallback(s: ptr sqlite3_stmt) = - let - pubsubTopic = queryRowPubsubTopicCallback(s, pubsubTopicCol = 3) - message = queryRowWakuMessageCallback( - s, - contentTopicCol = 1, - payloadCol = 2, - versionCol = 4, - senderTimestampCol = 5, - metaCol = 8, - ) - digest = queryRowDigestCallback(s, digestCol = 6) - storedAt = queryRowReceiverTimestampCallback(s, storedAtCol = 0) - hash = queryRowWakuMessageHashCallback(s, hashCol = 7) - - messages.add((pubsubTopic, message, digest, storedAt, hash)) - - let query = block: - let where = whereClause( - newCursor.isSome(), - pubsubTopic, - contentTopic, - startTime, - endTime, - hashes, - ascending, - ) - - selectMessagesWithLimitQuery(DbTable, where, limit, ascending, true) - - let dbStmt = ?db.prepareStmt(query) - ?dbStmt.execSelectMessagesWithLimitStmt( - newCursor, pubsubTopic, contentTopic, startTime, endTime, hashes, queryRowCallback - ) - dbStmt.dispose() - - return ok(messages) diff --git a/waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim b/waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim deleted file mode 100644 index 63e7c7eac..000000000 --- a/waku/waku_archive_legacy/driver/sqlite_driver/sqlite_driver.nim +++ /dev/null @@ -1,220 +0,0 @@ -# The code in this file is an adaptation of the Sqlite KV Store found in nim-eth. -# https://github.com/status-im/nim-eth/blob/master/eth/db/kvstore_sqlite3.nim -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} - -import std/options, stew/byteutils, chronicles, chronos, results -import - ../../../common/databases/db_sqlite, - ../../../waku_core, - ../../../waku_core/message/digest, - ../../common, - ../../driver, - ./cursor, - ./queries - -logScope: - topics = "waku archive sqlite" - -proc init(db: SqliteDatabase): ArchiveDriverResult[void] = - ## Misconfiguration can lead to nil DB - if db.isNil(): - return err("db not initialized") - - # Create table, if doesn't exist - createTable(db).isOkOr: - return err("failed to create table: " & error) - - # Create indices, if don't exist - createOldestMessageTimestampIndex(db).isOkOr: - return err("failed to create i_rt index: " & error) - - createHistoryQueryIndex(db).isOkOr: - return err("failed to create i_query index: " & error) - - return ok() - -type SqliteDriver* = ref object of ArchiveDriver - db: SqliteDatabase - insertStmt: SqliteStmt[InsertMessageParams, void] - -proc new*(T: type SqliteDriver, db: SqliteDatabase): ArchiveDriverResult[T] = - # Database initialization - ?init(db) - - # General initialization - let insertStmt = db.prepareInsertMessageStmt() - return ok(SqliteDriver(db: db, insertStmt: insertStmt)) - -method put*( - s: SqliteDriver, - pubsubTopic: PubsubTopic, - message: WakuMessage, - digest: MessageDigest, - messageHash: WakuMessageHash, - receivedTime: Timestamp, -): Future[ArchiveDriverResult[void]] {.async.} = - ## Inserts a message into the store - let res = s.insertStmt.exec( - ( - @(digest.data), # id - @(messageHash), # messageHash - receivedTime, # storedAt - toBytes(message.contentTopic), # contentTopic - message.payload, # payload - toBytes(pubsubTopic), # pubsubTopic - int64(message.version), # version - message.timestamp, # senderTimestamp - message.meta, # meta - ) - ) - - return res - -method getAllMessages*( - s: SqliteDriver -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - ## Retrieve all messages from the store. - return s.db.selectAllMessages() - -method getMessagesV2*( - s: SqliteDriver, - contentTopic = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId: string, -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async, deprecated.} = - let cursor = cursor.map(toDbCursor) - - let rowsRes = s.db.selectMessagesByHistoryQueryWithLimit( - contentTopic, - pubsubTopic, - cursor, - startTime, - endTime, - limit = maxPageSize, - ascending = ascendingOrder, - ) - - return rowsRes - -method getMessages*( - s: SqliteDriver, - includeData = true, - contentTopic = newSeq[ContentTopic](0), - pubsubTopic = none(PubsubTopic), - cursor = none(ArchiveCursor), - startTime = none(Timestamp), - endTime = none(Timestamp), - hashes = newSeq[WakuMessageHash](0), - maxPageSize = DefaultPageSize, - ascendingOrder = true, - requestId = "", -): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - let cursor = - if cursor.isSome(): - some(cursor.get().hash) - else: - none(WakuMessageHash) - - let rowsRes = s.db.selectMessagesByStoreQueryWithLimit( - contentTopic, - pubsubTopic, - cursor, - startTime, - endTime, - hashes, - limit = maxPageSize, - ascending = ascendingOrder, - ) - - return rowsRes - -method getMessagesCount*( - s: SqliteDriver -): Future[ArchiveDriverResult[int64]] {.async.} = - return s.db.getMessageCount() - -method getPagesCount*(s: SqliteDriver): Future[ArchiveDriverResult[int64]] {.async.} = - return s.db.getPageCount() - -method getPagesSize*(s: SqliteDriver): Future[ArchiveDriverResult[int64]] {.async.} = - return s.db.getPageSize() - -method getDatabaseSize*(s: SqliteDriver): Future[ArchiveDriverResult[int64]] {.async.} = - return s.db.getDatabaseSize() - -method performVacuum*(s: SqliteDriver): Future[ArchiveDriverResult[void]] {.async.} = - return s.db.performSqliteVacuum() - -method getOldestMessageTimestamp*( - s: SqliteDriver -): Future[ArchiveDriverResult[Timestamp]] {.async.} = - return s.db.selectOldestReceiverTimestamp() - -method getNewestMessageTimestamp*( - s: SqliteDriver -): Future[ArchiveDriverResult[Timestamp]] {.async.} = - return s.db.selectnewestReceiverTimestamp() - -method deleteMessagesOlderThanTimestamp*( - s: SqliteDriver, ts: Timestamp -): Future[ArchiveDriverResult[void]] {.async.} = - return s.db.deleteMessagesOlderThanTimestamp(ts) - -method deleteOldestMessagesNotWithinLimit*( - s: SqliteDriver, limit: int -): Future[ArchiveDriverResult[void]] {.async.} = - return s.db.deleteOldestMessagesNotWithinLimit(limit) - -method decreaseDatabaseSize*( - driver: SqliteDriver, targetSizeInBytes: int64, forceRemoval: bool = false -): Future[ArchiveDriverResult[void]] {.async.} = - ## To remove 20% of the outdated data from database - const DeleteLimit = 0.80 - - ## when db size overshoots the database limit, shread 20% of outdated messages - ## get size of database - let dbSize = (await driver.getDatabaseSize()).valueOr: - return err("failed to get database size: " & $error) - - ## database size in bytes - let totalSizeOfDB: int64 = int64(dbSize) - - if totalSizeOfDB < targetSizeInBytes: - return ok() - - ## to shread/delete messsges, get the total row/message count - let numMessages = (await driver.getMessagesCount()).valueOr: - return err("failed to get messages count: " & error) - - ## NOTE: Using SQLite vacuuming is done manually, we delete a percentage of rows - ## if vacumming is done automatically then we aim to check DB size periodially for efficient - ## retention policy implementation. - - ## 80% of the total messages are to be kept, delete others - let pageDeleteWindow = int(float(numMessages) * DeleteLimit) - - (await driver.deleteOldestMessagesNotWithinLimit(limit = pageDeleteWindow)).isOkOr: - return err("deleting oldest messages failed: " & error) - - return ok() - -method close*(s: SqliteDriver): Future[ArchiveDriverResult[void]] {.async.} = - ## Close the database connection - # Dispose statements - s.insertStmt.dispose() - # Close connection - s.db.close() - return ok() - -method existsTable*( - s: SqliteDriver, tableName: string -): Future[ArchiveDriverResult[bool]] {.async.} = - return err("existsTable method not implemented in sqlite_driver") diff --git a/waku/waku_core/codecs.nim b/waku/waku_core/codecs.nim index 0d9394c71..f0f0c977e 100644 --- a/waku/waku_core/codecs.nim +++ b/waku/waku_core/codecs.nim @@ -9,5 +9,4 @@ const WakuTransferCodec* = "/vac/waku/transfer/1.0.0" WakuMetadataCodec* = "/vac/waku/metadata/1.0.0" WakuPeerExchangeCodec* = "/vac/waku/peer-exchange/2.0.0-alpha1" - WakuLegacyStoreCodec* = "/vac/waku/store/2.0.0-beta4" WakuRendezVousCodec* = "/vac/waku/rendezvous/1.0.0" diff --git a/waku/waku_store_legacy.nim b/waku/waku_store_legacy.nim deleted file mode 100644 index 9dac194c7..000000000 --- a/waku/waku_store_legacy.nim +++ /dev/null @@ -1,3 +0,0 @@ -import ./waku_store_legacy/common, ./waku_store_legacy/protocol - -export common, protocol diff --git a/waku/waku_store_legacy/README.md b/waku/waku_store_legacy/README.md deleted file mode 100644 index f2068734f..000000000 --- a/waku/waku_store_legacy/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Waku Store protocol - -The store protocol implements historical message support. See https://rfc.vac.dev/spec/13/ for more information. diff --git a/waku/waku_store_legacy/client.nim b/waku/waku_store_legacy/client.nim deleted file mode 100644 index 3965e06cf..000000000 --- a/waku/waku_store_legacy/client.nim +++ /dev/null @@ -1,241 +0,0 @@ -{.push raises: [].} - -import std/options, results, chronicles, chronos, metrics, bearssl/rand -import - ../node/peer_manager, - ../utils/requests, - ./protocol_metrics, - ./common, - ./rpc, - ./rpc_codec - -when defined(waku_exp_store_resume): - import std/[sequtils, times] - import ../waku_archive - import ../waku_core/message/digest - -logScope: - topics = "waku legacy store client" - -const DefaultPageSize*: uint = 20 - # A recommended default number of waku messages per page - -type WakuStoreClient* = ref object - peerManager: PeerManager - rng: ref rand.HmacDrbgContext - - # TODO: Move outside of the client - when defined(waku_exp_store_resume): - store: ArchiveDriver - -proc new*( - T: type WakuStoreClient, peerManager: PeerManager, rng: ref rand.HmacDrbgContext -): T = - WakuStoreClient(peerManager: peerManager, rng: rng) - -proc sendHistoryQueryRPC( - w: WakuStoreClient, req: HistoryQuery, peer: RemotePeerInfo -): Future[HistoryResult] {.async, gcsafe.} = - let connOpt = await w.peerManager.dialPeer(peer, WakuLegacyStoreCodec) - if connOpt.isNone(): - waku_legacy_store_errors.inc(labelValues = [dialFailure]) - return err(HistoryError(kind: HistoryErrorKind.PEER_DIAL_FAILURE, address: $peer)) - - let connection = connOpt.get() - - defer: - await connection.closeWithEof() - - let requestId = - if req.requestId != "": - req.requestId - else: - generateRequestId(w.rng) - - let reqRpc = HistoryRPC(requestId: requestId, query: some(req.toRPC())) - await connection.writeLP(reqRpc.encode().buffer) - - #TODO: I see a challenge here, if storeNode uses a different MaxRPCSize this read will fail. - # Need to find a workaround for this. - let buf = await connection.readLp(DefaultMaxRpcSize.int) - let respRpc = HistoryRPC.decode(buf).valueOr: - waku_legacy_store_errors.inc(labelValues = [decodeRpcFailure]) - return - err(HistoryError(kind: HistoryErrorKind.BAD_RESPONSE, cause: decodeRpcFailure)) - - # Disabled ,for now, since the default response is a possible case (no messages, pagesize = 0, error = NONE(0)) - # TODO: Rework the RPC protocol to differentiate the default value from an empty value (e.g., status = 200 (OK)) - # and rework the protobuf parsing to return Option[T] when empty values are received - if respRpc.response.isNone(): - waku_legacy_store_errors.inc(labelValues = [emptyRpcResponseFailure]) - return err( - HistoryError(kind: HistoryErrorKind.BAD_RESPONSE, cause: emptyRpcResponseFailure) - ) - - let resp = respRpc.response.get() - - return resp.toAPI() - -proc query*( - w: WakuStoreClient, req: HistoryQuery, peer: RemotePeerInfo -): Future[HistoryResult] {.async, gcsafe.} = - return await w.sendHistoryQueryRPC(req, peer) - -# TODO: Move outside of the client -when defined(waku_exp_store_resume): - ## Resume store - - const StoreResumeTimeWindowOffset: Timestamp = getNanosecondTime(20) - ## Adjust the time window with an offset of 20 seconds - - proc new*( - T: type WakuStoreClient, - peerManager: PeerManager, - rng: ref rand.HmacDrbgContext, - store: ArchiveDriver, - ): T = - WakuStoreClient(peerManager: peerManager, rng: rng, store: store) - - proc queryAll( - w: WakuStoreClient, query: HistoryQuery, peer: RemotePeerInfo - ): Future[WakuStoreResult[seq[WakuMessage]]] {.async, gcsafe.} = - ## A thin wrapper for query. Sends the query to the given peer. when the query has a valid pagingInfo, - ## it retrieves the historical messages in pages. - ## Returns all the fetched messages, if error occurs, returns an error string - - # Make a copy of the query - var req = query - - var messageList: seq[WakuMessage] = @[] - - while true: - let response = (await w.query(req, peer)).valueOr: - return err($error) - - messageList.add(response.messages) - - # Check whether it is the last page - if response.cursor.isNone(): - break - - # Update paging cursor - req.cursor = response.cursor - - return ok(messageList) - - proc queryLoop( - w: WakuStoreClient, req: HistoryQuery, peers: seq[RemotePeerInfo] - ): Future[WakuStoreResult[seq[WakuMessage]]] {.async, gcsafe.} = - ## Loops through the peers candidate list in order and sends the query to each - ## - ## Once all responses have been received, the retrieved messages are consolidated into one deduplicated list. - ## if no messages have been retrieved, the returned future will resolve into a result holding an empty seq. - let queryFuturesList = peers.mapIt(w.queryAll(req, it)) - - await allFutures(queryFuturesList) - - let messagesList = queryFuturesList - .map( - proc(fut: Future[WakuStoreResult[seq[WakuMessage]]]): seq[WakuMessage] = - try: - # fut.read() can raise a CatchableError - # These futures have been awaited before using allFutures(). Call completed() just as a sanity check. - if not fut.completed() or fut.read().isErr(): - return @[] - - fut.read().value - except CatchableError: - return @[] - ) - .concat() - .deduplicate() - - return ok(messagesList) - - proc put( - store: ArchiveDriver, pubsubTopic: PubsubTopic, message: WakuMessage - ): Result[void, string] = - let - digest = waku_archive.computeDigest(message) - messageHash = computeMessageHash(pubsubTopic, message) - receivedTime = - if message.timestamp > 0: - message.timestamp - else: - getNanosecondTime(getTime().toUnixFloat()) - - store.put(pubsubTopic, message, digest, messageHash, receivedTime) - - proc resume*( - w: WakuStoreClient, - peerList = none(seq[RemotePeerInfo]), - pageSize = DefaultPageSize, - pubsubTopic = DefaultPubsubTopic, - ): Future[WakuStoreResult[uint64]] {.async, gcsafe.} = - ## resume proc retrieves the history of waku messages published on the default waku pubsub topic since the last time the waku store node has been online - ## messages are stored in the store node's messages field and in the message db - ## the offline time window is measured as the difference between the current time and the timestamp of the most recent persisted waku message - ## an offset of 20 second is added to the time window to count for nodes asynchrony - ## peerList indicates the list of peers to query from. - ## The history is fetched from all available peers in this list and then consolidated into one deduplicated list. - ## Such candidates should be found through a discovery method (to be developed). - ## if no peerList is passed, one of the peers in the underlying peer manager unit of the store protocol is picked randomly to fetch the history from. - ## The history gets fetched successfully if the dialed peer has been online during the queried time window. - ## the resume proc returns the number of retrieved messages if no error occurs, otherwise returns the error string - - # If store has not been provided, don't even try - if w.store.isNil(): - return err("store not provided (nil)") - - # NOTE: Original implementation is based on the message's sender timestamp. At the moment - # of writing, the sqlite store implementation returns the last message's receiver - # timestamp. - # lastSeenTime = lastSeenItem.get().msg.timestamp - let - lastSeenTime = w.store.getNewestMessageTimestamp().get(Timestamp(0)) - now = getNanosecondTime(getTime().toUnixFloat()) - - info "resuming with offline time window", - lastSeenTime = lastSeenTime, currentTime = now - - let - queryEndTime = now + StoreResumeTimeWindowOffset - queryStartTime = max(lastSeenTime - StoreResumeTimeWindowOffset, 0) - - let req = HistoryQuery( - pubsubTopic: some(pubsubTopic), - startTime: some(queryStartTime), - endTime: some(queryEndTime), - pageSize: uint64(pageSize), - direction: default(), - ) - - var res: WakuStoreResult[seq[WakuMessage]] - if peerList.isSome(): - info "trying the candidate list to fetch the history" - res = await w.queryLoop(req, peerList.get()) - else: - info "no candidate list is provided, selecting a random peer" - # if no peerList is set then query from one of the peers stored in the peer manager - let peerOpt = w.peerManager.selectPeer(WakuLegacyStoreCodec) - if peerOpt.isNone(): - warn "no suitable remote peers" - waku_legacy_store_errors.inc(labelValues = [peerNotFoundFailure]) - return err("no suitable remote peers") - - info "a peer is selected from peer manager" - res = await w.queryAll(req, peerOpt.get()) - - res.isOkOr: - info "failed to resume the history" - return err("failed to resume the history") - - # Save the retrieved messages in the store - var added: uint = 0 - for msg in res.get(): - w.store.put(pubsubTopic, msg).isOkOr: - continue - - added.inc() - - return ok(added) diff --git a/waku/waku_store_legacy/common.nim b/waku/waku_store_legacy/common.nim deleted file mode 100644 index 6da7f267e..000000000 --- a/waku/waku_store_legacy/common.nim +++ /dev/null @@ -1,108 +0,0 @@ -{.push raises: [].} - -import std/[options, sequtils], results, stew/byteutils, nimcrypto/sha2 -import ../waku_core, ../common/paging - -from ../waku_core/codecs import WakuLegacyStoreCodec -export WakuLegacyStoreCodec - -const - DefaultPageSize*: uint64 = 20 - - MaxPageSize*: uint64 = 100 - -type WakuStoreResult*[T] = Result[T, string] - -## Waku message digest - -type MessageDigest* = MDigest[256] - -proc computeDigest*(msg: WakuMessage): MessageDigest = - var ctx: sha256 - ctx.init() - defer: - ctx.clear() - - ctx.update(msg.contentTopic.toBytes()) - ctx.update(msg.payload) - - # Computes the hash - return ctx.finish() - -## API types - -type - HistoryCursor* = object - pubsubTopic*: PubsubTopic - senderTime*: Timestamp - storeTime*: Timestamp - digest*: MessageDigest - - HistoryQuery* = object - pubsubTopic*: Option[PubsubTopic] - contentTopics*: seq[ContentTopic] - cursor*: Option[HistoryCursor] - startTime*: Option[Timestamp] - endTime*: Option[Timestamp] - pageSize*: uint64 - direction*: PagingDirection - requestId*: string - - HistoryResponse* = object - messages*: seq[WakuMessage] - cursor*: Option[HistoryCursor] - - HistoryErrorKind* {.pure.} = enum - UNKNOWN = uint32(000) - BAD_RESPONSE = uint32(300) - BAD_REQUEST = uint32(400) - TOO_MANY_REQUESTS = uint32(429) - SERVICE_UNAVAILABLE = uint32(503) - PEER_DIAL_FAILURE = uint32(504) - - HistoryError* = object - case kind*: HistoryErrorKind - of PEER_DIAL_FAILURE: - address*: string - of BAD_RESPONSE, BAD_REQUEST: - cause*: string - else: - discard - - HistoryResult* = Result[HistoryResponse, HistoryError] - -proc parse*(T: type HistoryErrorKind, kind: uint32): T = - case kind - of 000, 200, 300, 400, 429, 503: - HistoryErrorKind(kind) - else: - HistoryErrorKind.UNKNOWN - -proc `$`*(err: HistoryError): string = - case err.kind - of HistoryErrorKind.PEER_DIAL_FAILURE: - "PEER_DIAL_FAILURE: " & err.address - of HistoryErrorKind.BAD_RESPONSE: - "BAD_RESPONSE: " & err.cause - of HistoryErrorKind.BAD_REQUEST: - "BAD_REQUEST: " & err.cause - of HistoryErrorKind.TOO_MANY_REQUESTS: - "TOO_MANY_REQUESTS" - of HistoryErrorKind.SERVICE_UNAVAILABLE: - "SERVICE_UNAVAILABLE" - of HistoryErrorKind.UNKNOWN: - "UNKNOWN" - -proc checkHistCursor*(self: HistoryCursor): Result[void, HistoryError] = - if self.pubsubTopic.len == 0: - return err(HistoryError(kind: BAD_REQUEST, cause: "empty pubsubTopic")) - if self.senderTime == 0: - return err(HistoryError(kind: BAD_REQUEST, cause: "invalid senderTime")) - if self.storeTime == 0: - return err(HistoryError(kind: BAD_REQUEST, cause: "invalid storeTime")) - if self.digest.data.all( - proc(x: byte): bool = - x == 0 - ): - return err(HistoryError(kind: BAD_REQUEST, cause: "empty digest")) - return ok() diff --git a/waku/waku_store_legacy/protocol.nim b/waku/waku_store_legacy/protocol.nim deleted file mode 100644 index 8916e8ac0..000000000 --- a/waku/waku_store_legacy/protocol.nim +++ /dev/null @@ -1,188 +0,0 @@ -## Waku Store protocol for historical messaging support. -## See spec for more details: -## https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md -{.push raises: [].} - -import - std/[options, times], - results, - chronicles, - chronos, - bearssl/rand, - libp2p/crypto/crypto, - libp2p/protocols/protocol, - libp2p/protobuf/minprotobuf, - libp2p/stream/connection, - metrics -import - ../waku_core, - ../node/peer_manager, - ./common, - ./rpc, - ./rpc_codec, - ./protocol_metrics, - ../common/rate_limit/request_limiter - -logScope: - topics = "waku legacy store" - -type HistoryQueryHandler* = - proc(req: HistoryQuery): Future[HistoryResult] {.async, gcsafe.} - -type WakuStore* = ref object of LPProtocol - peerManager: PeerManager - rng: ref rand.HmacDrbgContext - queryHandler*: HistoryQueryHandler - requestRateLimiter*: RequestRateLimiter - -## Protocol - -type StoreResp = tuple[resp: seq[byte], requestId: string] - -proc handleLegacyQueryRequest( - self: WakuStore, requestor: PeerId, raw_request: seq[byte] -): Future[StoreResp] {.async.} = - let reqRpc = HistoryRPC.decode(raw_request).valueOr: - error "failed to decode rpc", peerId = requestor, error = $error - waku_legacy_store_errors.inc(labelValues = [decodeRpcFailure]) - return (newSeq[byte](), "failed to decode rpc") - - if reqRpc.query.isNone(): - error "empty query rpc", peerId = requestor, requestId = reqRpc.requestId - waku_legacy_store_errors.inc(labelValues = [emptyRpcQueryFailure]) - return (newSeq[byte](), "empty query rpc") - - let requestId = reqRpc.requestId - var request = reqRpc.query.get().toAPI() - request.requestId = requestId - - info "received history query", - peerId = requestor, requestId = requestId, query = request - waku_legacy_store_queries.inc() - - var responseRes: HistoryResult - try: - responseRes = await self.queryHandler(request) - except Exception: - error "history query failed", - peerId = requestor, requestId = requestId, error = getCurrentExceptionMsg() - - let error = HistoryError(kind: HistoryErrorKind.UNKNOWN).toRPC() - let response = HistoryResponseRPC(error: error) - return ( - HistoryRPC(requestId: requestId, response: some(response)).encode().buffer, - requestId, - ) - - responseRes.isOkOr: - error "history query failed", - peerId = requestor, requestId = requestId, error = error - - let response = responseRes.toRPC() - return ( - HistoryRPC(requestId: requestId, response: some(response)).encode().buffer, - requestId, - ) - - let response = responseRes.toRPC() - - info "sending history response", - peerId = requestor, requestId = requestId, messages = response.messages.len - - return ( - HistoryRPC(requestId: requestId, response: some(response)).encode().buffer, - requestId, - ) - -proc initProtocolHandler(ws: WakuStore) = - let rejectResponseBuf = HistoryRPC( - ## We will not copy and decode RPC buffer from stream only for requestId - ## in reject case as it is comparably too expensive and opens possible - ## attack surface - requestId: "N/A", - response: some( - HistoryResponseRPC( - error: HistoryError(kind: HistoryErrorKind.TOO_MANY_REQUESTS).toRPC() - ) - ), - ).encode().buffer - - proc handler(conn: Connection, proto: string) {.async: (raises: [CancelledError]).} = - var successfulQuery = false ## only consider the correct queries in metrics - var resBuf: StoreResp - var queryDuration: float - - defer: - await conn.closeWithEof() - - ws.requestRateLimiter.checkUsageLimit(WakuLegacyStoreCodec, conn): - let readRes = catch: - await conn.readLp(DefaultMaxRpcSize.int) - - let reqBuf = readRes.valueOr: - error "Connection read error", error = error.msg - return - - waku_service_network_bytes.inc( - amount = reqBuf.len().int64, labelValues = [WakuLegacyStoreCodec, "in"] - ) - - let queryStartTime = getTime().toUnixFloat() - try: - resBuf = await ws.handleLegacyQueryRequest(conn.peerId, reqBuf) - except CatchableError: - error "legacy store query handler failed", - remote_peer_id = conn.peerId, error = getCurrentExceptionMsg() - return - - queryDuration = getTime().toUnixFloat() - queryStartTime - waku_legacy_store_time_seconds.set(queryDuration, ["query-db-time"]) - successfulQuery = true - do: - info "Legacy store query request rejected due rate limit exceeded", - peerId = conn.peerId, limit = $ws.requestRateLimiter.setting - resBuf = (rejectResponseBuf, "rejected") - - let writeRespStartTime = getTime().toUnixFloat() - let writeRes = catch: - await conn.writeLp(resBuf.resp) - - writeRes.isOkOr: - error "Connection write error", error = error.msg - return - - if successfulQuery: - let writeDuration = getTime().toUnixFloat() - writeRespStartTime - waku_legacy_store_time_seconds.set(writeDuration, ["send-store-resp-time"]) - info "after sending response", - requestId = resBuf.requestId, - queryDurationSecs = queryDuration, - writeStreamDurationSecs = writeDuration - - waku_service_network_bytes.inc( - amount = resBuf.resp.len().int64, labelValues = [WakuLegacyStoreCodec, "out"] - ) - - ws.handler = handler - ws.codec = WakuLegacyStoreCodec - -proc new*( - T: type WakuStore, - peerManager: PeerManager, - rng: ref rand.HmacDrbgContext, - queryHandler: HistoryQueryHandler, - rateLimitSetting: Option[RateLimitSetting] = none[RateLimitSetting](), -): T = - # Raise a defect if history query handler is nil - if queryHandler.isNil(): - raise newException(NilAccessDefect, "history query handler is nil") - - let ws = WakuStore( - rng: rng, - peerManager: peerManager, - queryHandler: queryHandler, - requestRateLimiter: newRequestRateLimiter(rateLimitSetting), - ) - ws.initProtocolHandler() - setServiceLimitMetric(WakuLegacyStoreCodec, rateLimitSetting) - ws diff --git a/waku/waku_store_legacy/protocol_metrics.nim b/waku/waku_store_legacy/protocol_metrics.nim deleted file mode 100644 index 45a848998..000000000 --- a/waku/waku_store_legacy/protocol_metrics.nim +++ /dev/null @@ -1,21 +0,0 @@ -{.push raises: [].} - -import metrics - -declarePublicCounter waku_legacy_store_errors, - "number of legacy store protocol errors", ["type"] -declarePublicCounter waku_legacy_store_queries, - "number of legacy store queries received" - -## "query-db-time" phase considers the time when node performs the query to the database. -## "send-store-resp-time" phase is the time when node writes the store response to the store-client. -declarePublicGauge waku_legacy_store_time_seconds, - "Time in seconds spent by each store phase", labels = ["phase"] - -# Error types (metric label values) -const - dialFailure* = "dial_failure_legacy" - decodeRpcFailure* = "decode_rpc_failure_legacy" - peerNotFoundFailure* = "peer_not_found_failure_legacy" - emptyRpcQueryFailure* = "empty_rpc_query_failure_legacy" - emptyRpcResponseFailure* = "empty_rpc_response_failure_legacy" diff --git a/waku/waku_store_legacy/rpc.nim b/waku/waku_store_legacy/rpc.nim deleted file mode 100644 index 44aad8d07..000000000 --- a/waku/waku_store_legacy/rpc.nim +++ /dev/null @@ -1,218 +0,0 @@ -{.push raises: [].} - -import std/[options, sequtils], results -import ../waku_core, ../common/paging, ./common - -## Wire protocol - -const HistoryQueryDirectionDefaultValue = default(type HistoryQuery.direction) - -type PagingIndexRPC* = object - ## This type contains the description of an Index used in the pagination of WakuMessages - pubsubTopic*: PubsubTopic - senderTime*: Timestamp # the time at which the message is generated - receiverTime*: Timestamp - digest*: MessageDigest # calculated over payload and content topic - -proc `==`*(x, y: PagingIndexRPC): bool = - ## receiverTime plays no role in index equality - (x.senderTime == y.senderTime) and (x.digest == y.digest) and - (x.pubsubTopic == y.pubsubTopic) - -proc compute*( - T: type PagingIndexRPC, - msg: WakuMessage, - receivedTime: Timestamp, - pubsubTopic: PubsubTopic, -): T = - ## Takes a WakuMessage with received timestamp and returns its Index. - let - digest = computeDigest(msg) - senderTime = msg.timestamp - - PagingIndexRPC( - pubsubTopic: pubsubTopic, - senderTime: senderTime, - receiverTime: receivedTime, - digest: digest, - ) - -type PagingInfoRPC* = object - ## This type holds the information needed for the pagination - pageSize*: Option[uint64] - cursor*: Option[PagingIndexRPC] - direction*: Option[PagingDirection] - -type - HistoryContentFilterRPC* = object - contentTopic*: ContentTopic - - HistoryQueryRPC* = object - contentFilters*: seq[HistoryContentFilterRPC] - pubsubTopic*: Option[PubsubTopic] - pagingInfo*: Option[PagingInfoRPC] - startTime*: Option[int64] - endTime*: Option[int64] - - HistoryResponseErrorRPC* {.pure.} = enum - ## HistoryResponseErrorRPC contains error message to inform the querying node about - ## the state of its request - NONE = uint32(0) - INVALID_CURSOR = uint32(1) - TOO_MANY_REQUESTS = uint32(429) - SERVICE_UNAVAILABLE = uint32(503) - - HistoryResponseRPC* = object - messages*: seq[WakuMessage] - pagingInfo*: Option[PagingInfoRPC] - error*: HistoryResponseErrorRPC - - HistoryRPC* = object - requestId*: string - query*: Option[HistoryQueryRPC] - response*: Option[HistoryResponseRPC] - -proc parse*(T: type HistoryResponseErrorRPC, kind: uint32): T = - case kind - of 0, 1, 429, 503: - cast[HistoryResponseErrorRPC](kind) - else: - # TODO: Improve error variants/move to satus codes - HistoryResponseErrorRPC.INVALID_CURSOR - -## Wire protocol type mappings - -proc toRPC*(cursor: HistoryCursor): PagingIndexRPC {.gcsafe.} = - PagingIndexRPC( - pubsubTopic: cursor.pubsubTopic, - senderTime: cursor.senderTime, - receiverTime: cursor.storeTime, - digest: cursor.digest, - ) - -proc toAPI*(rpc: PagingIndexRPC): HistoryCursor = - HistoryCursor( - pubsubTopic: rpc.pubsubTopic, - senderTime: rpc.senderTime, - storeTime: rpc.receiverTime, - digest: rpc.digest, - ) - -proc toRPC*(query: HistoryQuery): HistoryQueryRPC = - var rpc = HistoryQueryRPC() - - rpc.contentFilters = - query.contentTopics.mapIt(HistoryContentFilterRPC(contentTopic: it)) - - rpc.pubsubTopic = query.pubsubTopic - - rpc.pagingInfo = block: - if query.cursor.isNone() and query.pageSize == default(type query.pageSize) and - query.direction == HistoryQueryDirectionDefaultValue: - none(PagingInfoRPC) - else: - let - pageSize = some(query.pageSize) - cursor = query.cursor.map(toRPC) - direction = some(query.direction) - - some(PagingInfoRPC(pageSize: pageSize, cursor: cursor, direction: direction)) - - rpc.startTime = query.startTime - rpc.endTime = query.endTime - - rpc - -proc toAPI*(rpc: HistoryQueryRPC): HistoryQuery = - let - pubsubTopic = rpc.pubsubTopic - - contentTopics = rpc.contentFilters.mapIt(it.contentTopic) - - cursor = - if rpc.pagingInfo.isNone() or rpc.pagingInfo.get().cursor.isNone(): - none(HistoryCursor) - else: - rpc.pagingInfo.get().cursor.map(toAPI) - - startTime = rpc.startTime - - endTime = rpc.endTime - - pageSize = - if rpc.pagingInfo.isNone() or rpc.pagingInfo.get().pageSize.isNone(): - 0'u64 - else: - rpc.pagingInfo.get().pageSize.get() - - direction = - if rpc.pagingInfo.isNone() or rpc.pagingInfo.get().direction.isNone(): - HistoryQueryDirectionDefaultValue - else: - rpc.pagingInfo.get().direction.get() - - HistoryQuery( - pubsubTopic: pubsubTopic, - contentTopics: contentTopics, - cursor: cursor, - startTime: startTime, - endTime: endTime, - pageSize: pageSize, - direction: direction, - ) - -proc toRPC*(err: HistoryError): HistoryResponseErrorRPC = - # TODO: Better error mappings/move to error codes - case err.kind - of HistoryErrorKind.BAD_REQUEST: - # TODO: Respond aksi with the reason - HistoryResponseErrorRPC.INVALID_CURSOR - of HistoryErrorKind.TOO_MANY_REQUESTS: - HistoryResponseErrorRPC.TOO_MANY_REQUESTS - of HistoryErrorKind.SERVICE_UNAVAILABLE: - HistoryResponseErrorRPC.SERVICE_UNAVAILABLE - else: - HistoryResponseErrorRPC.INVALID_CURSOR - -proc toAPI*(err: HistoryResponseErrorRPC): HistoryError = - # TODO: Better error mappings/move to error codes - case err - of HistoryResponseErrorRPC.INVALID_CURSOR: - HistoryError(kind: HistoryErrorKind.BAD_REQUEST, cause: "invalid cursor") - of HistoryResponseErrorRPC.TOO_MANY_REQUESTS: - HistoryError(kind: HistoryErrorKind.TOO_MANY_REQUESTS) - of HistoryResponseErrorRPC.SERVICE_UNAVAILABLE: - HistoryError(kind: HistoryErrorKind.SERVICE_UNAVAILABLE) - else: - HistoryError(kind: HistoryErrorKind.UNKNOWN) - -proc toRPC*(res: HistoryResult): HistoryResponseRPC = - let resp = res.valueOr: - return HistoryResponseRPC(error: error.toRPC()) - let - messages = resp.messages - - pagingInfo = block: - if resp.cursor.isNone(): - none(PagingInfoRPC) - else: - some(PagingInfoRPC(cursor: resp.cursor.map(toRPC))) - - error = HistoryResponseErrorRPC.NONE - - HistoryResponseRPC(messages: messages, pagingInfo: pagingInfo, error: error) - -proc toAPI*(rpc: HistoryResponseRPC): HistoryResult = - if rpc.error != HistoryResponseErrorRPC.NONE: - err(rpc.error.toAPI()) - else: - let - messages = rpc.messages - - cursor = - if rpc.pagingInfo.isNone(): - none(HistoryCursor) - else: - rpc.pagingInfo.get().cursor.map(toAPI) - - ok(HistoryResponse(messages: messages, cursor: cursor)) diff --git a/waku/waku_store_legacy/rpc_codec.nim b/waku/waku_store_legacy/rpc_codec.nim deleted file mode 100644 index f9c518e83..000000000 --- a/waku/waku_store_legacy/rpc_codec.nim +++ /dev/null @@ -1,255 +0,0 @@ -{.push raises: [].} - -import std/options, nimcrypto/hash -import ../common/[protobuf, paging], ../waku_core, ./common, ./rpc - -const DefaultMaxRpcSize* = -1 - -## Pagination - -proc encode*(index: PagingIndexRPC): ProtoBuffer = - ## Encode an Index object into a ProtoBuffer - ## returns the resultant ProtoBuffer - var pb = initProtoBuffer() - - pb.write3(1, index.digest.data) - pb.write3(2, zint64(index.receiverTime)) - pb.write3(3, zint64(index.senderTime)) - pb.write3(4, index.pubsubTopic) - pb.finish3() - - pb - -proc decode*(T: type PagingIndexRPC, buffer: seq[byte]): ProtobufResult[T] = - ## creates and returns an Index object out of buffer - var rpc = PagingIndexRPC() - let pb = initProtoBuffer(buffer) - - var data: seq[byte] - if not ?pb.getField(1, data): - return err(ProtobufError.missingRequiredField("digest")) - else: - var digest = MessageDigest() - for count, b in data: - digest.data[count] = b - - rpc.digest = digest - - var receiverTime: zint64 - if not ?pb.getField(2, receiverTime): - return err(ProtobufError.missingRequiredField("receiver_time")) - else: - rpc.receiverTime = int64(receiverTime) - - var senderTime: zint64 - if not ?pb.getField(3, senderTime): - return err(ProtobufError.missingRequiredField("sender_time")) - else: - rpc.senderTime = int64(senderTime) - - var pubsubTopic: string - if not ?pb.getField(4, pubsubTopic): - return err(ProtobufError.missingRequiredField("pubsub_topic")) - else: - rpc.pubsubTopic = pubsubTopic - - ok(rpc) - -proc encode*(rpc: PagingInfoRPC): ProtoBuffer = - ## Encodes a PagingInfo object into a ProtoBuffer - ## returns the resultant ProtoBuffer - var pb = initProtoBuffer() - - pb.write3(1, rpc.pageSize) - pb.write3(2, rpc.cursor.map(encode)) - pb.write3( - 3, - rpc.direction.map( - proc(d: PagingDirection): uint32 = - uint32(ord(d)) - ), - ) - pb.finish3() - - pb - -proc decode*(T: type PagingInfoRPC, buffer: seq[byte]): ProtobufResult[T] = - ## creates and returns a PagingInfo object out of buffer - var rpc = PagingInfoRPC() - let pb = initProtoBuffer(buffer) - - var pageSize: uint64 - if not ?pb.getField(1, pageSize): - rpc.pageSize = none(uint64) - else: - rpc.pageSize = some(pageSize) - - var cursorBuffer: seq[byte] - if not ?pb.getField(2, cursorBuffer): - rpc.cursor = none(PagingIndexRPC) - else: - let cursor = ?PagingIndexRPC.decode(cursorBuffer) - rpc.cursor = some(cursor) - - var direction: uint32 - if not ?pb.getField(3, direction): - rpc.direction = none(PagingDirection) - else: - rpc.direction = some(PagingDirection(direction)) - - ok(rpc) - -## Wire protocol - -proc encode*(rpc: HistoryContentFilterRPC): ProtoBuffer = - var pb = initProtoBuffer() - - pb.write3(1, rpc.contentTopic) - pb.finish3() - - pb - -proc decode*(T: type HistoryContentFilterRPC, buffer: seq[byte]): ProtobufResult[T] = - let pb = initProtoBuffer(buffer) - - var contentTopic: ContentTopic - if not ?pb.getField(1, contentTopic): - return err(ProtobufError.missingRequiredField("content_topic")) - ok(HistoryContentFilterRPC(contentTopic: contentTopic)) - -proc encode*(rpc: HistoryQueryRPC): ProtoBuffer = - var pb = initProtoBuffer() - pb.write3(2, rpc.pubsubTopic) - - for filter in rpc.contentFilters: - pb.write3(3, filter.encode()) - - pb.write3(4, rpc.pagingInfo.map(encode)) - pb.write3( - 5, - rpc.startTime.map( - proc(time: int64): zint64 = - zint64(time) - ), - ) - pb.write3( - 6, - rpc.endTime.map( - proc(time: int64): zint64 = - zint64(time) - ), - ) - pb.finish3() - - pb - -proc decode*(T: type HistoryQueryRPC, buffer: seq[byte]): ProtobufResult[T] = - var rpc = HistoryQueryRPC() - let pb = initProtoBuffer(buffer) - - var pubsubTopic: string - if not ?pb.getField(2, pubsubTopic): - rpc.pubsubTopic = none(string) - else: - rpc.pubsubTopic = some(pubsubTopic) - - var buffs: seq[seq[byte]] - if not ?pb.getRepeatedField(3, buffs): - rpc.contentFilters = @[] - else: - for pb in buffs: - let filter = ?HistoryContentFilterRPC.decode(pb) - rpc.contentFilters.add(filter) - - var pagingInfoBuffer: seq[byte] - if not ?pb.getField(4, pagingInfoBuffer): - rpc.pagingInfo = none(PagingInfoRPC) - else: - let pagingInfo = ?PagingInfoRPC.decode(pagingInfoBuffer) - rpc.pagingInfo = some(pagingInfo) - - var startTime: zint64 - if not ?pb.getField(5, startTime): - rpc.startTime = none(int64) - else: - rpc.startTime = some(int64(startTime)) - - var endTime: zint64 - if not ?pb.getField(6, endTime): - rpc.endTime = none(int64) - else: - rpc.endTime = some(int64(endTime)) - - ok(rpc) - -proc encode*(response: HistoryResponseRPC): ProtoBuffer = - var pb = initProtoBuffer() - - for rpc in response.messages: - pb.write3(2, rpc.encode()) - - pb.write3(3, response.pagingInfo.map(encode)) - pb.write3(4, uint32(ord(response.error))) - pb.finish3() - - pb - -proc decode*(T: type HistoryResponseRPC, buffer: seq[byte]): ProtobufResult[T] = - var rpc = HistoryResponseRPC() - let pb = initProtoBuffer(buffer) - - var messages: seq[seq[byte]] - if ?pb.getRepeatedField(2, messages): - for pb in messages: - let message = ?WakuMessage.decode(pb) - rpc.messages.add(message) - else: - rpc.messages = @[] - - var pagingInfoBuffer: seq[byte] - if ?pb.getField(3, pagingInfoBuffer): - let pagingInfo = ?PagingInfoRPC.decode(pagingInfoBuffer) - rpc.pagingInfo = some(pagingInfo) - else: - rpc.pagingInfo = none(PagingInfoRPC) - - var error: uint32 - if not ?pb.getField(4, error): - return err(ProtobufError.missingRequiredField("error")) - else: - rpc.error = HistoryResponseErrorRPC.parse(error) - - ok(rpc) - -proc encode*(rpc: HistoryRPC): ProtoBuffer = - var pb = initProtoBuffer() - - pb.write3(1, rpc.requestId) - pb.write3(2, rpc.query.map(encode)) - pb.write3(3, rpc.response.map(encode)) - pb.finish3() - - pb - -proc decode*(T: type HistoryRPC, buffer: seq[byte]): ProtobufResult[T] = - var rpc = HistoryRPC() - let pb = initProtoBuffer(buffer) - - if not ?pb.getField(1, rpc.requestId): - return err(ProtobufError.missingRequiredField("request_id")) - - var queryBuffer: seq[byte] - if not ?pb.getField(2, queryBuffer): - rpc.query = none(HistoryQueryRPC) - else: - let query = ?HistoryQueryRPC.decode(queryBuffer) - rpc.query = some(query) - - var responseBuffer: seq[byte] - if not ?pb.getField(3, responseBuffer): - rpc.response = none(HistoryResponseRPC) - else: - let response = ?HistoryResponseRPC.decode(responseBuffer) - rpc.response = some(response) - - ok(rpc) diff --git a/waku/waku_store_legacy/self_req_handler.nim b/waku/waku_store_legacy/self_req_handler.nim deleted file mode 100644 index e465d9e5b..000000000 --- a/waku/waku_store_legacy/self_req_handler.nim +++ /dev/null @@ -1,31 +0,0 @@ -## -## This file is aimed to attend the requests that come directly -## from the 'self' node. It is expected to attend the store requests that -## come from REST-store endpoint when those requests don't indicate -## any store-peer address. -## -## Notice that the REST-store requests normally assume that the REST -## server is acting as a store-client. In this module, we allow that -## such REST-store node can act as store-server as well by retrieving -## its own stored messages. The typical use case for that is when -## using `nwaku-compose`, which spawn a Waku node connected to a local -## database, and the user is interested in retrieving the messages -## stored by that local store node. -## - -import results, chronos -import ./protocol, ./common - -proc handleSelfStoreRequest*( - self: WakuStore, histQuery: HistoryQuery -): Future[WakuStoreResult[HistoryResponse]] {.async.} = - ## Handles the store requests made by the node to itself. - ## Normally used in REST-store requests - - try: - let resp: HistoryResponse = (await self.queryHandler(histQuery)).valueOr: - return err("error in handleSelfStoreRequest: " & $error) - - return WakuStoreResult[HistoryResponse].ok(resp) - except Exception: - return err("exception in handleSelfStoreRequest: " & getCurrentExceptionMsg()) From dc026bbff19050155f0ef1abb3e03a64c8bba81e Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Mon, 30 Mar 2026 08:30:34 -0300 Subject: [PATCH 14/19] feat: active filter subscription management for edge nodes (#3773) feat: active filter subscription management for edge nodes ## Subscription Manager * edgeFilterSubLoop reconciles desired vs actual filter subscriptions * edgeFilterHealthLoop pings filter peers, evicts stale ones * EdgeFilterSubState per-shard tracking of confirmed peers and health * best-effort unsubscribe on peer removal * RequestEdgeShardHealth and RequestEdgeFilterPeerCount broker providers ## WakuNode * Remove old edge health loop (loopEdgeHealth, edgeHealthEvent, calculateEdgeTopicHealth) * Register MessageSeenEvent push handler on filter client during start * startDeliveryService now returns `Result[void, string]` and propagates errors ## Health Monitor * getFilterClientHealth queries RequestEdgeFilterPeerCount via broker * Shard/content health providers fall back to RequestEdgeShardHealth when relay inactive * Listen to EventShardTopicHealthChange for health recalculation * Add missing return p.notReady() on failed edge filter peer count request * HealthyThreshold constant moved to `connection_status.nim` ## Broker types * RequestEdgeShardHealth, RequestEdgeFilterPeerCount request types * EventShardTopicHealthChange event type ## Filter Client * Add timeout parameter to ping proc ## Tests * Health monitor event tests with per-node lockNewGlobalBrokerContext * Edge (light client) health update test * Edge health driven by confirmed filter subscriptions test * API subscription tests: sub/receive, failover, peer replacement Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Co-authored by Zoltan Nagy --- tests/api/test_api_subscription.nim | 433 +++++++++++++++- tests/node/test_wakunode_health_monitor.nim | 209 ++++++-- waku/events/peer_events.nim | 2 +- waku/factory/waku.nim | 3 +- .../delivery_service/delivery_service.nim | 14 +- .../recv_service/recv_service.nim | 12 +- .../delivery_service/subscription_manager.nim | 464 ++++++++++++++++-- .../node/health_monitor/connection_status.nim | 3 + .../health_monitor/node_health_monitor.nim | 35 +- waku/node/peer_manager/peer_manager.nim | 37 +- waku/node/waku_node.nim | 93 +--- waku/requests/health_requests.nim | 12 + waku/waku_core/subscription.nim | 4 +- .../subscription/subscription_manager.nim | 52 -- waku/waku_filter_v2/client.nim | 10 +- waku/waku_relay/protocol.nim | 9 +- 16 files changed, 1126 insertions(+), 266 deletions(-) delete mode 100644 waku/waku_core/subscription/subscription_manager.nim diff --git a/tests/api/test_api_subscription.nim b/tests/api/test_api_subscription.nim index 6639e3dea..e0ceb9226 100644 --- a/tests/api/test_api_subscription.nim +++ b/tests/api/test_api_subscription.nim @@ -1,6 +1,6 @@ {.used.} -import std/[strutils, net, options, sets] +import std/[strutils, sequtils, net, options, sets, tables] import chronos, testutils/unittests, stew/byteutils import libp2p/[peerid, peerinfo, multiaddress, crypto/crypto] import ../testlib/[common, wakucore, wakunode, testasync] @@ -13,12 +13,12 @@ import common/broker/broker_context, events/message_events, waku_relay/protocol, + node/kernel_api/filter, + node/delivery_service/subscription_manager, ] import waku/factory/waku_conf import tools/confutils/cli_args -# TODO: Edge testing (after MAPI edge support is completed) - const TestTimeout = chronos.seconds(10) const NegativeTestTimeout = chronos.seconds(2) @@ -60,8 +60,10 @@ proc waitForEvents( return await manager.receivedEvent.wait().withTimeout(timeout) type TestNetwork = ref object - publisher: WakuNode + publisher: WakuNode # Relay node that publishes messages in tests. + meshBuddy: WakuNode # Extra relay peer for publisher's mesh (Edge tests only). subscriber: Waku + # The receiver node in tests. Edge node in edge tests, Core node in relay tests. publisherPeerInfo: RemotePeerInfo proc createApiNodeConf( @@ -94,8 +96,12 @@ proc setupNetwork( lockNewGlobalBrokerContext: net.publisher = newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) - net.publisher.mountMetadata(3, @[0'u16]).expect("Failed to mount metadata") + net.publisher.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata" + ) (await net.publisher.mountRelay()).expect("Failed to mount relay") + if mode == cli_args.WakuMode.Edge: + await net.publisher.mountFilter() await net.publisher.mountLibp2pPing() await net.publisher.start() @@ -104,16 +110,32 @@ proc setupNetwork( proc dummyHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = discard - # Subscribe the publisher to all shards to guarantee a GossipSub mesh with the subscriber. - # Currently, Core/Relay nodes auto-subscribe to all network shards on boot, but if - # that changes, this will be needed to cause the publisher to have shard interest - # for any shards the subscriber may want to use, which is required for waitForMesh to work. + var shards: seq[PubsubTopic] for i in 0 ..< numShards.int: - let shard = PubsubTopic("/waku/2/rs/3/" & $i) + shards.add(PubsubTopic("/waku/2/rs/3/" & $i)) + + for shard in shards: net.publisher.subscribe((kind: PubsubSub, topic: shard), dummyHandler).expect( "Failed to sub publisher" ) + if mode == cli_args.WakuMode.Edge: + lockNewGlobalBrokerContext: + net.meshBuddy = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + net.meshBuddy.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on meshBuddy" + ) + (await net.meshBuddy.mountRelay()).expect("Failed to mount relay on meshBuddy") + await net.meshBuddy.start() + + for shard in shards: + net.meshBuddy.subscribe((kind: PubsubSub, topic: shard), dummyHandler).expect( + "Failed to sub meshBuddy" + ) + + await net.meshBuddy.connectToNodes(@[net.publisherPeerInfo]) + net.subscriber = await setupSubscriberNode(createApiNodeConf(mode, numShards)) await net.subscriber.node.connectToNodes(@[net.publisherPeerInfo]) @@ -125,6 +147,10 @@ proc teardown(net: TestNetwork) {.async.} = (await net.subscriber.stop()).expect("Failed to stop subscriber node") net.subscriber = nil + if not isNil(net.meshBuddy): + await net.meshBuddy.stop() + net.meshBuddy = nil + if not isNil(net.publisher): await net.publisher.stop() net.publisher = nil @@ -141,18 +167,34 @@ proc waitForMesh(node: WakuNode, shard: PubsubTopic) {.async.} = await sleepAsync(100.milliseconds) raise newException(ValueError, "GossipSub Mesh failed to stabilize on " & shard) +proc waitForEdgeSubs(w: Waku, shard: PubsubTopic) {.async.} = + let sm = w.deliveryService.subscriptionManager + for _ in 0 ..< 50: + if sm.edgeFilterPeerCount(shard) > 0: + return + await sleepAsync(100.milliseconds) + raise newException(ValueError, "Edge filter subscription failed on " & shard) + proc publishToMesh( net: TestNetwork, contentTopic: ContentTopic, payload: seq[byte] ): Future[Result[int, string]] {.async.} = + # Publishes a message from "publisher" via relay into the gossipsub mesh. let shard = net.subscriber.node.getRelayShard(contentTopic) - await waitForMesh(net.publisher, shard) - let msg = WakuMessage( payload: payload, contentTopic: contentTopic, version: 0, timestamp: now() ) return await net.publisher.publish(some(shard), msg) +proc publishToMeshAfterEdgeReady( + net: TestNetwork, contentTopic: ContentTopic, payload: seq[byte] +): Future[Result[int, string]] {.async.} = + # First, ensure "subscriber" node (an edge node) is subscribed and ready to receive. + # Afterwards, "publisher" (relay node) sends the message in the gossipsub network. + let shard = net.subscriber.node.getRelayShard(contentTopic) + await waitForEdgeSubs(net.subscriber, shard) + return await net.publishToMesh(contentTopic, payload) + suite "Messaging API, SubscriptionManager": asyncTest "Subscription API, relay node auto subscribe and receive message": let net = await setupNetwork(1) @@ -398,3 +440,370 @@ suite "Messaging API, SubscriptionManager": activeSubs.add(t) await verifyNetworkState(activeSubs) + + asyncTest "Subscription API, edge node subscribe and receive message": + let net = await setupNetwork(1, cli_args.WakuMode.Edge) + defer: + await net.teardown() + + let testTopic = ContentTopic("/waku/2/test-content/proto") + (await net.subscriber.subscribe(testTopic)).expect("failed to subscribe") + + let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) + defer: + eventManager.teardown() + + discard (await net.publishToMeshAfterEdgeReady(testTopic, "Hello, edge!".toBytes())).expect( + "Publish failed" + ) + + require await eventManager.waitForEvents(TestTimeout) + require eventManager.receivedMessages.len == 1 + check eventManager.receivedMessages[0].contentTopic == testTopic + + asyncTest "Subscription API, edge node ignores unsubscribed content topics": + let net = await setupNetwork(1, cli_args.WakuMode.Edge) + defer: + await net.teardown() + + let subbedTopic = ContentTopic("/waku/2/subbed-topic/proto") + let ignoredTopic = ContentTopic("/waku/2/ignored-topic/proto") + (await net.subscriber.subscribe(subbedTopic)).expect("failed to subscribe") + + let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) + defer: + eventManager.teardown() + + discard (await net.publishToMesh(ignoredTopic, "Ghost Msg".toBytes())).expect( + "Publish failed" + ) + + check not await eventManager.waitForEvents(NegativeTestTimeout) + check eventManager.receivedMessages.len == 0 + + asyncTest "Subscription API, edge node unsubscribe stops message receipt": + let net = await setupNetwork(1, cli_args.WakuMode.Edge) + defer: + await net.teardown() + + let testTopic = ContentTopic("/waku/2/unsub-test/proto") + + (await net.subscriber.subscribe(testTopic)).expect("failed to subscribe") + net.subscriber.unsubscribe(testTopic).expect("failed to unsubscribe") + + let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) + defer: + eventManager.teardown() + + discard (await net.publishToMesh(testTopic, "Should be dropped".toBytes())).expect( + "Publish failed" + ) + + check not await eventManager.waitForEvents(NegativeTestTimeout) + check eventManager.receivedMessages.len == 0 + + asyncTest "Subscription API, edge node overlapping topics isolation": + let net = await setupNetwork(1, cli_args.WakuMode.Edge) + defer: + await net.teardown() + + let topicA = ContentTopic("/waku/2/topic-a/proto") + let topicB = ContentTopic("/waku/2/topic-b/proto") + (await net.subscriber.subscribe(topicA)).expect("failed to sub A") + (await net.subscriber.subscribe(topicB)).expect("failed to sub B") + + let shard = net.subscriber.node.getRelayShard(topicA) + await waitForEdgeSubs(net.subscriber, shard) + + let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) + defer: + eventManager.teardown() + + net.subscriber.unsubscribe(topicA).expect("failed to unsub A") + + discard (await net.publishToMesh(topicA, "Dropped Message".toBytes())).expect( + "Publish A failed" + ) + discard + (await net.publishToMesh(topicB, "Kept Msg".toBytes())).expect("Publish B failed") + + require await eventManager.waitForEvents(TestTimeout) + require eventManager.receivedMessages.len == 1 + check eventManager.receivedMessages[0].contentTopic == topicB + + asyncTest "Subscription API, edge node resubscribe after unsubscribe": + let net = await setupNetwork(1, cli_args.WakuMode.Edge) + defer: + await net.teardown() + + let testTopic = ContentTopic("/waku/2/resub-test/proto") + + (await net.subscriber.subscribe(testTopic)).expect("Initial sub failed") + + var eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) + discard (await net.publishToMeshAfterEdgeReady(testTopic, "Msg 1".toBytes())).expect( + "Pub 1 failed" + ) + + require await eventManager.waitForEvents(TestTimeout) + eventManager.teardown() + + net.subscriber.unsubscribe(testTopic).expect("Unsub failed") + eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) + + discard + (await net.publishToMesh(testTopic, "Ghost".toBytes())).expect("Ghost pub failed") + + check not await eventManager.waitForEvents(NegativeTestTimeout) + eventManager.teardown() + + (await net.subscriber.subscribe(testTopic)).expect("Resub failed") + eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1) + + discard (await net.publishToMeshAfterEdgeReady(testTopic, "Msg 2".toBytes())).expect( + "Pub 2 failed" + ) + + require await eventManager.waitForEvents(TestTimeout) + check eventManager.receivedMessages[0].payload == "Msg 2".toBytes() + + asyncTest "Subscription API, edge node failover after service peer dies": + # NOTE: This test is a bit more verbose because it defines a custom topology. + # It doesn't use the shared TestNetwork helper. + # This mounts two service peers for the edge node then fails one. + let numShards: uint16 = 1 + let shards = @[PubsubTopic("/waku/2/rs/3/0")] + + proc dummyHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + discard + + var publisher: WakuNode + lockNewGlobalBrokerContext: + publisher = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + publisher.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on publisher" + ) + (await publisher.mountRelay()).expect("Failed to mount relay on publisher") + await publisher.mountFilter() + await publisher.mountLibp2pPing() + await publisher.start() + + for shard in shards: + publisher.subscribe((kind: PubsubSub, topic: shard), dummyHandler).expect( + "Failed to sub publisher" + ) + + let publisherPeerInfo = publisher.peerInfo.toRemotePeerInfo() + + var meshBuddy: WakuNode + lockNewGlobalBrokerContext: + meshBuddy = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + meshBuddy.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on meshBuddy" + ) + (await meshBuddy.mountRelay()).expect("Failed to mount relay on meshBuddy") + await meshBuddy.mountFilter() + await meshBuddy.mountLibp2pPing() + await meshBuddy.start() + + for shard in shards: + meshBuddy.subscribe((kind: PubsubSub, topic: shard), dummyHandler).expect( + "Failed to sub meshBuddy" + ) + + let meshBuddyPeerInfo = meshBuddy.peerInfo.toRemotePeerInfo() + + await meshBuddy.connectToNodes(@[publisherPeerInfo]) + + let conf = createApiNodeConf(cli_args.WakuMode.Edge, numShards) + var subscriber: Waku + lockNewGlobalBrokerContext: + subscriber = (await createNode(conf)).expect("Failed to create edge subscriber") + (await startWaku(addr subscriber)).expect("Failed to start edge subscriber") + + # Connect edge subscriber to both filter servers so selectPeers finds both + await subscriber.node.connectToNodes(@[publisherPeerInfo, meshBuddyPeerInfo]) + + let testTopic = ContentTopic("/waku/2/failover-test/proto") + let shard = subscriber.node.getRelayShard(testTopic) + + (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") + + # Wait for dialing both filter servers (HealthyThreshold = 2) + for _ in 0 ..< 100: + if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: + break + await sleepAsync(100.milliseconds) + + check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2 + + # Verify message delivery with both servers alive + await waitForMesh(publisher, shard) + + var eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1) + let msg1 = WakuMessage( + payload: "Before failover".toBytes(), + contentTopic: testTopic, + version: 0, + timestamp: now(), + ) + discard (await publisher.publish(some(shard), msg1)).expect("Publish 1 failed") + + require await eventManager.waitForEvents(TestTimeout) + check eventManager.receivedMessages[0].payload == "Before failover".toBytes() + eventManager.teardown() + + # Disconnect meshBuddy from edge (keeps relay mesh alive for publishing) + await subscriber.node.disconnectNode(meshBuddyPeerInfo) + + # Wait for the dead peer to be pruned + for _ in 0 ..< 50: + if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) < 2: + break + await sleepAsync(100.milliseconds) + + check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 1 + + # Verify messages still arrive through the surviving filter server (publisher) + eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1) + let msg2 = WakuMessage( + payload: "After failover".toBytes(), + contentTopic: testTopic, + version: 0, + timestamp: now(), + ) + discard (await publisher.publish(some(shard), msg2)).expect("Publish 2 failed") + + require await eventManager.waitForEvents(TestTimeout) + check eventManager.receivedMessages[0].payload == "After failover".toBytes() + eventManager.teardown() + + (await subscriber.stop()).expect("Failed to stop subscriber") + await meshBuddy.stop() + await publisher.stop() + + asyncTest "Subscription API, edge node dials replacement after peer eviction": + # 3 service peers: publisher, meshBuddy, sparePeer. Edge subscribes and + # confirms 2 (HealthyThreshold). After one is disconnected, the sub loop + # should detect the loss and dial the spare to recover back to threshold. + let numShards: uint16 = 1 + let shards = @[PubsubTopic("/waku/2/rs/3/0")] + + proc dummyHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + discard + + var publisher: WakuNode + lockNewGlobalBrokerContext: + publisher = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + publisher.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on publisher" + ) + (await publisher.mountRelay()).expect("Failed to mount relay on publisher") + await publisher.mountFilter() + await publisher.mountLibp2pPing() + await publisher.start() + + for shard in shards: + publisher.subscribe((kind: PubsubSub, topic: shard), dummyHandler).expect( + "Failed to sub publisher" + ) + + let publisherPeerInfo = publisher.peerInfo.toRemotePeerInfo() + + var meshBuddy: WakuNode + lockNewGlobalBrokerContext: + meshBuddy = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + meshBuddy.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on meshBuddy" + ) + (await meshBuddy.mountRelay()).expect("Failed to mount relay on meshBuddy") + await meshBuddy.mountFilter() + await meshBuddy.mountLibp2pPing() + await meshBuddy.start() + + for shard in shards: + meshBuddy.subscribe((kind: PubsubSub, topic: shard), dummyHandler).expect( + "Failed to sub meshBuddy" + ) + + let meshBuddyPeerInfo = meshBuddy.peerInfo.toRemotePeerInfo() + + var sparePeer: WakuNode + lockNewGlobalBrokerContext: + sparePeer = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + sparePeer.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on sparePeer" + ) + (await sparePeer.mountRelay()).expect("Failed to mount relay on sparePeer") + await sparePeer.mountFilter() + await sparePeer.mountLibp2pPing() + await sparePeer.start() + + for shard in shards: + sparePeer.subscribe((kind: PubsubSub, topic: shard), dummyHandler).expect( + "Failed to sub sparePeer" + ) + + let sparePeerInfo = sparePeer.peerInfo.toRemotePeerInfo() + + await meshBuddy.connectToNodes(@[publisherPeerInfo]) + await sparePeer.connectToNodes(@[publisherPeerInfo]) + + let conf = createApiNodeConf(cli_args.WakuMode.Edge, numShards) + var subscriber: Waku + lockNewGlobalBrokerContext: + subscriber = (await createNode(conf)).expect("Failed to create edge subscriber") + (await startWaku(addr subscriber)).expect("Failed to start edge subscriber") + + await subscriber.node.connectToNodes( + @[publisherPeerInfo, meshBuddyPeerInfo, sparePeerInfo] + ) + + let testTopic = ContentTopic("/waku/2/replacement-test/proto") + let shard = subscriber.node.getRelayShard(testTopic) + + (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") + + # Wait for 2 confirmed peers (HealthyThreshold). The 3rd is available but not dialed. + for _ in 0 ..< 100: + if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: + break + await sleepAsync(100.milliseconds) + + require subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) == + 2 + + await subscriber.node.disconnectNode(meshBuddyPeerInfo) + + # Wait for the sub loop to detect the loss and dial a replacement + for _ in 0 ..< 100: + if subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2: + break + await sleepAsync(100.milliseconds) + + check subscriber.deliveryService.subscriptionManager.edgeFilterPeerCount(shard) >= 2 + + await waitForMesh(publisher, shard) + + var eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1) + let msg = WakuMessage( + payload: "After replacement".toBytes(), + contentTopic: testTopic, + version: 0, + timestamp: now(), + ) + discard (await publisher.publish(some(shard), msg)).expect("Publish failed") + + require await eventManager.waitForEvents(TestTimeout) + check eventManager.receivedMessages[0].payload == "After replacement".toBytes() + eventManager.teardown() + + (await subscriber.stop()).expect("Failed to stop subscriber") + await sparePeer.stop() + await meshBuddy.stop() + await publisher.stop() diff --git a/tests/node/test_wakunode_health_monitor.nim b/tests/node/test_wakunode_health_monitor.nim index 416dc9dda..8a3ddd104 100644 --- a/tests/node/test_wakunode_health_monitor.nim +++ b/tests/node/test_wakunode_health_monitor.nim @@ -12,12 +12,18 @@ import node/health_monitor/health_status, node/health_monitor/connection_status, node/health_monitor/protocol_health, + node/health_monitor/topic_health, node/health_monitor/node_health_monitor, + node/delivery_service/delivery_service, + node/delivery_service/subscription_manager, node/kernel_api/relay, node/kernel_api/store, node/kernel_api/lightpush, node/kernel_api/filter, + events/health_events, + events/peer_events, waku_archive, + common/broker/broker_context, ] import ../testlib/[wakunode, wakucore], ../waku_archive/archive_utils @@ -129,13 +135,12 @@ suite "Health Monitor - health state calculation": suite "Health Monitor - events": asyncTest "Core (relay) health update": - let - nodeAKey = generateSecp256k1Key() + var nodeA: WakuNode + lockNewGlobalBrokerContext: + let nodeAKey = generateSecp256k1Key() nodeA = newTestWakuNode(nodeAKey, parseIpAddress("127.0.0.1"), Port(0)) - - (await nodeA.mountRelay()).expect("Node A failed to mount Relay") - - await nodeA.start() + (await nodeA.mountRelay()).expect("Node A failed to mount Relay") + await nodeA.start() let monitorA = NodeHealthMonitor.new(nodeA) @@ -151,17 +156,15 @@ suite "Health Monitor - events": monitorA.startHealthMonitor().expect("Health monitor failed to start") - let - nodeBKey = generateSecp256k1Key() + var nodeB: WakuNode + lockNewGlobalBrokerContext: + let nodeBKey = generateSecp256k1Key() nodeB = newTestWakuNode(nodeBKey, parseIpAddress("127.0.0.1"), Port(0)) - - let driver = newSqliteArchiveDriver() - nodeB.mountArchive(driver).expect("Node B failed to mount archive") - - (await nodeB.mountRelay()).expect("Node B failed to mount relay") - await nodeB.mountStore() - - await nodeB.start() + let driver = newSqliteArchiveDriver() + nodeB.mountArchive(driver).expect("Node B failed to mount archive") + (await nodeB.mountRelay()).expect("Node B failed to mount relay") + await nodeB.mountStore() + await nodeB.start() await nodeA.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()]) @@ -214,15 +217,20 @@ suite "Health Monitor - events": await nodeA.stop() asyncTest "Edge (light client) health update": - let - nodeAKey = generateSecp256k1Key() + var nodeA: WakuNode + lockNewGlobalBrokerContext: + let nodeAKey = generateSecp256k1Key() nodeA = newTestWakuNode(nodeAKey, parseIpAddress("127.0.0.1"), Port(0)) + nodeA.mountLightpushClient() + await nodeA.mountFilterClient() + nodeA.mountStoreClient() + require nodeA.mountAutoSharding(1, 8).isOk + nodeA.mountMetadata(1, @[0'u16]).expect("Node A failed to mount metadata") + await nodeA.start() - nodeA.mountLightpushClient() - await nodeA.mountFilterClient() - nodeA.mountStoreClient() - - await nodeA.start() + let ds = + DeliveryService.new(false, nodeA).expect("Failed to create DeliveryService") + ds.startDeliveryService().expect("Failed to start DeliveryService") let monitorA = NodeHealthMonitor.new(nodeA) @@ -238,23 +246,40 @@ suite "Health Monitor - events": monitorA.startHealthMonitor().expect("Health monitor failed to start") - let - nodeBKey = generateSecp256k1Key() + var nodeB: WakuNode + lockNewGlobalBrokerContext: + let nodeBKey = generateSecp256k1Key() nodeB = newTestWakuNode(nodeBKey, parseIpAddress("127.0.0.1"), Port(0)) + let driver = newSqliteArchiveDriver() + nodeB.mountArchive(driver).expect("Node B failed to mount archive") + (await nodeB.mountRelay()).expect("Node B failed to mount relay") + (await nodeB.mountLightpush()).expect("Node B failed to mount lightpush") + await nodeB.mountFilter() + await nodeB.mountStore() + require nodeB.mountAutoSharding(1, 8).isOk + nodeB.mountMetadata(1, toSeq(0'u16 ..< 8'u16)).expect( + "Node B failed to mount metadata" + ) + await nodeB.start() - let driver = newSqliteArchiveDriver() - nodeB.mountArchive(driver).expect("Node B failed to mount archive") - - (await nodeB.mountRelay()).expect("Node B failed to mount relay") - - (await nodeB.mountLightpush()).expect("Node B failed to mount lightpush") - await nodeB.mountFilter() - await nodeB.mountStore() - - await nodeB.start() + var metadataFut = newFuture[void]("waitForMetadata") + let metadataLis = WakuPeerEvent + .listen( + nodeA.brokerCtx, + proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = + if not metadataFut.finished and + evt.kind == WakuPeerEventKind.EventMetadataUpdated: + metadataFut.complete() + , + ) + .expect("Failed to listen for metadata") await nodeA.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()]) + let metadataOk = await metadataFut.withTimeout(TestConnectivityTimeLimit) + WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis) + require metadataOk + let connectTimeLimit = Moment.now() + TestConnectivityTimeLimit var gotConnected = false @@ -292,4 +317,118 @@ suite "Health Monitor - events": lastStatus == ConnectionStatus.Disconnected await monitorA.stopHealthMonitor() + await ds.stopDeliveryService() + await nodeA.stop() + + asyncTest "Edge health driven by confirmed filter subscriptions": + var nodeA: WakuNode + lockNewGlobalBrokerContext: + let nodeAKey = generateSecp256k1Key() + nodeA = newTestWakuNode(nodeAKey, parseIpAddress("127.0.0.1"), Port(0)) + await nodeA.mountFilterClient() + nodeA.mountLightpushClient() + nodeA.mountStoreClient() + require nodeA.mountAutoSharding(1, 8).isOk + nodeA.mountMetadata(1, @[0'u16]).expect("Node A failed to mount metadata") + await nodeA.start() + + let ds = + DeliveryService.new(false, nodeA).expect("Failed to create DeliveryService") + ds.startDeliveryService().expect("Failed to start DeliveryService") + let subMgr = ds.subscriptionManager + + var nodeB: WakuNode + lockNewGlobalBrokerContext: + let nodeBKey = generateSecp256k1Key() + nodeB = newTestWakuNode(nodeBKey, parseIpAddress("127.0.0.1"), Port(0)) + let driver = newSqliteArchiveDriver() + nodeB.mountArchive(driver).expect("Node B failed to mount archive") + (await nodeB.mountRelay()).expect("Node B failed to mount relay") + (await nodeB.mountLightpush()).expect("Node B failed to mount lightpush") + await nodeB.mountFilter() + await nodeB.mountStore() + require nodeB.mountAutoSharding(1, 8).isOk + nodeB.mountMetadata(1, toSeq(0'u16 ..< 8'u16)).expect( + "Node B failed to mount metadata" + ) + await nodeB.start() + + let monitorA = NodeHealthMonitor.new(nodeA) + + var + lastStatus = ConnectionStatus.Disconnected + healthSignal = newAsyncEvent() + + monitorA.onConnectionStatusChange = proc(status: ConnectionStatus) {.async.} = + lastStatus = status + healthSignal.fire() + + monitorA.startHealthMonitor().expect("Health monitor failed to start") + + var metadataFut = newFuture[void]("waitForMetadata") + let metadataLis = WakuPeerEvent + .listen( + nodeA.brokerCtx, + proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = + if not metadataFut.finished and + evt.kind == WakuPeerEventKind.EventMetadataUpdated: + metadataFut.complete() + , + ) + .expect("Failed to listen for metadata") + + await nodeA.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()]) + + let metadataOk = await metadataFut.withTimeout(TestConnectivityTimeLimit) + WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis) + require metadataOk + + var deadline = Moment.now() + TestConnectivityTimeLimit + while Moment.now() < deadline: + if lastStatus == ConnectionStatus.PartiallyConnected: + break + if await healthSignal.wait().withTimeout(deadline - Moment.now()): + healthSignal.clear() + + check lastStatus == ConnectionStatus.PartiallyConnected + + var shardHealthFut = newFuture[EventShardTopicHealthChange]("waitForShardHealth") + + let shardHealthLis = EventShardTopicHealthChange + .listen( + nodeA.brokerCtx, + proc( + evt: EventShardTopicHealthChange + ): Future[void] {.async: (raises: []), gcsafe.} = + if not shardHealthFut.finished and ( + evt.health == TopicHealth.MINIMALLY_HEALTHY or + evt.health == TopicHealth.SUFFICIENTLY_HEALTHY + ): + shardHealthFut.complete(evt) + , + ) + .expect("Failed to listen for shard health") + + let contentTopic = ContentTopic("/waku/2/default-content/proto") + subMgr.subscribe(contentTopic).expect("Failed to subscribe") + + let shardHealthOk = await shardHealthFut.withTimeout(TestConnectivityTimeLimit) + EventShardTopicHealthChange.dropListener(nodeA.brokerCtx, shardHealthLis) + + check shardHealthOk == true + check subMgr.edgeFilterSubStates.len > 0 + + healthSignal.clear() + deadline = Moment.now() + TestConnectivityTimeLimit + while Moment.now() < deadline: + if lastStatus == ConnectionStatus.PartiallyConnected: + break + if await healthSignal.wait().withTimeout(deadline - Moment.now()): + healthSignal.clear() + + check lastStatus == ConnectionStatus.PartiallyConnected + + await ds.stopDeliveryService() + await monitorA.stopHealthMonitor() + await nodeB.stop() await nodeA.stop() diff --git a/waku/events/peer_events.nim b/waku/events/peer_events.nim index 49dfa9f9a..dd02841f7 100644 --- a/waku/events/peer_events.nim +++ b/waku/events/peer_events.nim @@ -8,6 +8,6 @@ type WakuPeerEventKind* {.pure.} = enum EventMetadataUpdated EventBroker: - type EventWakuPeer* = object + type WakuPeerEvent* = object peerId*: PeerId kind*: WakuPeerEventKind diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim index dbee8d093..45e0edee0 100644 --- a/waku/factory/waku.nim +++ b/waku/factory/waku.nim @@ -416,7 +416,8 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: ## Reliability if not waku[].deliveryService.isNil(): - waku[].deliveryService.startDeliveryService() + waku[].deliveryService.startDeliveryService().isOkOr: + return err("failed to start delivery service: " & $error) ## Health Monitor waku[].healthMonitor.startHealthMonitor().isOkOr: diff --git a/waku/node/delivery_service/delivery_service.nim b/waku/node/delivery_service/delivery_service.nim index 258c01e95..fd728d048 100644 --- a/waku/node/delivery_service/delivery_service.nim +++ b/waku/node/delivery_service/delivery_service.nim @@ -1,18 +1,13 @@ ## This module helps to ensure the correct transmission and reception of messages import results -import chronos +import chronos, chronicles import ./recv_service, ./send_service, ./subscription_manager, waku/[ - waku_core, - waku_node, - waku_store/client, - waku_relay/protocol, - waku_lightpush/client, - waku_filter_v2/client, + waku_core, waku_node, waku_store/client, waku_relay/protocol, waku_lightpush/client ] type DeliveryService* = ref object @@ -37,10 +32,11 @@ proc new*( ) ) -proc startDeliveryService*(self: DeliveryService) = - self.subscriptionManager.startSubscriptionManager() +proc startDeliveryService*(self: DeliveryService): Result[void, string] = + ?self.subscriptionManager.startSubscriptionManager() self.recvService.startRecvService() self.sendService.startSendService() + return ok() proc stopDeliveryService*(self: DeliveryService) {.async.} = await self.sendService.stopSendService() diff --git a/waku/node/delivery_service/recv_service/recv_service.nim b/waku/node/delivery_service/recv_service/recv_service.nim index 0eba2c450..9a85df2f9 100644 --- a/waku/node/delivery_service/recv_service/recv_service.nim +++ b/waku/node/delivery_service/recv_service/recv_service.nim @@ -91,20 +91,20 @@ proc msgChecker(self: RecvService) {.async.} = self.endTimeToCheck = getNowInNanosecondTime() var msgHashesInStore = newSeq[WakuMessageHash](0) - for sub in self.subscriptionManager.getActiveSubscriptions(): + for pubsubTopic, contentTopics in self.subscriptionManager.subscribedTopics: let storeResp: StoreQueryResponse = ( await self.node.wakuStoreClient.queryToAny( StoreQueryRequest( includeData: false, - pubsubTopic: some(PubsubTopic(sub.pubsubTopic)), - contentTopics: sub.contentTopics, + pubsubTopic: some(pubsubTopic), + contentTopics: toSeq(contentTopics), startTime: some(self.startTimeToCheck - DelayExtra.nanos), endTime: some(self.endTimeToCheck + DelayExtra.nanos), ) ) ).valueOr: error "msgChecker failed to get remote msgHashes", - pubsubTopic = sub.pubsubTopic, cTopics = sub.contentTopics, error = $error + pubsubTopic = pubsubTopic, cTopics = toSeq(contentTopics), error = $error continue msgHashesInStore.add(storeResp.messages.mapIt(it.messageHash)) @@ -154,10 +154,6 @@ proc new*(T: typedesc[RecvService], node: WakuNode, s: SubscriptionManager): T = recentReceivedMsgs: @[], ) - # TODO: For MAPI Edge support, either call node.wakuFilterClient.registerPushHandler - # so that the RecvService listens to incoming filter messages, - # or have the filter client emit MessageSeenEvent. - return recvService proc loopPruneOldMessages(self: RecvService) {.async.} = diff --git a/waku/node/delivery_service/subscription_manager.nim b/waku/node/delivery_service/subscription_manager.nim index 22df47413..70d8df7f0 100644 --- a/waku/node/delivery_service/subscription_manager.nim +++ b/waku/node/delivery_service/subscription_manager.nim @@ -1,4 +1,5 @@ -import std/[sets, tables, options, strutils], chronos, chronicles, results +import std/[sequtils, sets, tables, options, strutils], chronos, chronicles, results +import libp2p/[peerid, peerinfo] import waku/[ waku_core, @@ -6,16 +7,67 @@ import waku_core/topics/sharding, waku_node, waku_relay, + waku_filter_v2/common as filter_common, + waku_filter_v2/client as filter_client, + waku_filter_v2/protocol as filter_protocol, common/broker/broker_context, - events/delivery_events, + events/health_events, + events/peer_events, + requests/health_requests, + node/peer_manager, + node/health_monitor/topic_health, + node/health_monitor/connection_status, ] +# --------------------------------------------------------------------------- +# Logos Messaging API SubscriptionManager +# +# Maps all topic subscription intent and centralizes all consistency +# maintenance of the pubsub and content topic subscription model across +# the various network drivers that handle topics (Edge/Filter and Core/Relay). +# --------------------------------------------------------------------------- + +type EdgeFilterSubState* = object + peers: seq[RemotePeerInfo] + ## Filter service peers with confirmed subscriptions on this shard. + pending: seq[Future[void]] ## In-flight dial futures for peers not yet confirmed. + pendingPeers: HashSet[PeerId] ## PeerIds of peers currently being dialed. + currentHealth: TopicHealth + ## Cached health derived from peers.len; updated on every peer set change. + +func toTopicHealth*(peersCount: int): TopicHealth = + if peersCount >= HealthyThreshold: + TopicHealth.SUFFICIENTLY_HEALTHY + elif peersCount > 0: + TopicHealth.MINIMALLY_HEALTHY + else: + TopicHealth.UNHEALTHY + type SubscriptionManager* = ref object of RootObj node: WakuNode contentTopicSubs: Table[PubsubTopic, HashSet[ContentTopic]] ## Map of Shard to ContentTopic needed because e.g. WakuRelay is PubsubTopic only. ## A present key with an empty HashSet value means pubsubtopic already subscribed ## (via subscribePubsubTopics()) but there's no specific content topic interest yet. + edgeFilterSubStates*: Table[PubsubTopic, EdgeFilterSubState] + ## Per-shard filter subscription state for edge mode. + edgeFilterWakeup: AsyncEvent + ## Signalled when the edge filter sub loop should re-reconcile. + edgeFilterSubLoopFut: Future[void] + edgeFilterHealthLoopFut: Future[void] + peerEventListener: WakuPeerEventListener + ## Listener for peer connect/disconnect events (edge filter wakeup). + +iterator subscribedTopics*( + self: SubscriptionManager +): (PubsubTopic, HashSet[ContentTopic]) = + for pubsub, topics in self.contentTopicSubs.pairs: + yield (pubsub, topics) + +proc edgeFilterPeerCount*(sm: SubscriptionManager, shard: PubsubTopic): int = + sm.edgeFilterSubStates.withValue(shard, state): + return state.peers.len + return 0 proc new*(T: typedesc[SubscriptionManager], node: WakuNode): T = SubscriptionManager( @@ -25,30 +77,35 @@ proc new*(T: typedesc[SubscriptionManager], node: WakuNode): T = proc addContentTopicInterest( self: SubscriptionManager, shard: PubsubTopic, topic: ContentTopic ): Result[void, string] = + var changed = false if not self.contentTopicSubs.hasKey(shard): self.contentTopicSubs[shard] = initHashSet[ContentTopic]() + changed = true self.contentTopicSubs.withValue(shard, cTopics): if not cTopics[].contains(topic): cTopics[].incl(topic) + changed = true - # TODO: Call a "subscribe(shard, topic)" on filter client here, - # so the filter client can know that subscriptions changed. + if changed and not isNil(self.edgeFilterWakeup): + self.edgeFilterWakeup.fire() return ok() proc removeContentTopicInterest( self: SubscriptionManager, shard: PubsubTopic, topic: ContentTopic ): Result[void, string] = + var changed = false self.contentTopicSubs.withValue(shard, cTopics): if cTopics[].contains(topic): cTopics[].excl(topic) + changed = true if cTopics[].len == 0 and isNil(self.node.wakuRelay): self.contentTopicSubs.del(shard) # We're done with cTopics here - # TODO: Call a "unsubscribe(shard, topic)" on filter client here, - # so the filter client can know that subscriptions changed. + if changed and not isNil(self.edgeFilterWakeup): + self.edgeFilterWakeup.fire() return ok() @@ -73,46 +130,6 @@ proc subscribePubsubTopics( return ok() -proc startSubscriptionManager*(self: SubscriptionManager) = - if isNil(self.node.wakuRelay): - return - - if self.node.wakuAutoSharding.isSome(): - # Subscribe relay to all shards in autosharding. - let autoSharding = self.node.wakuAutoSharding.get() - let clusterId = autoSharding.clusterId - let numShards = autoSharding.shardCountGenZero - - if numShards > 0: - var clusterPubsubTopics = newSeqOfCap[PubsubTopic](numShards) - - for i in 0 ..< numShards: - let shardObj = RelayShard(clusterId: clusterId, shardId: uint16(i)) - clusterPubsubTopics.add(PubsubTopic($shardObj)) - - self.subscribePubsubTopics(clusterPubsubTopics).isOkOr: - error "Failed to auto-subscribe Relay to cluster shards: ", error = error - else: - info "SubscriptionManager has no AutoSharding configured; skipping auto-subscribe." - -proc stopSubscriptionManager*(self: SubscriptionManager) {.async.} = - discard - -proc getActiveSubscriptions*( - self: SubscriptionManager -): seq[tuple[pubsubTopic: string, contentTopics: seq[ContentTopic]]] = - var activeSubs: seq[tuple[pubsubTopic: string, contentTopics: seq[ContentTopic]]] = - @[] - - for pubsub, cTopicSet in self.contentTopicSubs.pairs: - if cTopicSet.len > 0: - var cTopicSeq = newSeqOfCap[ContentTopic](cTopicSet.len) - for t in cTopicSet: - cTopicSeq.add(t) - activeSubs.add((pubsub, cTopicSeq)) - - return activeSubs - proc getShardForContentTopic( self: SubscriptionManager, topic: ContentTopic ): Result[PubsubTopic, string] = @@ -162,3 +179,358 @@ proc unsubscribe*( ?self.removeContentTopicInterest(shard, topic) return ok() + +# --------------------------------------------------------------------------- +# Edge Filter driver for the Logos Messaging API +# +# The SubscriptionManager absorbs natively the responsibility of using the +# Edge Filter protocol to effect subscriptions and message receipt for edge. +# --------------------------------------------------------------------------- + +const EdgeFilterSubscribeTimeout = chronos.seconds(15) + ## Timeout for a single filter subscribe/unsubscribe RPC to a service peer. +const EdgeFilterPingTimeout = chronos.seconds(5) + ## Timeout for a filter ping health check. +const EdgeFilterLoopInterval = chronos.seconds(30) + ## Interval for the edge filter health ping loop. +const EdgeFilterSubLoopDebounce = chronos.seconds(1) + ## Debounce delay to coalesce rapid-fire wakeups into a single reconciliation pass. + +proc updateShardHealth( + self: SubscriptionManager, shard: PubsubTopic, state: var EdgeFilterSubState +) = + ## Recompute and emit health for a shard after its peer set changed. + let newHealth = toTopicHealth(state.peers.len) + if newHealth != state.currentHealth: + state.currentHealth = newHealth + EventShardTopicHealthChange.emit(self.node.brokerCtx, shard, newHealth) + +proc removePeer(self: SubscriptionManager, shard: PubsubTopic, peerId: PeerId) = + ## Remove a peer from edgeFilterSubStates for the given shard, + ## update health, and wake the sub loop to dial a replacement. + ## Best-effort unsubscribe so the service peer stops pushing to us. + self.edgeFilterSubStates.withValue(shard, state): + var peer: RemotePeerInfo + var found = false + for p in state.peers: + if p.peerId == peerId: + peer = p + found = true + break + if not found: + return + + state.peers.keepItIf(it.peerId != peerId) + self.updateShardHealth(shard, state[]) + self.edgeFilterWakeup.fire() + + if not self.node.wakuFilterClient.isNil(): + self.contentTopicSubs.withValue(shard, topics): + let ct = toSeq(topics[]) + if ct.len > 0: + proc doUnsubscribe() {.async.} = + discard await self.node.wakuFilterClient.unsubscribe(peer, shard, ct) + + asyncSpawn doUnsubscribe() + +type SendChunkedFilterRpcKind = enum + FilterSubscribe + FilterUnsubscribe + +proc sendChunkedFilterRpc( + self: SubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + topics: seq[ContentTopic], + kind: SendChunkedFilterRpcKind, +): Future[bool] {.async.} = + ## Send a chunked filter subscribe or unsubscribe RPC. Returns true on + ## success. On failure the peer is removed and false is returned. + try: + var i = 0 + while i < topics.len: + let chunk = + topics[i ..< min(i + filter_protocol.MaxContentTopicsPerRequest, topics.len)] + let fut = + case kind + of FilterSubscribe: + self.node.wakuFilterClient.subscribe(peer, shard, chunk) + of FilterUnsubscribe: + self.node.wakuFilterClient.unsubscribe(peer, shard, chunk) + if not (await fut.withTimeout(EdgeFilterSubscribeTimeout)) or fut.read().isErr(): + trace "sendChunkedFilterRpc: chunk failed", + op = kind, shard = shard, peer = peer.peerId + self.removePeer(shard, peer.peerId) + return false + i += filter_protocol.MaxContentTopicsPerRequest + except CatchableError as exc: + debug "sendChunkedFilterRpc: failed", + op = kind, shard = shard, peer = peer.peerId, err = exc.msg + self.removePeer(shard, peer.peerId) + return false + return true + +proc syncFilterDeltas( + self: SubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + added: seq[ContentTopic], + removed: seq[ContentTopic], +) {.async.} = + ## Push content topic changes (adds/removes) to an already-tracked peer. + if added.len > 0: + if not await self.sendChunkedFilterRpc(peer, shard, added, FilterSubscribe): + return + + if removed.len > 0: + discard await self.sendChunkedFilterRpc(peer, shard, removed, FilterUnsubscribe) + +proc dialFilterPeer( + self: SubscriptionManager, + peer: RemotePeerInfo, + shard: PubsubTopic, + contentTopics: seq[ContentTopic], +) {.async.} = + ## Subscribe a new peer to all content topics on a shard and start tracking it. + self.edgeFilterSubStates.withValue(shard, state): + state.pendingPeers.incl(peer.peerId) + + try: + if not await self.sendChunkedFilterRpc(peer, shard, contentTopics, FilterSubscribe): + return + + self.edgeFilterSubStates.withValue(shard, state): + if state.peers.anyIt(it.peerId == peer.peerId): + trace "dialFilterPeer: peer already tracked, skipping duplicate", + shard = shard, peer = peer.peerId + return + + state.peers.add(peer) + self.updateShardHealth(shard, state[]) + trace "dialFilterPeer: successfully subscribed to all chunks", + shard = shard, peer = peer.peerId, totalPeers = state.peers.len + do: + trace "dialFilterPeer: shard removed while subscribing, discarding result", + shard = shard, peer = peer.peerId + finally: + self.edgeFilterSubStates.withValue(shard, state): + state.pendingPeers.excl(peer.peerId) + +proc edgeFilterHealthLoop*(self: SubscriptionManager) {.async.} = + ## Periodically pings all connected filter service peers to verify they are + ## still alive at the application layer. Peers that fail the ping are removed. + while true: + await sleepAsync(EdgeFilterLoopInterval) + + if self.node.wakuFilterClient.isNil(): + warn "filter client is nil within edge filter health loop" + continue + + var connected = initTable[PeerId, RemotePeerInfo]() + for state in self.edgeFilterSubStates.values: + for peer in state.peers: + if self.node.peerManager.switch.peerStore.isConnected(peer.peerId): + connected[peer.peerId] = peer + + var alive = initHashSet[PeerId]() + + if connected.len > 0: + var pingTasks: seq[(PeerId, Future[FilterSubscribeResult])] = @[] + for peer in connected.values: + pingTasks.add( + (peer.peerId, self.node.wakuFilterClient.ping(peer, EdgeFilterPingTimeout)) + ) + + # extract future tasks from (PeerId, Future) tuples and await them + await allFutures(pingTasks.mapIt(it[1])) + + for (peerId, task) in pingTasks: + if task.read().isOk(): + alive.incl(peerId) + + var changed = false + for shard, state in self.edgeFilterSubStates.mpairs: + let oldLen = state.peers.len + state.peers.keepItIf(it.peerId notin connected or alive.contains(it.peerId)) + + if state.peers.len < oldLen: + changed = true + self.updateShardHealth(shard, state) + trace "Edge Filter health degraded by Ping failure", + shard = shard, new = state.currentHealth + + if changed: + self.edgeFilterWakeup.fire() + +proc edgeFilterSubLoop*(self: SubscriptionManager) {.async.} = + ## Reconciles filter subscriptions with the desired state from SubscriptionManager. + var lastSynced = initTable[PubsubTopic, HashSet[ContentTopic]]() + + while true: + await self.edgeFilterWakeup.wait() + await sleepAsync(EdgeFilterSubLoopDebounce) + self.edgeFilterWakeup.clear() + trace "edgeFilterSubLoop: woke up" + + if isNil(self.node.wakuFilterClient): + trace "edgeFilterSubLoop: wakuFilterClient is nil, skipping" + continue + + let desired = self.contentTopicSubs + + trace "edgeFilterSubLoop: desired state", numShards = desired.len + + let allShards = toHashSet(toSeq(desired.keys)) + toHashSet(toSeq(lastSynced.keys)) + + for shard in allShards: + let currTopics = desired.getOrDefault(shard) + let prevTopics = lastSynced.getOrDefault(shard) + + if shard notin self.edgeFilterSubStates: + self.edgeFilterSubStates[shard] = + EdgeFilterSubState(currentHealth: TopicHealth.UNHEALTHY) + + let addedTopics = toSeq(currTopics - prevTopics) + let removedTopics = toSeq(prevTopics - currTopics) + + self.edgeFilterSubStates.withValue(shard, state): + state.peers.keepItIf( + self.node.peerManager.switch.peerStore.isConnected(it.peerId) + ) + state.pending.keepItIf(not it.finished) + + if addedTopics.len > 0 or removedTopics.len > 0: + for peer in state.peers: + asyncSpawn self.syncFilterDeltas(peer, shard, addedTopics, removedTopics) + + if currTopics.len == 0: + for fut in state.pending: + if not fut.finished: + await fut.cancelAndWait() + self.edgeFilterSubStates.del(shard) + # invalidates `state` — do not use after this + else: + self.updateShardHealth(shard, state[]) + + let needed = max(0, HealthyThreshold - state.peers.len - state.pending.len) + + if needed > 0: + let tracked = state.peers.mapIt(it.peerId).toHashSet() + state.pendingPeers + var candidates = self.node.peerManager.selectPeers( + filter_common.WakuFilterSubscribeCodec, some(shard) + ) + candidates.keepItIf(it.peerId notin tracked) + + let toDial = min(needed, candidates.len) + + trace "edgeFilterSubLoop: shard reconciliation", + shard = shard, + num_peers = state.peers.len, + num_pending = state.pending.len, + num_needed = needed, + num_available = candidates.len, + toDial = toDial + + for i in 0 ..< toDial: + let fut = self.dialFilterPeer(candidates[i], shard, toSeq(currTopics)) + state.pending.add(fut) + + lastSynced = desired + +proc startEdgeFilterLoops(self: SubscriptionManager): Result[void, string] = + ## Start the edge filter orchestration loops. + ## Caller must ensure this is only called in edge mode (relay nil, filter client present). + self.edgeFilterWakeup = newAsyncEvent() + + self.peerEventListener = WakuPeerEvent.listen( + self.node.brokerCtx, + proc(evt: WakuPeerEvent) {.async: (raises: []), gcsafe.} = + if evt.kind == WakuPeerEventKind.EventDisconnected or + evt.kind == WakuPeerEventKind.EventMetadataUpdated: + self.edgeFilterWakeup.fire() + , + ).valueOr: + return err("Failed to listen to peer events for edge filter: " & error) + + self.edgeFilterSubLoopFut = self.edgeFilterSubLoop() + self.edgeFilterHealthLoopFut = self.edgeFilterHealthLoop() + return ok() + +proc stopEdgeFilterLoops(self: SubscriptionManager) {.async: (raises: []).} = + ## Stop the edge filter orchestration loops and clean up pending futures. + if not isNil(self.edgeFilterSubLoopFut): + await self.edgeFilterSubLoopFut.cancelAndWait() + self.edgeFilterSubLoopFut = nil + + if not isNil(self.edgeFilterHealthLoopFut): + await self.edgeFilterHealthLoopFut.cancelAndWait() + self.edgeFilterHealthLoopFut = nil + + for shard, state in self.edgeFilterSubStates: + for fut in state.pending: + if not fut.finished: + await fut.cancelAndWait() + + WakuPeerEvent.dropListener(self.node.brokerCtx, self.peerEventListener) + +# --------------------------------------------------------------------------- +# SubscriptionManager Lifecycle (calls Edge behavior above) +# +# startSubscriptionManager and stopSubscriptionManager orchestrate both the +# core (relay) and edge (filter) paths, and register/clear broker providers. +# --------------------------------------------------------------------------- + +proc startSubscriptionManager*(self: SubscriptionManager): Result[void, string] = + # Register edge filter broker providers. The shard/content health providers + # in WakuNode query these via the broker as a fallback when relay health is + # not available. If edge mode is not active, these providers simply return + # NOT_SUBSCRIBED / strength 0, which is harmless. + RequestEdgeShardHealth.setProvider( + self.node.brokerCtx, + proc(shard: PubsubTopic): Result[RequestEdgeShardHealth, string] = + self.edgeFilterSubStates.withValue(shard, state): + return ok(RequestEdgeShardHealth(health: state.currentHealth)) + return ok(RequestEdgeShardHealth(health: TopicHealth.NOT_SUBSCRIBED)), + ).isOkOr: + error "Can't set provider for RequestEdgeShardHealth", error = error + + RequestEdgeFilterPeerCount.setProvider( + self.node.brokerCtx, + proc(): Result[RequestEdgeFilterPeerCount, string] = + var minPeers = high(int) + for state in self.edgeFilterSubStates.values: + minPeers = min(minPeers, state.peers.len) + if minPeers == high(int): + minPeers = 0 + return ok(RequestEdgeFilterPeerCount(peerCount: minPeers)), + ).isOkOr: + error "Can't set provider for RequestEdgeFilterPeerCount", error = error + + if self.node.wakuRelay.isNil(): + return self.startEdgeFilterLoops() + + # Core mode: auto-subscribe relay to all shards in autosharding. + if self.node.wakuAutoSharding.isSome(): + let autoSharding = self.node.wakuAutoSharding.get() + let clusterId = autoSharding.clusterId + let numShards = autoSharding.shardCountGenZero + + if numShards > 0: + var clusterPubsubTopics = newSeqOfCap[PubsubTopic](numShards) + + for i in 0 ..< numShards: + let shardObj = RelayShard(clusterId: clusterId, shardId: uint16(i)) + clusterPubsubTopics.add(PubsubTopic($shardObj)) + + self.subscribePubsubTopics(clusterPubsubTopics).isOkOr: + error "Failed to auto-subscribe Relay to cluster shards: ", error = error + else: + info "SubscriptionManager has no AutoSharding configured; skipping auto-subscribe." + + return ok() + +proc stopSubscriptionManager*(self: SubscriptionManager) {.async: (raises: []).} = + if self.node.wakuRelay.isNil(): + await self.stopEdgeFilterLoops() + RequestEdgeShardHealth.clearProvider(self.node.brokerCtx) + RequestEdgeFilterPeerCount.clearProvider(self.node.brokerCtx) diff --git a/waku/node/health_monitor/connection_status.nim b/waku/node/health_monitor/connection_status.nim index 77696130a..68ec9d4be 100644 --- a/waku/node/health_monitor/connection_status.nim +++ b/waku/node/health_monitor/connection_status.nim @@ -2,6 +2,9 @@ import chronos, results, std/strutils, ../../api/types export ConnectionStatus +const HealthyThreshold* = 2 + ## Minimum peers required per service protocol for a "Connected" status (excluding Relay). + proc init*( t: typedesc[ConnectionStatus], strRep: string ): Result[ConnectionStatus, string] = diff --git a/waku/node/health_monitor/node_health_monitor.nim b/waku/node/health_monitor/node_health_monitor.nim index 79bf9f92a..066e7776a 100644 --- a/waku/node/health_monitor/node_health_monitor.nim +++ b/waku/node/health_monitor/node_health_monitor.nim @@ -21,6 +21,7 @@ import node/health_monitor/health_report, node/health_monitor/connection_status, node/health_monitor/protocol_health, + requests/health_requests, ] ## This module is aimed to check the state of the "self" Waku Node @@ -29,9 +30,6 @@ import # if not called, the outcome of randomization procedures will be the same in every run random.randomize() -const HealthyThreshold* = 2 - ## minimum peers required for all services for a Connected status, excluding Relay - type NodeHealthMonitor* = ref object nodeHealth: HealthStatus node: WakuNode @@ -48,7 +46,8 @@ type NodeHealthMonitor* = ref object ## latest known connectivity strength (e.g. connected peer count) metric for each protocol. ## if it doesn't make sense for the protocol in question, this is set to zero. relayObserver: PubSubObserver - peerEventListener: EventWakuPeerListener + peerEventListener: WakuPeerEventListener + shardHealthListener: EventShardTopicHealthChangeListener func getHealth*(report: HealthReport, kind: WakuProtocol): ProtocolHealth = for h in report.protocolsHealth: @@ -198,6 +197,17 @@ proc getFilterClientHealth(hm: NodeHealthMonitor): ProtocolHealth = hm.strength[WakuProtocol.FilterClientProtocol] = 0 return p.notMounted() + if isNil(hm.node.wakuRelay): + let edgeRes = RequestEdgeFilterPeerCount.request(hm.node.brokerCtx) + if edgeRes.isOk(): + let peerCount = edgeRes.get().peerCount + if peerCount > 0: + hm.strength[WakuProtocol.FilterClientProtocol] = peerCount + return p.ready() + else: + error "Failed to request edge filter peer count", error = edgeRes.error + return p.notReady("Failed to request edge filter peer count: " & edgeRes.error) + let peerCount = countCapablePeers(hm, WakuFilterSubscribeCodec) hm.strength[WakuProtocol.FilterClientProtocol] = peerCount @@ -663,14 +673,23 @@ proc startHealthMonitor*(hm: NodeHealthMonitor): Result[void, string] = ) hm.node.wakuRelay.addObserver(hm.relayObserver) - hm.peerEventListener = EventWakuPeer.listen( + hm.peerEventListener = WakuPeerEvent.listen( hm.node.brokerCtx, - proc(evt: EventWakuPeer): Future[void] {.async: (raises: []), gcsafe.} = + proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = ## Recompute health on any peer changing anything (join, leave, identify, metadata update) hm.healthUpdateEvent.fire(), ).valueOr: return err("Failed to subscribe to peer events: " & error) + hm.shardHealthListener = EventShardTopicHealthChange.listen( + hm.node.brokerCtx, + proc( + evt: EventShardTopicHealthChange + ): Future[void] {.async: (raises: []), gcsafe.} = + hm.healthUpdateEvent.fire(), + ).valueOr: + return err("Failed to subscribe to shard health events: " & error) + hm.healthUpdateEvent = newAsyncEvent() hm.healthUpdateEvent.fire() @@ -690,8 +709,8 @@ proc stopHealthMonitor*(hm: NodeHealthMonitor) {.async.} = if not isNil(hm.healthLoopFut): await hm.healthLoopFut.cancelAndWait() - if hm.peerEventListener.id != 0: - EventWakuPeer.dropListener(hm.node.brokerCtx, hm.peerEventListener) + WakuPeerEvent.dropListener(hm.node.brokerCtx, hm.peerEventListener) + EventShardTopicHealthChange.dropListener(hm.node.brokerCtx, hm.shardHealthListener) if not isNil(hm.node.wakuRelay) and not isNil(hm.relayObserver): hm.node.wakuRelay.removeObserver(hm.relayObserver) diff --git a/waku/node/peer_manager/peer_manager.nim b/waku/node/peer_manager/peer_manager.nim index dc0da9624..e3eb8d75b 100644 --- a/waku/node/peer_manager/peer_manager.nim +++ b/waku/node/peer_manager/peer_manager.nim @@ -215,31 +215,34 @@ proc loadFromStorage(pm: PeerManager) {.gcsafe.} = trace "recovered peers from storage", amount = amount -proc selectPeer*( +proc selectPeers*( pm: PeerManager, proto: string, shard: Option[PubsubTopic] = none(PubsubTopic) -): Option[RemotePeerInfo] = - # Selects the best peer for a given protocol - +): seq[RemotePeerInfo] = + ## Returns all peers that support the given protocol (and optionally shard), + ## shuffled randomly. Callers can further filter or pick from this list. var peers = pm.switch.peerStore.getPeersByProtocol(proto) - trace "Selecting peer from peerstore", - protocol = proto, peers, address = cast[uint](pm.switch.peerStore) + trace "Selecting peers from peerstore", + protocol = proto, num_peers = peers.len, address = cast[uint](pm.switch.peerStore) if shard.isSome(): - # Parse the shard from the pubsub topic to get cluster and shard ID let shardInfo = RelayShard.parse(shard.get()).valueOr: trace "Failed to parse shard from pubsub topic", topic = shard.get() - return none(RemotePeerInfo) + return @[] - # Filter peers that support the requested shard - # Check both ENR (if present) and the shards field on RemotePeerInfo peers.keepItIf( - # Check ENR if available (it.enr.isSome() and it.enr.get().containsShard(shard.get())) or - # Otherwise check the shards field directly - (it.shards.len > 0 and it.shards.contains(shardInfo.shardId)) + (it.shards.len > 0 and it.shards.contains(shardInfo.shardId)) ) shuffle(peers) + return peers + +proc selectPeer*( + pm: PeerManager, proto: string, shard: Option[PubsubTopic] = none(PubsubTopic) +): Option[RemotePeerInfo] = + ## Selects a single peer for a given protocol, checking service slots first + ## (for non-relay protocols). + let peers = pm.selectPeers(proto, shard) # No criteria for selecting a peer for WakuRelay, random one if proto == WakuRelayCodec: @@ -742,7 +745,7 @@ proc refreshPeerMetadata(pm: PeerManager, peerId: PeerId) {.async.} = # TODO: should only trigger an event if metadata actually changed # should include the shard subscription delta in the event when # it is a MetadataUpdated event - EventWakuPeer.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventMetadataUpdated) + WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventMetadataUpdated) return info "disconnecting from peer", peerId = peerId, reason = reason @@ -787,7 +790,7 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = asyncSpawn(pm.switch.disconnect(peerId)) peerStore.delete(peerId) - EventWakuPeer.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventConnected) + WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventConnected) if not pm.onConnectionChange.isNil(): # we don't want to await for the callback to finish @@ -804,7 +807,7 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = pm.ipTable.del(ip) break - EventWakuPeer.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventDisconnected) + WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventDisconnected) if not pm.onConnectionChange.isNil(): # we don't want to await for the callback to finish @@ -812,7 +815,7 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = of PeerEventKind.Identified: info "event identified", peerId = peerId - EventWakuPeer.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventIdentified) + WakuPeerEvent.emit(pm.brokerCtx, peerId, WakuPeerEventKind.EventIdentified) peerStore[ConnectionBook][peerId] = connectedness peerStore[DirectionBook][peerId] = direction diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 92528c7b9..506a3e592 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -1,7 +1,7 @@ {.push raises: [].} import - std/[options, tables, strutils, sequtils, os, net, random], + std/[options, tables, strutils, sequtils, os, net, random, sets], chronos, chronicles, metrics, @@ -38,7 +38,6 @@ import waku_store/resume, waku_store_sync, waku_filter_v2, - waku_filter_v2/common as filter_common, waku_filter_v2/client as filter_client, waku_metadata, waku_rendezvous/protocol, @@ -60,7 +59,7 @@ import requests/node_requests, requests/health_requests, events/health_events, - events/peer_events, + events/message_events, ], waku/discovery/waku_kademlia, ./net_config, @@ -95,9 +94,6 @@ const clientId* = "Nimbus Waku v2 node" const WakuNodeVersionString* = "version / git commit hash: " & git_version -const EdgeTopicHealthyThreshold = 2 - ## Lightpush server and filter server requirement for a healthy topic in edge mode - # key and crypto modules different type # TODO: Move to application instance (e.g., `WakuNode2`) @@ -142,10 +138,6 @@ type legacyAppHandlers*: Table[PubsubTopic, WakuRelayHandler] ## Kernel API Relay appHandlers (if any) wakuMix*: WakuMix - edgeTopicsHealth*: Table[PubsubTopic, TopicHealth] - edgeHealthEvent*: AsyncEvent - edgeHealthLoop: Future[void] - peerEventListener*: EventWakuPeerListener kademliaDiscoveryLoop*: Future[void] wakuKademlia*: WakuKademlia @@ -498,52 +490,7 @@ proc updateAnnouncedAddrWithPrimaryIpAddr*(node: WakuNode): Result[void, string] return ok() -proc calculateEdgeTopicHealth(node: WakuNode, shard: PubsubTopic): TopicHealth = - let filterPeers = - node.peerManager.getPeersForShard(filter_common.WakuFilterSubscribeCodec, shard) - let lightpushPeers = - node.peerManager.getPeersForShard(lightpush_protocol.WakuLightPushCodec, shard) - - if filterPeers >= EdgeTopicHealthyThreshold and - lightpushPeers >= EdgeTopicHealthyThreshold: - return TopicHealth.SUFFICIENTLY_HEALTHY - elif filterPeers > 0 and lightpushPeers > 0: - return TopicHealth.MINIMALLY_HEALTHY - - return TopicHealth.UNHEALTHY - -proc loopEdgeHealth(node: WakuNode) {.async.} = - while node.started: - await node.edgeHealthEvent.wait() - node.edgeHealthEvent.clear() - - try: - for shard in node.edgeTopicsHealth.keys: - if not node.wakuRelay.isNil and node.wakuRelay.isSubscribed(shard): - continue - - let oldHealth = node.edgeTopicsHealth.getOrDefault(shard, TopicHealth.UNHEALTHY) - let newHealth = node.calculateEdgeTopicHealth(shard) - if newHealth != oldHealth: - node.edgeTopicsHealth[shard] = newHealth - EventShardTopicHealthChange.emit(node.brokerCtx, shard, newHealth) - except CancelledError: - break - except CatchableError as e: - warn "Error in edge health check", error = e.msg - - # safety cooldown to protect from edge cases - await sleepAsync(100.milliseconds) - proc startProvidersAndListeners*(node: WakuNode) = - node.peerEventListener = EventWakuPeer.listen( - node.brokerCtx, - proc(evt: EventWakuPeer) {.async: (raises: []), gcsafe.} = - node.edgeHealthEvent.fire(), - ).valueOr: - error "Failed to listen to peer events", error = error - return - RequestRelayShard.setProvider( node.brokerCtx, proc( @@ -561,14 +508,23 @@ proc startProvidersAndListeners*(node: WakuNode) = var response: RequestShardTopicsHealth for shard in topics: - var healthStatus = TopicHealth.UNHEALTHY + # Health resolution order: + # 1. Relay topicsHealth (computed from gossipsub mesh state) + # 2. If relay is active but topicsHealth hasn't computed yet, UNHEALTHY + # 3. Otherwise, ask edge filter (via broker; no-op if no provider set) + var healthStatus = TopicHealth.NOT_SUBSCRIBED if not node.wakuRelay.isNil: healthStatus = node.wakuRelay.topicsHealth.getOrDefault(shard, TopicHealth.NOT_SUBSCRIBED) if healthStatus == TopicHealth.NOT_SUBSCRIBED: - healthStatus = node.calculateEdgeTopicHealth(shard) + if not node.wakuRelay.isNil and node.wakuRelay.isSubscribed(shard): + healthStatus = TopicHealth.UNHEALTHY + else: + let edgeRes = RequestEdgeShardHealth.request(node.brokerCtx, shard) + if edgeRes.isOk(): + healthStatus = edgeRes.get().health response.topicHealth.add((shard, healthStatus)) @@ -594,9 +550,10 @@ proc startProvidersAndListeners*(node: WakuNode) = pubsubTopic, TopicHealth.NOT_SUBSCRIBED ) - if topicHealth == TopicHealth.NOT_SUBSCRIBED and - pubsubTopic in node.edgeTopicsHealth: - topicHealth = node.calculateEdgeTopicHealth(pubsubTopic) + if topicHealth == TopicHealth.NOT_SUBSCRIBED: + let edgeRes = RequestEdgeShardHealth.request(node.brokerCtx, pubsubTopic) + if edgeRes.isOk(): + topicHealth = edgeRes.get().health response.contentTopicHealth.add((topic: contentTopic, health: topicHealth)) @@ -605,7 +562,6 @@ proc startProvidersAndListeners*(node: WakuNode) = error "Can't set provider for RequestContentTopicsHealth", error = error proc stopProvidersAndListeners*(node: WakuNode) = - EventWakuPeer.dropListener(node.brokerCtx, node.peerEventListener) RequestRelayShard.clearProvider(node.brokerCtx) RequestContentTopicsHealth.clearProvider(node.brokerCtx) RequestShardTopicsHealth.clearProvider(node.brokerCtx) @@ -658,13 +614,16 @@ proc start*(node: WakuNode) {.async.} = ## The switch will update addresses after start using the addressMapper await node.switch.start() - node.edgeHealthEvent = newAsyncEvent() - node.edgeHealthLoop = loopEdgeHealth(node) + node.started = true + + if not node.wakuFilterClient.isNil(): + node.wakuFilterClient.registerPushHandler( + proc(pubsubTopic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + MessageSeenEvent.emit(node.brokerCtx, pubsubTopic, msg) + ) node.startProvidersAndListeners() - node.started = true - if not zeroPortPresent: updateAnnouncedAddrWithPrimaryIpAddr(node).isOkOr: error "failed update announced addr", error = $error @@ -678,10 +637,6 @@ proc stop*(node: WakuNode) {.async.} = node.stopProvidersAndListeners() - if not node.edgeHealthLoop.isNil: - await node.edgeHealthLoop.cancelAndWait() - node.edgeHealthLoop = nil - await node.switch.stop() node.peerManager.stop() diff --git a/waku/requests/health_requests.nim b/waku/requests/health_requests.nim index 3554922b3..c3a0ce286 100644 --- a/waku/requests/health_requests.nim +++ b/waku/requests/health_requests.nim @@ -37,3 +37,15 @@ RequestBroker: healthStatus*: ProtocolHealth proc signature(protocol: WakuProtocol): Future[Result[RequestProtocolHealth, string]] + +# Get edge filter health for a single shard (set by DeliveryService when edge mode is active) +RequestBroker(sync): + type RequestEdgeShardHealth* = object + health*: TopicHealth + + proc signature(shard: PubsubTopic): Result[RequestEdgeShardHealth, string] + +# Get edge filter confirmed peer count (set by DeliveryService when edge mode is active) +RequestBroker(sync): + type RequestEdgeFilterPeerCount* = object + peerCount*: int diff --git a/waku/waku_core/subscription.nim b/waku/waku_core/subscription.nim index 19f3386ef..8694efa1b 100644 --- a/waku/waku_core/subscription.nim +++ b/waku/waku_core/subscription.nim @@ -1,3 +1,3 @@ -import ./subscription/subscription_manager, ./subscription/push_handler +import ./subscription/push_handler -export subscription_manager, push_handler +export push_handler diff --git a/waku/waku_core/subscription/subscription_manager.nim b/waku/waku_core/subscription/subscription_manager.nim deleted file mode 100644 index ccade763b..000000000 --- a/waku/waku_core/subscription/subscription_manager.nim +++ /dev/null @@ -1,52 +0,0 @@ -{.push raises: [].} - -import std/tables, results, chronicles, chronos - -import ./push_handler, ../topics, ../message - -## Subscription manager -type LegacySubscriptionManager* = object - subscriptions: TableRef[(string, ContentTopic), FilterPushHandler] - -proc init*(T: type LegacySubscriptionManager): T = - LegacySubscriptionManager( - subscriptions: newTable[(string, ContentTopic), FilterPushHandler]() - ) - -proc clear*(m: var LegacySubscriptionManager) = - m.subscriptions.clear() - -proc registerSubscription*( - m: LegacySubscriptionManager, - pubsubTopic: PubsubTopic, - contentTopic: ContentTopic, - handler: FilterPushHandler, -) = - try: - # TODO: Handle over subscription surprises - m.subscriptions[(pubsubTopic, contentTopic)] = handler - except CatchableError: - error "failed to register filter subscription", error = getCurrentExceptionMsg() - -proc removeSubscription*( - m: LegacySubscriptionManager, pubsubTopic: PubsubTopic, contentTopic: ContentTopic -) = - m.subscriptions.del((pubsubTopic, contentTopic)) - -proc notifySubscriptionHandler*( - m: LegacySubscriptionManager, - pubsubTopic: PubsubTopic, - contentTopic: ContentTopic, - message: WakuMessage, -) = - if not m.subscriptions.hasKey((pubsubTopic, contentTopic)): - return - - try: - let handler = m.subscriptions[(pubsubTopic, contentTopic)] - asyncSpawn handler(pubsubTopic, message) - except CatchableError: - discard - -proc getSubscriptionsCount*(m: LegacySubscriptionManager): int = - m.subscriptions.len() diff --git a/waku/waku_filter_v2/client.nim b/waku/waku_filter_v2/client.nim index ba8cd3d0c..265bf5e7b 100644 --- a/waku/waku_filter_v2/client.nim +++ b/waku/waku_filter_v2/client.nim @@ -101,12 +101,20 @@ proc sendSubscribeRequest( return ok() proc ping*( - wfc: WakuFilterClient, servicePeer: RemotePeerInfo + wfc: WakuFilterClient, servicePeer: RemotePeerInfo, timeout = chronos.seconds(0) ): Future[FilterSubscribeResult] {.async.} = info "sending ping", servicePeer = shortLog($servicePeer) let requestId = generateRequestId(wfc.rng) let filterSubscribeRequest = FilterSubscribeRequest.ping(requestId) + if timeout > chronos.seconds(0): + let fut = wfc.sendSubscribeRequest(servicePeer, filterSubscribeRequest) + if not await fut.withTimeout(timeout): + return err( + FilterSubscribeError.parse(uint32(FilterSubscribeErrorKind.PEER_DIAL_FAILURE)) + ) + return fut.read() + return await wfc.sendSubscribeRequest(servicePeer, filterSubscribeRequest) proc subscribe*( diff --git a/waku/waku_relay/protocol.nim b/waku/waku_relay/protocol.nim index 17470af29..490feae87 100644 --- a/waku/waku_relay/protocol.nim +++ b/waku/waku_relay/protocol.nim @@ -157,7 +157,7 @@ type ): Future[ValidationResult] {.gcsafe, raises: [Defect].} WakuRelay* = ref object of GossipSub brokerCtx: BrokerContext - peerEventListener: EventWakuPeerListener + peerEventListener: WakuPeerEventListener # seq of tuples: the first entry in the tuple contains the validators are called for every topic # the second entry contains the error messages to be returned when the validator fails wakuValidators: seq[tuple[handler: WakuValidatorHandler, errorMessage: string]] @@ -376,9 +376,9 @@ proc new*( w.initProtocolHandler() w.initRelayObservers() - w.peerEventListener = EventWakuPeer.listen( + w.peerEventListener = WakuPeerEvent.listen( w.brokerCtx, - proc(evt: EventWakuPeer): Future[void] {.async: (raises: []), gcsafe.} = + proc(evt: WakuPeerEvent): Future[void] {.async: (raises: []), gcsafe.} = if evt.kind == WakuPeerEventKind.EventDisconnected: w.topicHealthCheckAll = true w.topicHealthUpdateEvent.fire() @@ -524,8 +524,7 @@ method stop*(w: WakuRelay) {.async, base.} = info "stop" await procCall GossipSub(w).stop() - if w.peerEventListener.id != 0: - EventWakuPeer.dropListener(w.brokerCtx, w.peerEventListener) + WakuPeerEvent.dropListener(w.brokerCtx, w.peerEventListener) if not w.topicHealthLoopHandle.isNil(): await w.topicHealthLoopHandle.cancelAndWait() From b0c0e0b63746d0f345ed741900aca5a3b2abd721 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Thu, 2 Apr 2026 07:10:02 -0300 Subject: [PATCH 15/19] chore: optimize release builds for speed (#3735) (#3777) * Add -flto (lto_incremental, link-time optimization) for release builds * Add -s (strip symbols) for release builds * Switch library builds from --opt:size to --opt:speed * Change -d:marchOptimized to x86-64-v2 target from broadwell * Remove obsolete chronicles_colors=off for Windows * Remove obsolete withoutPCRE define --- Makefile | 2 +- config.nims | 14 +++++--------- waku.nimble | 8 ++++---- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 8f98e90bd..afd2389d2 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ deps: | deps-common nat-libs waku.nims # "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims ifeq ($(DEBUG), 0) -NIM_PARAMS := $(NIM_PARAMS) -d:release +NIM_PARAMS := $(NIM_PARAMS) -d:release -d:lto_incremental -d:strip else NIM_PARAMS := $(NIM_PARAMS) -d:debug endif diff --git a/config.nims b/config.nims index f74fe183f..0655bf092 100644 --- a/config.nims +++ b/config.nims @@ -26,10 +26,6 @@ if defined(windows): # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM switch("passL", "-Wl,--large-address-aware") - # The dynamic Chronicles output currently prevents us from using colors on Windows - # because these require direct manipulations of the stdout File object. - switch("define", "chronicles_colors=off") - # https://github.com/status-im/nimbus-eth2/blob/stable/docs/cpu_features.md#ssse3-supplemental-sse3 # suggests that SHA256 hashing with SSSE3 is 20% faster than without SSSE3, so # given its near-ubiquity in the x86 installed base, it renders a distribution @@ -52,9 +48,10 @@ if defined(disableMarchNative): switch("passL", "-march=haswell -mtune=generic") else: if defined(marchOptimized): - # https://github.com/status-im/nimbus-eth2/blob/stable/docs/cpu_features.md#bmi2--adx - switch("passC", "-march=broadwell -mtune=generic") - switch("passL", "-march=broadwell -mtune=generic") + # -march=broadwell: https://github.com/status-im/nimbus-eth2/blob/stable/docs/cpu_features.md#bmi2--adx + # Changed to x86-64-v2 for broader support + switch("passC", "-march=x86-64-v2 -mtune=generic") + switch("passL", "-march=x86-64-v2 -mtune=generic") else: switch("passC", "-mssse3") switch("passL", "-mssse3") @@ -76,6 +73,7 @@ else: on --opt: speed + --excessiveStackTrace: on # enable metric collection @@ -85,8 +83,6 @@ else: --define: nimTypeNames -switch("define", "withoutPCRE") - # the default open files limit is too low on macOS (512), breaking the # "--debugger:native" build. It can be increased with `ulimit -n 1024`. if not defined(macosx) and not defined(android): diff --git a/waku.nimble b/waku.nimble index d879bc0e1..cbbe6aa97 100644 --- a/waku.nimble +++ b/waku.nimble @@ -73,11 +73,11 @@ proc buildLibrary(lib_name: string, srcDir = "./", params = "", `type` = "static extra_params &= " " & paramStr(i) if `type` == "static": exec "nim c" & " --out:build/" & lib_name & - " --threads:on --app:staticlib --opt:size --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:on -d:discv5_protocol_id=d5waku " & + " --threads:on --app:staticlib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:on -d:discv5_protocol_id=d5waku " & extra_params & " " & srcDir & srcFile else: exec "nim c" & " --out:build/" & lib_name & - " --threads:on --app:lib --opt:size --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:off -d:discv5_protocol_id=d5waku " & + " --threads:on --app:lib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:off -d:discv5_protocol_id=d5waku " & extra_params & " " & srcDir & srcFile proc buildMobileAndroid(srcDir = ".", params = "") = @@ -93,7 +93,7 @@ proc buildMobileAndroid(srcDir = ".", params = "") = extra_params &= " " & paramStr(i) exec "nim c" & " --out:" & outDir & - "/libwaku.so --threads:on --app:lib --opt:size --noMain --mm:refc -d:chronicles_sinks=textlines[dynamic] --header -d:chronosEventEngine=epoll --passL:-L" & + "/libwaku.so --threads:on --app:lib --opt:speed --noMain --mm:refc -d:chronicles_sinks=textlines[dynamic] --header -d:chronosEventEngine=epoll --passL:-L" & outdir & " --passL:-lrln --passL:-llog --cpu:" & cpu & " --os:android -d:androidNDK " & extra_params & " " & srcDir & "/libwaku.nim" @@ -266,7 +266,7 @@ proc buildMobileIOS(srcDir = ".", params = "") = " --os:ios --cpu:" & cpu & " --compileOnly:on" & " --noMain --mm:refc" & - " --threads:on --opt:size --header" & + " --threads:on --opt:speed --header" & " -d:metrics -d:discv5_protocol_id=d5waku" & " --nimMainPrefix:libwaku --skipParentCfg:on" & " --cc:clang" & From 39719e124770d62346ad79cdc1aaad165462e445 Mon Sep 17 00:00:00 2001 From: Darshan <35736874+darshankabariya@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:53:45 +0530 Subject: [PATCH 16/19] increase default timeout to 20s and add debug logging (#3792) --- apps/wakucanary/wakucanary.nim | 40 +++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/apps/wakucanary/wakucanary.nim b/apps/wakucanary/wakucanary.nim index bb68f7237..e7b1ff9aa 100644 --- a/apps/wakucanary/wakucanary.nim +++ b/apps/wakucanary/wakucanary.nim @@ -6,12 +6,15 @@ import os import libp2p/protocols/ping, + libp2p/protocols/protocol, libp2p/crypto/[crypto, secp], libp2p/nameresolving/dnsresolver, libp2p/multicodec import ./certsgenerator, - waku/[waku_enr, node/peer_manager, waku_core, waku_node, factory/builder] + waku/[waku_enr, node/peer_manager, waku_core, waku_node, factory/builder], + waku/waku_metadata/protocol, + waku/common/callbacks # protocols and their tag const ProtocolsTable = { @@ -45,7 +48,7 @@ type WakuCanaryConf* = object timeout* {. desc: "Timeout to consider that the connection failed", - defaultValue: chronos.seconds(10), + defaultValue: chronos.seconds(20), name: "timeout", abbr: "t" .}: chronos.Duration @@ -251,12 +254,26 @@ proc main(rng: ref HmacDrbgContext): Future[int] {.async.} = error "failed to mount libp2p ping protocol: " & getCurrentExceptionMsg() quit(QuitFailure) - node.mountMetadata(conf.clusterId, conf.shards).isOkOr: - error "failed to mount metadata protocol", error + # Mount metadata with a custom getter that returns CLI shards directly, + # since the canary doesn't mount relay (which is what the default getter reads from). + # Without this fix, the canary always sends remoteShards=[] in metadata requests. + let cliShards = conf.shards + let shardsGetter: GetShards = proc(): seq[uint16] {.closure, gcsafe, raises: [].} = + return cliShards + + let metadata = WakuMetadata.new(conf.clusterId, shardsGetter) + node.wakuMetadata = metadata + node.peerManager.wakuMetadata = metadata + let mountRes = catch: + node.switch.mount(metadata, protocolMatcher(WakuMetadataCodec)) + mountRes.isOkOr: + error "failed to mount metadata protocol", error = error.msg quit(QuitFailure) await node.start() + debug "Connecting to peer", peer = peer, timeout = conf.timeout + var pingFut: Future[bool] if conf.ping: pingFut = pingNode(node, peer).withTimeout(conf.timeout) @@ -266,8 +283,18 @@ proc main(rng: ref HmacDrbgContext): Future[int] {.async.} = error "Timedout after", timeout = conf.timeout quit(QuitFailure) + # Clean disconnect with defer so the remote node doesn't see + # "Stream Underlying Connection Closed!" when we exit + defer: + debug "Cleanly disconnecting from peer", peerId = peer.peerId + await node.peerManager.disconnectNode(peer.peerId) + await node.stop() + + debug "Connected, checking connection status", peerId = peer.peerId + let lp2pPeerStore = node.switch.peerStore let conStatus = node.peerManager.switch.peerStore[ConnectionBook][peer.peerId] + debug "Connection status", peerId = peer.peerId, conStatus = conStatus var pingSuccess = true if conf.ping: @@ -283,14 +310,15 @@ proc main(rng: ref HmacDrbgContext): Future[int] {.async.} = if conStatus in [Connected, CanConnect]: let nodeProtocols = lp2pPeerStore[ProtoBook][peer.peerId] + debug "Peer protocols", peerId = peer.peerId, protocols = nodeProtocols if not areProtocolsSupported(conf.protocols, nodeProtocols): error "Not all protocols are supported", expected = conf.protocols, supported = nodeProtocols - quit(QuitFailure) + return 1 elif conStatus == CannotConnect: error "Could not connect", peerId = peer.peerId - quit(QuitFailure) + return 1 return 0 when isMainModule: From 56359e49ed60cb78c03f5e9c1519e43fe90deaab Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Mon, 6 Apr 2026 11:08:47 -0300 Subject: [PATCH 17/19] prefer reusing service peers across shards in edge filter reconciliation (#3789) * selectFilterCandidates prefers peers already serving other shards * restructure edgeFilterSubLoop (plan all dials then execute) for safety --- .../delivery_service/subscription_manager.nim | 78 +++++++++++++++---- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/waku/node/delivery_service/subscription_manager.nim b/waku/node/delivery_service/subscription_manager.nim index 70d8df7f0..f00d9024c 100644 --- a/waku/node/delivery_service/subscription_manager.nim +++ b/waku/node/delivery_service/subscription_manager.nim @@ -115,7 +115,7 @@ proc subscribePubsubTopics( if isNil(self.node.wakuRelay): return err("subscribePubsubTopics requires a Relay") - var errors: seq[string] = @[] + var errors: seq[string] for shard in shards: if not self.contentTopicSubs.hasKey(shard): @@ -196,6 +196,11 @@ const EdgeFilterLoopInterval = chronos.seconds(30) const EdgeFilterSubLoopDebounce = chronos.seconds(1) ## Debounce delay to coalesce rapid-fire wakeups into a single reconciliation pass. +type EdgeDialTask = object + peer: RemotePeerInfo + shard: PubsubTopic + topics: seq[ContentTopic] + proc updateShardHealth( self: SubscriptionManager, shard: PubsubTopic, state: var EdgeFilterSubState ) = @@ -335,7 +340,7 @@ proc edgeFilterHealthLoop*(self: SubscriptionManager) {.async.} = var alive = initHashSet[PeerId]() if connected.len > 0: - var pingTasks: seq[(PeerId, Future[FilterSubscribeResult])] = @[] + var pingTasks: seq[(PeerId, Future[FilterSubscribeResult])] for peer in connected.values: pingTasks.add( (peer.peerId, self.node.wakuFilterClient.ping(peer, EdgeFilterPingTimeout)) @@ -362,6 +367,36 @@ proc edgeFilterHealthLoop*(self: SubscriptionManager) {.async.} = if changed: self.edgeFilterWakeup.fire() +proc selectFilterCandidates( + self: SubscriptionManager, shard: PubsubTopic, exclude: HashSet[PeerId], needed: int +): seq[RemotePeerInfo] = + ## Select filter service peer candidates for a shard. + + # Start with every filter server peer that can serve the shard + var allCandidates = self.node.peerManager.selectPeers( + filter_common.WakuFilterSubscribeCodec, some(shard) + ) + + # Remove all already used in this shard or being dialed for it + allCandidates.keepItIf(it.peerId notin exclude) + + # Collect peer IDs already tracked on other shards + var trackedOnOther = initHashSet[PeerId]() + for otherShard, otherState in self.edgeFilterSubStates.pairs: + if otherShard != shard: + for peer in otherState.peers: + trackedOnOther.incl(peer.peerId) + + # Prefer peers we already have a connection to first, preserving shuffle + var candidates = + allCandidates.filterIt(it.peerId in trackedOnOther) & + allCandidates.filterIt(it.peerId notin trackedOnOther) + + # We need to return 'needed' peers only + if candidates.len > needed: + candidates.setLen(needed) + return candidates + proc edgeFilterSubLoop*(self: SubscriptionManager) {.async.} = ## Reconciles filter subscriptions with the desired state from SubscriptionManager. var lastSynced = initTable[PubsubTopic, HashSet[ContentTopic]]() @@ -382,6 +417,12 @@ proc edgeFilterSubLoop*(self: SubscriptionManager) {.async.} = let allShards = toHashSet(toSeq(desired.keys)) + toHashSet(toSeq(lastSynced.keys)) + # Step 1: read state across all shards at once and + # create a list of peer dial tasks and shard tracking to delete. + + var dialTasks: seq[EdgeDialTask] + var shardsToDelete: seq[PubsubTopic] + for shard in allShards: let currTopics = desired.getOrDefault(shard) let prevTopics = lastSynced.getOrDefault(shard) @@ -404,11 +445,7 @@ proc edgeFilterSubLoop*(self: SubscriptionManager) {.async.} = asyncSpawn self.syncFilterDeltas(peer, shard, addedTopics, removedTopics) if currTopics.len == 0: - for fut in state.pending: - if not fut.finished: - await fut.cancelAndWait() - self.edgeFilterSubStates.del(shard) - # invalidates `state` — do not use after this + shardsToDelete.add(shard) else: self.updateShardHealth(shard, state[]) @@ -416,11 +453,7 @@ proc edgeFilterSubLoop*(self: SubscriptionManager) {.async.} = if needed > 0: let tracked = state.peers.mapIt(it.peerId).toHashSet() + state.pendingPeers - var candidates = self.node.peerManager.selectPeers( - filter_common.WakuFilterSubscribeCodec, some(shard) - ) - candidates.keepItIf(it.peerId notin tracked) - + let candidates = self.selectFilterCandidates(shard, tracked, needed) let toDial = min(needed, candidates.len) trace "edgeFilterSubLoop: shard reconciliation", @@ -432,8 +465,25 @@ proc edgeFilterSubLoop*(self: SubscriptionManager) {.async.} = toDial = toDial for i in 0 ..< toDial: - let fut = self.dialFilterPeer(candidates[i], shard, toSeq(currTopics)) - state.pending.add(fut) + dialTasks.add( + EdgeDialTask( + peer: candidates[i], shard: shard, topics: toSeq(currTopics) + ) + ) + + # Step 2: execute deferred shard tracking deletion and dial tasks. + + for shard in shardsToDelete: + self.edgeFilterSubStates.withValue(shard, state): + for fut in state.pending: + if not fut.finished: + await fut.cancelAndWait() + self.edgeFilterSubStates.del(shard) + + for task in dialTasks: + let fut = self.dialFilterPeer(task.peer, task.shard, task.topics) + self.edgeFilterSubStates.withValue(task.shard, state): + state.pending.add(fut) lastSynced = desired From 549bf8bc4309aadb30530a181fa1933bced2c9a7 Mon Sep 17 00:00:00 2001 From: Danish Arora <35004822+danisharora099@users.noreply.github.com> Date: Tue, 7 Apr 2026 13:14:32 +0530 Subject: [PATCH 18/19] fix(nix): fetch git submodules automatically via inputs.self (#3738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Nix build fails when consumers use `nix build github:logos-messaging/logos-delivery#liblogosdelivery` without appending `?submodules=1` — vendor/nimbus-build-system is missing, causing patchShebangs and substituteInPlace to fail. Two fixes: 1. Add `inputs.self.submodules = true` to flake.nix (Nix >= 2.27) so submodules are fetched automatically without requiring callers to pass `?submodules=1`. 2. Fix the assertion in nix/default.nix: `(src.submodules or true)` always evaluates to true, silently masking the missing-submodules error. Changed to `builtins.pathExists` check on the actual submodule directory so it fails with a helpful message when submodules are genuinely absent. --- flake.nix | 4 ++++ nix/default.nix | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ee24c8f13..13ca5e618 100644 --- a/flake.nix +++ b/flake.nix @@ -7,6 +7,10 @@ }; inputs = { + # Ensure Nix fetches git submodules (vendor/*) when evaluating this flake. + # Requires Nix >= 2.27. Consumers no longer need '?submodules=1' in the URL. + self.submodules = true; + # We are pinning the commit because ultimately we want to use same commit across different projects. # A commit from nixpkgs 24.11 release : https://github.com/NixOS/nixpkgs/tree/release-24.11 nixpkgs.url = "github:NixOS/nixpkgs/0ef228213045d2cdb5a169a95d63ded38670b293"; diff --git a/nix/default.nix b/nix/default.nix index 7df58df60..816d0aed8 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -12,7 +12,7 @@ zerokitRln, }: -assert pkgs.lib.assertMsg ((src.submodules or true) == true) +assert pkgs.lib.assertMsg (builtins.pathExists "${src}/vendor/nimbus-build-system/scripts") "Unable to build without submodules. Append '?submodules=1#' to the URI."; let From 9a344553e765775bb9859b58d0f5e284c51e9a98 Mon Sep 17 00:00:00 2001 From: Darshan <35736874+darshankabariya@users.noreply.github.com> Date: Tue, 7 Apr 2026 21:30:57 +0530 Subject: [PATCH 19/19] chore: update master changelog after v0.37.4 (#3802) --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be2b795f6..ac15321c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.37.4 (2026-04-03) + +### Changes + +- Optimize release builds for speed ([#3735](https://github.com/logos-messaging/logos-delivery/pull/3735)) ([#3777](https://github.com/logos-messaging/logos-delivery/pull/3777)) + +### Bug Fixes + +- Properly add DEBUG flag into Dockerfile + ## v0.37.3 (2026-03-25) ### Features